diff --git a/make/CompileJavaModules.gmk b/make/CompileJavaModules.gmk index 38849f760e0..b0219d96fb9 100644 --- a/make/CompileJavaModules.gmk +++ b/make/CompileJavaModules.gmk @@ -460,6 +460,7 @@ jdk.internal.vm.compiler_EXCLUDES += \ org.graalvm.compiler.core.test \ org.graalvm.compiler.debug.test \ org.graalvm.compiler.graph.test \ + org.graalvm.compiler.hotspot.aarch64.test \ org.graalvm.compiler.hotspot.amd64.test \ org.graalvm.compiler.hotspot.jdk9.test \ org.graalvm.compiler.hotspot.lir.test \ diff --git a/make/test/JtregGraalUnit.gmk b/make/test/JtregGraalUnit.gmk index 96939566ab9..69fc28c79d8 100644 --- a/make/test/JtregGraalUnit.gmk +++ b/make/test/JtregGraalUnit.gmk @@ -91,6 +91,7 @@ ifeq ($(INCLUDE_GRAAL), true) $(SRC_DIR)/org.graalvm.compiler.core.test/src \ $(SRC_DIR)/org.graalvm.compiler.debug.test/src \ $(SRC_DIR)/org.graalvm.compiler.graph.test/src \ + $(SRC_DIR)/org.graalvm.compiler.hotspot.aarch64.test/src \ $(SRC_DIR)/org.graalvm.compiler.hotspot.amd64.test/src \ $(SRC_DIR)/org.graalvm.compiler.hotspot.jdk9.test/src \ $(SRC_DIR)/org.graalvm.compiler.hotspot.lir.test/src \ diff --git a/src/jdk.internal.vm.compiler/share/classes/jdk.internal.vm.compiler.collections/src/jdk/internal/vm/compiler/collections/package-info.java b/src/jdk.internal.vm.compiler/share/classes/jdk.internal.vm.compiler.collections/src/jdk/internal/vm/compiler/collections/package-info.java index 061262bd1c6..595fd630e6a 100644 --- a/src/jdk.internal.vm.compiler/share/classes/jdk.internal.vm.compiler.collections/src/jdk/internal/vm/compiler/collections/package-info.java +++ b/src/jdk.internal.vm.compiler/share/classes/jdk.internal.vm.compiler.collections/src/jdk/internal/vm/compiler/collections/package-info.java @@ -22,11 +22,11 @@ */ /* @ApiInfo( - group="Graal SDK" + group="GraalVM SDK" ) */ /** - * The Graal-SDK collections package contains memory efficient data structures. + * The GraalVM SDK collections package contains memory efficient data structures. * * @see jdk.internal.vm.compiler.collections.EconomicMap * @see jdk.internal.vm.compiler.collections.EconomicSet diff --git a/src/jdk.internal.vm.compiler/share/classes/jdk.internal.vm.compiler.libgraal/src/jdk/internal/vm/compiler/libgraal/LibGraal.java b/src/jdk.internal.vm.compiler/share/classes/jdk.internal.vm.compiler.libgraal/src/jdk/internal/vm/compiler/libgraal/LibGraal.java new file mode 100644 index 00000000000..6981f5d697a --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/jdk.internal.vm.compiler.libgraal/src/jdk/internal/vm/compiler/libgraal/LibGraal.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package jdk.internal.vm.compiler.libgraal; + +import jdk.vm.ci.hotspot.HotSpotJVMCIRuntime; +import jdk.vm.ci.services.Services; + +/** + * JDK13+ version of {@code LibGraal}. + */ +public class LibGraal { + + public static boolean isAvailable() { + return isCurrentRuntime() || libgraalIsolate != 0L; + } + + public static boolean isCurrentRuntime() { + return Services.IS_IN_NATIVE_IMAGE; + } + + public static long getIsolate() { + if (isCurrentRuntime() || !isAvailable()) { + throw new IllegalStateException(); + } + return libgraalIsolate; + } + + public static long getIsolateThread() { + if (isCurrentRuntime()) { + throw new IllegalStateException(); + } + return CURRENT_ISOLATE_THREAD.get(); + } + + @SuppressWarnings("unused") + public static long[] registerNativeMethods(HotSpotJVMCIRuntime runtime, Class clazz) { + if (clazz.isPrimitive()) { + throw new IllegalArgumentException(); + } + if (isCurrentRuntime() || !isAvailable()) { + throw new IllegalStateException(); + } + // Waiting for https://bugs.openjdk.java.net/browse/JDK-8220623 + // return runtime.registerNativeMethods(clazz); + throw new IllegalStateException("Requires JDK-8220623"); + } + + @SuppressWarnings("unused") + public static long translate(HotSpotJVMCIRuntime runtime, Object obj) { + if (!isAvailable()) { + throw new IllegalStateException(); + } + // return runtime.translate(obj); + throw new IllegalStateException("Requires JDK-8220623"); + } + + @SuppressWarnings("unused") + public static T unhand(HotSpotJVMCIRuntime runtime, Class type, long handle) { + if (!isAvailable()) { + throw new IllegalStateException(); + } + // return runtime.unhand(type, handle); + throw new IllegalStateException("Requires JDK-8220623"); + } + + private static final ThreadLocal CURRENT_ISOLATE_THREAD = new ThreadLocal<>() { + @Override + protected Long initialValue() { + return attachThread(libgraalIsolate); + } + }; + + private static final long libgraalIsolate = Services.IS_BUILDING_NATIVE_IMAGE ? 0L : initializeLibgraal(); + + private static long initializeLibgraal() { + try { + // Initialize JVMCI to ensure JVMCI opens its packages to + // Graal otherwise the call to HotSpotJVMCIRuntime.runtime() + // below will fail on JDK13+. + Services.initializeJVMCI(); + + // Waiting for https://bugs.openjdk.java.net/browse/JDK-8220623 + // HotSpotJVMCIRuntime runtime = HotSpotJVMCIRuntime.runtime(); + // long[] nativeInterface = runtime.registerNativeMethods(LibGraal.class); + // return nativeInterface[1]; + return 0L; + } catch (UnsupportedOperationException e) { + return 0L; + } + } + + /** + * Attaches the current thread to a thread in {@code isolate}. + * + * @param isolate + */ + private static native long attachThread(long isolate); +} diff --git a/src/jdk.internal.vm.compiler/share/classes/jdk.internal.vm.compiler.libgraal/src/jdk/internal/vm/compiler/libgraal/OptionsEncoder.java b/src/jdk.internal.vm.compiler/share/classes/jdk.internal.vm.compiler.libgraal/src/jdk/internal/vm/compiler/libgraal/OptionsEncoder.java new file mode 100644 index 00000000000..e9c16ff69d4 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/jdk.internal.vm.compiler.libgraal/src/jdk/internal/vm/compiler/libgraal/OptionsEncoder.java @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package jdk.internal.vm.compiler.libgraal; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Facilities for encoding/decoding a set of options to/from a byte array. + */ +public final class OptionsEncoder { + + private OptionsEncoder() { + } + + /** + * Determines if {@code value} is supported by {@link #encode(Map)}. + */ + public static boolean isValueSupported(Object value) { + if (value == null) { + return false; + } + Class valueClass = value.getClass(); + return valueClass == Boolean.class || + valueClass == Byte.class || + valueClass == Short.class || + valueClass == Character.class || + valueClass == Integer.class || + valueClass == Long.class || + valueClass == Float.class || + valueClass == Double.class || + valueClass == String.class || + value.getClass().isEnum(); + } + + /** + * Encodes {@code options} into a byte array. + * + * @throws IllegalArgumentException if any value in {@code options} is not + * {@linkplain #isValueSupported(Object) supported} + */ + public static byte[] encode(final Map options) { + try (ByteArrayOutputStream baout = new ByteArrayOutputStream()) { + try (DataOutputStream out = new DataOutputStream(baout)) { + out.writeInt(options.size()); + for (Map.Entry e : options.entrySet()) { + final String key = e.getKey(); + out.writeUTF(key); + final Object value = e.getValue(); + final Class valueClz = value.getClass(); + if (valueClz == Boolean.class) { + out.writeByte('Z'); + out.writeBoolean((Boolean) value); + } else if (valueClz == Byte.class) { + out.writeByte('B'); + out.writeByte((Byte) value); + } else if (valueClz == Short.class) { + out.writeByte('S'); + out.writeShort((Short) value); + } else if (valueClz == Character.class) { + out.writeByte('C'); + out.writeChar((Character) value); + } else if (valueClz == Integer.class) { + out.writeByte('I'); + out.writeInt((Integer) value); + } else if (valueClz == Long.class) { + out.writeByte('J'); + out.writeLong((Long) value); + } else if (valueClz == Float.class) { + out.writeByte('F'); + out.writeFloat((Float) value); + } else if (valueClz == Double.class) { + out.writeByte('D'); + out.writeDouble((Double) value); + } else if (valueClz == String.class) { + out.writeByte('U'); + out.writeUTF((String) value); + } else if (valueClz.isEnum()) { + out.writeByte('U'); + out.writeUTF(((Enum) value).name()); + } else { + throw new IllegalArgumentException(String.format("Key: %s, Value: %s, Value type: %s", key, value, valueClz)); + } + } + } + return baout.toByteArray(); + } catch (IOException ioe) { + throw new IllegalArgumentException(ioe); + } + } + + /** + * Decodes {@code input} into a name/value map. + * + * @throws IllegalArgumentException if {@code input} cannot be decoded + */ + public static Map decode(byte[] input) { + Map res = new HashMap<>(); + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(input))) { + final int size = in.readInt(); + for (int i = 0; i < size; i++) { + final String key = in.readUTF(); + final Object value; + final byte type = in.readByte(); + switch (type) { + case 'Z': + value = in.readBoolean(); + break; + case 'B': + value = in.readByte(); + break; + case 'S': + value = in.readShort(); + break; + case 'C': + value = in.readChar(); + break; + case 'I': + value = in.readInt(); + break; + case 'J': + value = in.readLong(); + break; + case 'F': + value = in.readFloat(); + break; + case 'D': + value = in.readDouble(); + break; + case 'U': + value = in.readUTF(); + break; + default: + throw new IllegalArgumentException("Unsupported value type: " + Integer.toHexString(type)); + } + res.put(key, value); + } + } catch (IOException ioe) { + throw new IllegalArgumentException(ioe); + } + return res; + } +} + diff --git a/src/jdk.internal.vm.compiler/share/classes/jdk.internal.vm.compiler.word/src/jdk/internal/vm/compiler/word/package-info.java b/src/jdk.internal.vm.compiler/share/classes/jdk.internal.vm.compiler.word/src/jdk/internal/vm/compiler/word/package-info.java index caecc6a078d..80626aa6283 100644 --- a/src/jdk.internal.vm.compiler/share/classes/jdk.internal.vm.compiler.word/src/jdk/internal/vm/compiler/word/package-info.java +++ b/src/jdk.internal.vm.compiler/share/classes/jdk.internal.vm.compiler.word/src/jdk/internal/vm/compiler/word/package-info.java @@ -22,7 +22,7 @@ */ /* @ApiInfo( - group="Graal SDK" + group="GraalVM SDK" ) */ /** diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64.test/src/org/graalvm/compiler/asm/aarch64/test/AArch64BitCountAssemblerTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64.test/src/org/graalvm/compiler/asm/aarch64/test/AArch64BitCountAssemblerTest.java new file mode 100644 index 00000000000..8df3c3132c7 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64.test/src/org/graalvm/compiler/asm/aarch64/test/AArch64BitCountAssemblerTest.java @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, Arm Limited. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.asm.aarch64.test; + +import static jdk.vm.ci.code.ValueUtil.asRegister; +import static org.junit.Assume.assumeTrue; + +import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler; +import org.graalvm.compiler.asm.test.AssemblerTest; +import org.graalvm.compiler.code.CompilationResult; +import org.junit.Before; +import org.junit.Test; + +import jdk.vm.ci.aarch64.AArch64; +import jdk.vm.ci.aarch64.AArch64Kind; +import jdk.vm.ci.code.CallingConvention; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.code.RegisterArray; +import jdk.vm.ci.code.RegisterConfig; +import jdk.vm.ci.code.TargetDescription; +import jdk.vm.ci.meta.JavaKind; + +public class AArch64BitCountAssemblerTest extends AssemblerTest { + @Before + public void checkAArch64() { + assumeTrue("skipping non AArch64 specific test", codeCache.getTarget().arch instanceof AArch64); + } + + public interface AArch64CodeGenTestCase { + CodeGenTest create(); + + int getExpected(); + } + + private class AArch64BitCountCodeGenTestCase implements AArch64CodeGenTestCase { + final T value; + final int size; + + AArch64BitCountCodeGenTestCase(T x, int size) { + assert x instanceof Integer || x instanceof Long; + this.value = x; + this.size = size; + } + + T getValue() { + return value; + } + + @Override + public CodeGenTest create() { + return (CompilationResult compResult, TargetDescription target, RegisterConfig registerConfig, CallingConvention cc) -> { + AArch64MacroAssembler masm = new AArch64MacroAssembler(target); + Register dst = registerConfig.getReturnRegister(JavaKind.Int); + Register src = asRegister(cc.getArgument(0)); + RegisterArray registers = registerConfig.filterAllocatableRegisters(AArch64Kind.V64_BYTE, registerConfig.getAllocatableRegisters()); + masm.popcnt(size, dst, src, registers.get(registers.size() - 1)); + masm.ret(AArch64.lr); + return masm.close(true); + }; + } + + @Override + public int getExpected() { + if (value instanceof Integer) { + return Integer.bitCount((Integer) value); + } else if (value instanceof Long) { + return Long.bitCount((Long) value); + } + return -1; + } + } + + @Test + @SuppressWarnings("unchecked") + public void testBitCount() { + AArch64CodeGenTestCase[] tests = { + new AArch64BitCountCodeGenTestCase<>(0, JavaKind.Int.getByteCount() * Byte.SIZE), + new AArch64BitCountCodeGenTestCase<>(1522767384, JavaKind.Int.getByteCount() * Byte.SIZE), + new AArch64BitCountCodeGenTestCase<>(0L, JavaKind.Long.getByteCount() * Byte.SIZE), + new AArch64BitCountCodeGenTestCase<>(81985529216486895L, JavaKind.Long.getByteCount() * Byte.SIZE), + }; + + assertReturn("intStub", tests[0].create(), tests[0].getExpected(), ((AArch64BitCountCodeGenTestCase) tests[0]).getValue()); + assertReturn("intStub", tests[1].create(), tests[1].getExpected(), ((AArch64BitCountCodeGenTestCase) tests[1]).getValue()); + assertReturn("longStub", tests[2].create(), tests[2].getExpected(), ((AArch64BitCountCodeGenTestCase) tests[2]).getValue()); + assertReturn("longStub", tests[3].create(), tests[3].getExpected(), ((AArch64BitCountCodeGenTestCase) tests[3]).getValue()); + } + + @SuppressWarnings("unused") + public static int intStub(int x) { + return 0; + } + + @SuppressWarnings("unused") + public static int longStub(long x) { + return 0; + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64.test/src/org/graalvm/compiler/asm/aarch64/test/AArch64InstructionEncodingTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64.test/src/org/graalvm/compiler/asm/aarch64/test/AArch64InstructionEncodingTest.java new file mode 100644 index 00000000000..e984b50a880 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64.test/src/org/graalvm/compiler/asm/aarch64/test/AArch64InstructionEncodingTest.java @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, Arm Limited. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.asm.aarch64.test; + +import static org.junit.Assume.assumeTrue; + +import java.nio.ByteBuffer; + +import org.graalvm.compiler.asm.aarch64.AArch64Assembler; +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.test.GraalTest; +import org.junit.Before; +import org.junit.Test; + +import jdk.vm.ci.aarch64.AArch64; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.code.TargetDescription; +import jdk.vm.ci.runtime.JVMCI; + +public class AArch64InstructionEncodingTest extends GraalTest { + @Before + public void checkAArch64() { + assumeTrue("skipping non AArch64 specific test", JVMCI.getRuntime().getHostJVMCIBackend().getTarget().arch instanceof AArch64); + } + + private abstract class AArch64InstructionEncodingTestCase { + private byte[] actual; + private byte[] expected; + TestProtectedAssembler assembler; + + AArch64InstructionEncodingTestCase(int expected) { + this.expected = ByteBuffer.allocate(Integer.BYTES).putInt(expected).array(); + TargetDescription target = JVMCI.getRuntime().getHostJVMCIBackend().getTarget(); + assembler = new TestProtectedAssembler(target); + } + + int getExpected() { + return ByteBuffer.wrap(expected).getInt(); + } + + int getActual() { + return ByteBuffer.wrap(actual).getInt(); + } + + void closeAssembler() { + this.actual = assembler.close(true); + } + } + + private class CntEncodingTestCase extends AArch64InstructionEncodingTestCase { + CntEncodingTestCase(int expected, int size, Register dst, Register src) { + super(expected); + assembler.cnt(size, dst, src); + closeAssembler(); + } + } + + private class AddvEncodingTestCase extends AArch64InstructionEncodingTestCase { + AddvEncodingTestCase(int expected, int size, AArch64Assembler.SIMDElementSize laneWidth, Register dst, Register src) { + super(expected); + assembler.addv(size, laneWidth, dst, src); + closeAssembler(); + } + } + + private class UmovEncodingTestCase extends AArch64InstructionEncodingTestCase { + UmovEncodingTestCase(int expected, int size, Register dst, int srcIdx, Register src) { + super(expected); + assembler.umov(size, dst, srcIdx, src); + closeAssembler(); + } + } + + private static final int invalidInstructionCode = 0x00000000; + + private void assertWrapper(AArch64InstructionEncodingTestCase testCase) { + assertDeepEquals(testCase.getActual(), testCase.getExpected()); + } + + @Test + public void testCnt() { + assertWrapper(new CntEncodingTestCase(0x0058200e, 64, AArch64.v0, AArch64.v0)); + assertWrapper(new CntEncodingTestCase(0x3f58204e, 128, AArch64.v31, AArch64.v1)); + } + + @Test(expected = AssertionError.class) + @SuppressWarnings("unused") + public void testCntWithInvalidDataSize() { + new CntEncodingTestCase(invalidInstructionCode, 32, AArch64.v5, AArch64.v5); + } + + @Test + public void testAddv() { + assertWrapper(new AddvEncodingTestCase(0x20b8310e, 64, AArch64Assembler.SIMDElementSize.Byte, AArch64.v0, AArch64.v1)); + assertWrapper(new AddvEncodingTestCase(0x42b8314e, 128, AArch64Assembler.SIMDElementSize.Byte, AArch64.v2, AArch64.v2)); + assertWrapper(new AddvEncodingTestCase(0xd2ba710e, 64, AArch64Assembler.SIMDElementSize.HalfWord, AArch64.v18, AArch64.v22)); + assertWrapper(new AddvEncodingTestCase(0x77ba714e, 128, AArch64Assembler.SIMDElementSize.HalfWord, AArch64.v23, AArch64.v19)); + assertWrapper(new AddvEncodingTestCase(0x18bbb14e, 128, AArch64Assembler.SIMDElementSize.Word, AArch64.v24, AArch64.v24)); + } + + @Test(expected = AssertionError.class) + @SuppressWarnings("unused") + public void testAddvWithInvalidSizeLaneCombo() { + new AddvEncodingTestCase(invalidInstructionCode, 64, AArch64Assembler.SIMDElementSize.Word, AArch64.v0, AArch64.v1); + } + + @Test(expected = AssertionError.class) + @SuppressWarnings("unused") + public void testAddvWithInvalidDataSize() { + new AddvEncodingTestCase(invalidInstructionCode, 32, AArch64Assembler.SIMDElementSize.Word, AArch64.v0, AArch64.v1); + } + + @Test(expected = AssertionError.class) + @SuppressWarnings("unused") + public void testAddvWithInvalidLane() { + new AddvEncodingTestCase(invalidInstructionCode, 128, AArch64Assembler.SIMDElementSize.DoubleWord, AArch64.v0, AArch64.v1); + } + + @Test + public void testUmov() { + assertWrapper(new UmovEncodingTestCase(0x1f3c084e, 64, AArch64.r31, 0, AArch64.v0)); + assertWrapper(new UmovEncodingTestCase(0xe13f184e, 64, AArch64.r1, 1, AArch64.v31)); + + assertWrapper(new UmovEncodingTestCase(0x5d3c040e, 32, AArch64.r29, 0, AArch64.v2)); + assertWrapper(new UmovEncodingTestCase(0x833f1c0e, 32, AArch64.r3, 3, AArch64.v28)); + + assertWrapper(new UmovEncodingTestCase(0x4b3d020e, 16, AArch64.r11, 0, AArch64.v10)); + assertWrapper(new UmovEncodingTestCase(0x893d1e0e, 16, AArch64.r9, 7, AArch64.v12)); + + assertWrapper(new UmovEncodingTestCase(0x0d3d010e, 8, AArch64.r13, 0, AArch64.v8)); + assertWrapper(new UmovEncodingTestCase(0xc73d1f0e, 8, AArch64.r7, 15, AArch64.v14)); + } + + @Test(expected = AssertionError.class) + @SuppressWarnings("unused") + public void testUmovInvalidSrcIdx() { + new UmovEncodingTestCase(invalidInstructionCode, 64, AArch64.r0, 2, AArch64.v0); + } + + @Test(expected = GraalError.class) + @SuppressWarnings("unused") + public void testUmovInvalidDataSize() { + new UmovEncodingTestCase(invalidInstructionCode, 31, AArch64.r0, 3, AArch64.v0); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64.test/src/org/graalvm/compiler/asm/aarch64/test/TestProtectedAssembler.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64.test/src/org/graalvm/compiler/asm/aarch64/test/TestProtectedAssembler.java index eaec8f9390e..98cab6a616f 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64.test/src/org/graalvm/compiler/asm/aarch64/test/TestProtectedAssembler.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64.test/src/org/graalvm/compiler/asm/aarch64/test/TestProtectedAssembler.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -549,4 +549,19 @@ class TestProtectedAssembler extends AArch64Assembler { public void ensureUniquePC() { throw new UnsupportedOperationException(); } + + @Override + public void cnt(int size, Register dst, Register src) { + super.cnt(size, dst, src); + } + + @Override + public void addv(int size, SIMDElementSize laneWidth, Register dst, Register src) { + super.addv(size, laneWidth, dst, src); + } + + @Override + public void umov(int size, Register dst, int srcIdx, Register src) { + super.umov(size, dst, srcIdx, src); + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64Assembler.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64Assembler.java index 4634e75d4e1..2901efed317 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64Assembler.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64Assembler.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2018, Red Hat Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -25,9 +25,15 @@ package org.graalvm.compiler.asm.aarch64; +import static jdk.vm.ci.aarch64.AArch64.CPU; +import static jdk.vm.ci.aarch64.AArch64.SIMD; import static jdk.vm.ci.aarch64.AArch64.cpuRegisters; +import static jdk.vm.ci.aarch64.AArch64.r0; +import static jdk.vm.ci.aarch64.AArch64.sp; +import static jdk.vm.ci.aarch64.AArch64.zr; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.ADD; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.ADDS; +import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.ADDV; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.ADR; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.ADRP; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.AND; @@ -44,6 +50,7 @@ import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.CCMP import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.CLREX; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.CLS; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.CLZ; +import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.CNT; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.CSEL; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.CSINC; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.CSNEG; @@ -107,27 +114,24 @@ import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.STXR import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.SUB; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.SUBS; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.SWP; -import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.TBZ; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.TBNZ; +import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.TBZ; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.UBFM; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.UDIV; +import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.UMOV; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.InstructionType.FP32; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.InstructionType.FP64; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.InstructionType.General32; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.InstructionType.General64; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.InstructionType.floatFromSize; import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.InstructionType.generalFromSize; -import static jdk.vm.ci.aarch64.AArch64.CPU; -import static jdk.vm.ci.aarch64.AArch64.SIMD; -import static jdk.vm.ci.aarch64.AArch64.r0; -import static jdk.vm.ci.aarch64.AArch64.sp; -import static jdk.vm.ci.aarch64.AArch64.zr; +import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.InstructionType.simdFromSize; import java.util.Arrays; import org.graalvm.compiler.asm.Assembler; -import org.graalvm.compiler.core.common.NumUtil; import org.graalvm.compiler.asm.aarch64.AArch64Address.AddressingMode; +import org.graalvm.compiler.core.common.NumUtil; import org.graalvm.compiler.debug.GraalError; import jdk.vm.ci.aarch64.AArch64; @@ -370,16 +374,42 @@ public abstract class AArch64Assembler extends Assembler { return n & NumUtil.getNbitNumberInt(sizeInBits); } + /** + * Enumeration of all different lane types of SIMD register. + * + * Byte(B):8b/lane; HalfWord(H):16b/lane; Word(S):32b/lane; DoubleWord(D):64b/lane. + */ + public enum SIMDElementSize { + Byte(0, 8), + HalfWord(1, 16), + Word(2, 32), + DoubleWord(3, 64); + + public final int encoding; + public final int nbits; + + SIMDElementSize(int encoding, int nbits) { + this.encoding = encoding; + this.nbits = nbits; + } + } + /** * Enumeration of all different instruction kinds: General32/64 are the general instructions * (integer, branch, etc.), for 32-, respectively 64-bit operands. FP32/64 is the encoding for - * the 32/64bit float operations + * the 32/64bit float operations. SIMDByte/HalfWord/Word/DoubleWord is the encoding for SIMD + * instructions */ protected enum InstructionType { General32(0b00 << 30, 32, true), General64(0b10 << 30, 64, true), FP32(0x00000000, 32, false), - FP64(0x00400000, 64, false); + FP64(0x00400000, 64, false), + + SIMDByte(0x01, 8, false), + SIMDHalfWord(0x02, 16, false), + SIMDWord(0x04, 32, false), + SIMDDoubleWord(0x08, 64, false); public final int encoding; public final int width; @@ -401,6 +431,20 @@ public abstract class AArch64Assembler extends Assembler { return size == 32 ? FP32 : FP64; } + public static InstructionType simdFromSize(int size) { + switch (size) { + case 8: + return SIMDByte; + case 16: + return SIMDHalfWord; + case 32: + return SIMDWord; + case 64: + return SIMDDoubleWord; + default: + throw GraalError.shouldNotReachHere(); + } + } } private static final int ImmediateOffset = 10; @@ -493,6 +537,10 @@ public abstract class AArch64Assembler extends Assembler { private static final int LDADDAcquireOffset = 23; private static final int LDADDReleaseOffset = 22; + private static final int SIMDImm5Offset = 16; + private static final int SIMDQBitOffset = 30; + private static final int SIMDSizeOffset = 22; + /** * Encoding for all instructions. */ @@ -611,7 +659,7 @@ public abstract class AArch64Assembler extends Assembler { FCSEL(0x1E200C00), INS(0x4e081c00), - UMOV(0x4e083c00), + UMOV(0x0e003c00), CNT(0xe205800), USRA(0x6f001400), @@ -626,7 +674,9 @@ public abstract class AArch64Assembler extends Assembler { MRS(0xD5300000), MSR(0xD5100000), - BLR_NATIVE(0xc0000000); + BLR_NATIVE(0xc0000000), + + ADDV(0x0e31b800); public final int encoding; @@ -2957,4 +3007,45 @@ public abstract class AArch64Assembler extends Assembler { } } + /** + * dst[0...n] = countBitCountOfEachByte(src[0...n]), n = size/8. + * + * @param size register size. Has to be 64 or 128. + * @param dst SIMD register. Should not be null. + * @param src SIMD register. Should not be null. + */ + public void cnt(int size, Register dst, Register src) { + assert 64 == size || 128 == size : "Invalid size for cnt"; + emitInt((size >> 7) << SIMDQBitOffset | CNT.encoding | rd(dst) | rs1(src)); + } + + /** + * dst = src[0] + ....+ src[n]. + * + * @param size register size. Has to be 64 or 128. + * @param laneWidth the width that SIMD register is treated as different lanes with. + * @param dst SIMD register. Should not be null. + * @param src SIMD register. Should not be null. + */ + public void addv(int size, SIMDElementSize laneWidth, Register dst, Register src) { + assert 64 == size || 128 == size : "Invalid size for addv"; + assert SIMDElementSize.DoubleWord != laneWidth : "Invalid lane width for addv"; + assert 64 != size || SIMDElementSize.Word != laneWidth : "Invalid size and lane combination for addv"; + emitInt((size >> 7) << SIMDQBitOffset | laneWidth.encoding << SIMDSizeOffset | ADDV.encoding | rd(dst) | rs1(src)); + } + + /** + * dst = src[srcIdx]. + * + * @param size register size. Can be 8, 16, 32 or 64. + * @param dst general purpose register. Should not be null or zero-register. + * @param srcIdx lane index of source register that dest data is from. + * @param src SIMD register. Should not be null. + */ + public void umov(int size, Register dst, int srcIdx, Register src) { + assert (srcIdx + 1) * size <= 128 : "Invalid src vectRegister index"; + InstructionType simdDataType = simdFromSize(size); + int imm5 = simdDataType.encoding | srcIdx << Integer.numberOfTrailingZeros(simdDataType.encoding) + 1; + emitInt((size >> 6) << SIMDQBitOffset | imm5 << SIMDImm5Offset | UMOV.encoding | rd(dst) | rs1(src)); + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64MacroAssembler.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64MacroAssembler.java index 1137b6c2cf9..c7e25d86df3 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64MacroAssembler.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64MacroAssembler.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1437,7 +1437,7 @@ public class AArch64MacroAssembler extends AArch64Assembler { int offset = label.position() - position(); super.adr(dst, offset); } else { - label.addPatchAt(position()); + label.addPatchAt(position(), this); // Encode condition flag so that we know how to patch the instruction later emitInt(PatchLabelKind.ADR.encoding | dst.encoding << PatchLabelKind.INFORMATION_OFFSET); } @@ -1456,7 +1456,7 @@ public class AArch64MacroAssembler extends AArch64Assembler { int offset = label.position() - position(); super.cbnz(size, cmp, offset); } else { - label.addPatchAt(position()); + label.addPatchAt(position(), this); int regEncoding = cmp.encoding << (PatchLabelKind.INFORMATION_OFFSET + 1); int sizeEncoding = (size == 64 ? 1 : 0) << PatchLabelKind.INFORMATION_OFFSET; // Encode condition flag so that we know how to patch the instruction later @@ -1477,7 +1477,7 @@ public class AArch64MacroAssembler extends AArch64Assembler { int offset = label.position() - position(); super.cbz(size, cmp, offset); } else { - label.addPatchAt(position()); + label.addPatchAt(position(), this); int regEncoding = cmp.encoding << (PatchLabelKind.INFORMATION_OFFSET + 1); int sizeEncoding = (size == 64 ? 1 : 0) << PatchLabelKind.INFORMATION_OFFSET; // Encode condition flag so that we know how to patch the instruction later @@ -1498,7 +1498,7 @@ public class AArch64MacroAssembler extends AArch64Assembler { int offset = label.position() - position(); super.tbnz(cmp, uimm6, offset); } else { - label.addPatchAt(position()); + label.addPatchAt(position(), this); int indexEncoding = uimm6 << PatchLabelKind.INFORMATION_OFFSET; int regEncoding = cmp.encoding << (PatchLabelKind.INFORMATION_OFFSET + 6); emitInt(PatchLabelKind.BRANCH_BIT_NONZERO.encoding | indexEncoding | regEncoding); @@ -1518,7 +1518,7 @@ public class AArch64MacroAssembler extends AArch64Assembler { int offset = label.position() - position(); super.tbz(cmp, uimm6, offset); } else { - label.addPatchAt(position()); + label.addPatchAt(position(), this); int indexEncoding = uimm6 << PatchLabelKind.INFORMATION_OFFSET; int regEncoding = cmp.encoding << (PatchLabelKind.INFORMATION_OFFSET + 6); emitInt(PatchLabelKind.BRANCH_BIT_ZERO.encoding | indexEncoding | regEncoding); @@ -1537,7 +1537,7 @@ public class AArch64MacroAssembler extends AArch64Assembler { int offset = label.position() - position(); super.b(condition, offset); } else { - label.addPatchAt(position()); + label.addPatchAt(position(), this); // Encode condition flag so that we know how to patch the instruction later emitInt(PatchLabelKind.BRANCH_CONDITIONALLY.encoding | condition.encoding << PatchLabelKind.INFORMATION_OFFSET); } @@ -1565,7 +1565,7 @@ public class AArch64MacroAssembler extends AArch64Assembler { int offset = label.position() - position(); super.b(offset); } else { - label.addPatchAt(position()); + label.addPatchAt(position(), this); emitInt(PatchLabelKind.BRANCH_UNCONDITIONALLY.encoding); } } @@ -1783,6 +1783,23 @@ public class AArch64MacroAssembler extends AArch64Assembler { a.lea(this, d); } + /** + * Count the set bits of src register. + * + * @param size src register size. Has to be 32 or 64. + * @param dst general purpose register. Should not be null or zero-register. + * @param src general purpose register. Should not be null. + * @param vreg SIMD register. Should not be null. + */ + public void popcnt(int size, Register dst, Register src, Register vreg) { + assert 32 == size || 64 == size : "Invalid data size"; + fmov(size, vreg, src); + final int fixedSize = 64; + cnt(fixedSize, vreg, vreg); + addv(fixedSize, SIMDElementSize.Byte, vreg, vreg); + umov(fixedSize, dst, 0, vreg); + } + public interface MacroInstruction { void patch(int codePos, int relative, byte[] code); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Assembler.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Assembler.java index 2236ebbac46..d493ee90b59 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Assembler.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Assembler.java @@ -75,7 +75,6 @@ import java.util.EnumSet; import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; import org.graalvm.compiler.asm.amd64.AVXKind.AVXSize; -import org.graalvm.compiler.core.common.NumUtil; import org.graalvm.compiler.core.common.calc.Condition; import org.graalvm.compiler.debug.GraalError; @@ -1882,7 +1881,7 @@ public class AMD64Assembler extends AMD64BaseAssembler { // is the same however, seems to be rather unlikely case. // Note: use jccb() if label to be bound is very close to get // an 8-bit displacement - l.addPatchAt(position()); + l.addPatchAt(position(), this); emitByte(0x0F); emitByte(0x80 | cc.getValue()); emitInt(0); @@ -1900,7 +1899,7 @@ public class AMD64Assembler extends AMD64BaseAssembler { emitByte(0x70 | cc.getValue()); emitByte((int) ((disp - shortSize) & 0xFF)); } else { - l.addPatchAt(position()); + l.addPatchAt(position(), this); emitByte(0x70 | cc.getValue()); emitByte(0); } @@ -1929,7 +1928,7 @@ public class AMD64Assembler extends AMD64BaseAssembler { // the forward jump will not run beyond 256 bytes, use jmpb to // force an 8-bit displacement. - l.addPatchAt(position()); + l.addPatchAt(position(), this); emitByte(0xE9); emitInt(0); } @@ -1950,14 +1949,13 @@ public class AMD64Assembler extends AMD64BaseAssembler { public final void jmpb(Label l) { if (l.isBound()) { int shortSize = 2; - int entry = l.position(); - assert isByte((entry - position()) + shortSize) : "Dispacement too large for a short jmp"; - long offs = entry - position(); + // Displacement is relative to byte just after jmpb instruction + int displacement = l.position() - position() - shortSize; + GraalError.guarantee(isByte(displacement), "Displacement too large to be encoded as a byte: %d", displacement); emitByte(0xEB); - emitByte((int) ((offs - shortSize) & 0xFF)); + emitByte(displacement & 0xFF); } else { - - l.addPatchAt(position()); + l.addPatchAt(position(), this); emitByte(0xEB); emitByte(0); } @@ -3397,9 +3395,7 @@ public class AMD64Assembler extends AMD64BaseAssembler { * Since a wrongly patched short branch can potentially lead to working but really bad * behaving code we should always fail with an exception instead of having an assert. */ - if (!NumUtil.isByte(imm8)) { - throw new InternalError("branch displacement out of range: " + imm8); - } + GraalError.guarantee(isByte(imm8), "Displacement too large to be encoded as a byte: %d", imm8); emitByte(imm8, branch + 1); } else { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.sparc/src/org/graalvm/compiler/asm/sparc/SPARCAssembler.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.sparc/src/org/graalvm/compiler/asm/sparc/SPARCAssembler.java index 867e6d4812c..314bb09cbd8 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.sparc/src/org/graalvm/compiler/asm/sparc/SPARCAssembler.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.sparc/src/org/graalvm/compiler/asm/sparc/SPARCAssembler.java @@ -1851,7 +1851,7 @@ public abstract class SPARCAssembler extends Assembler { } protected int patchUnbound(Label label) { - label.addPatchAt(position()); + label.addPatchAt(position(), this); return 0; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm/src/org/graalvm/compiler/asm/Assembler.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm/src/org/graalvm/compiler/asm/Assembler.java index dd8bde9c9cf..bae9f67ea60 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm/src/org/graalvm/compiler/asm/Assembler.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm/src/org/graalvm/compiler/asm/Assembler.java @@ -53,6 +53,11 @@ public abstract class Assembler { public final TargetDescription target; private List jumpDisplacementHints; + /** + * Labels with instructions to be patched when it is {@linkplain Label#bind bound}. + */ + Label labelsWithPatches; + /** * Backing code buffer. */ @@ -151,13 +156,26 @@ public abstract class Assembler { * @return the data in this buffer or a trimmed copy if {@code trimmedCopy} is {@code true} */ public byte[] close(boolean trimmedCopy) { + checkAndClearLabelsWithPatches(); return codeBuffer.close(trimmedCopy); } + private void checkAndClearLabelsWithPatches() throws InternalError { + Label label = labelsWithPatches; + while (label != null) { + if (label.patchPositions != null) { + throw new InternalError("Label used by instructions at following offsets has not been bound: " + label.patchPositions); + } + Label next = label.nextWithPatches; + label.nextWithPatches = null; + label = next; + } + labelsWithPatches = null; + } + public void bind(Label l) { assert !l.isBound() : "can bind label only once"; - l.bind(position()); - l.patchInstructions(this); + l.bind(position(), this); } public abstract void align(int modulus); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm/src/org/graalvm/compiler/asm/Label.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm/src/org/graalvm/compiler/asm/Label.java index e54cf261705..3a8a31a7f45 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm/src/org/graalvm/compiler/asm/Label.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm/src/org/graalvm/compiler/asm/Label.java @@ -35,10 +35,15 @@ public final class Label { private int blockId = -1; /** - * References to instructions that jump to this unresolved label. These instructions need to be - * patched when the label is bound using the {@link #patchInstructions(Assembler)} method. + * Positions of instructions that jump to this unresolved label. These instructions are patched + * when the label is bound. */ - private ArrayList patchPositions = null; + ArrayList patchPositions; + + /** + * Link in list of labels with instructions to be patched. + */ + Label nextWithPatches; /** * Returns the position of this label in the code buffer. @@ -62,36 +67,33 @@ public final class Label { } /** - * Binds the label to the specified position. - * - * @param pos the position + * Binds the label to {@code pos} and patches all instructions added by + * {@link #addPatchAt(int, Assembler)}. */ - protected void bind(int pos) { + protected void bind(int pos, Assembler asm) { + assert pos >= 0; this.position = pos; - assert isBound(); + if (patchPositions != null) { + for (int i = 0; i < patchPositions.size(); ++i) { + asm.patchJumpTarget(patchPositions.get(i), position); + } + patchPositions = null; + } } public boolean isBound() { return position >= 0; } - public void addPatchAt(int branchLocation) { + public void addPatchAt(int branchLocation, Assembler asm) { assert !isBound() : "Label is already bound " + this + " " + branchLocation + " at position " + position; if (patchPositions == null) { patchPositions = new ArrayList<>(2); + nextWithPatches = asm.labelsWithPatches; + asm.labelsWithPatches = this; } patchPositions.add(branchLocation); - } - protected void patchInstructions(Assembler masm) { - assert isBound() : "Label should be bound"; - if (patchPositions != null) { - int target = position; - for (int i = 0; i < patchPositions.size(); ++i) { - int pos = patchPositions.get(i); - masm.patchJumpTarget(pos, target); - } - } } public void reset() { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64TestBitAndBranchTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64TestBitAndBranchTest.java index da34e7ce9f5..ae07d293a1f 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64TestBitAndBranchTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64TestBitAndBranchTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,10 +24,9 @@ package org.graalvm.compiler.core.aarch64.test; -import static org.junit.Assume.assumeTrue; - -import java.util.function.Predicate; - +import jdk.vm.ci.aarch64.AArch64; +import jdk.vm.ci.code.TargetDescription; +import jdk.vm.ci.meta.Value; import org.graalvm.compiler.api.directives.GraalDirectives; import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler; import org.graalvm.compiler.lir.LIR; @@ -48,9 +47,9 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import jdk.vm.ci.aarch64.AArch64; -import jdk.vm.ci.code.TargetDescription; -import jdk.vm.ci.meta.Value; +import java.util.function.Predicate; + +import static org.junit.Assume.assumeTrue; public class AArch64TestBitAndBranchTest extends LIRTest { private static final Predicate checkForBitTestAndBranchOp = op -> (op instanceof AArch64ControlFlow.BitTestAndBranchOp); @@ -62,7 +61,7 @@ public class AArch64TestBitAndBranchTest extends LIRTest { } public static long testBit42Snippet(long a, long b, long c) { - if ((a & (1 << 42)) == 0) { + if ((a & (1L << 42)) == 0) { return b; } else { return c; @@ -71,12 +70,12 @@ public class AArch64TestBitAndBranchTest extends LIRTest { @Test public void testBit42() { - test("testBit42Snippet", 1L << 42L, Long.MAX_VALUE, Long.MIN_VALUE); - test("testBit42Snippet", ~(1L << 42L), Long.MAX_VALUE, Long.MIN_VALUE); + test("testBit42Snippet", 1L << 42, Long.MAX_VALUE, Long.MIN_VALUE); + test("testBit42Snippet", ~(1L << 42), Long.MAX_VALUE, Long.MIN_VALUE); checkLIR("testBit42Snippet", checkForBitTestAndBranchOp, 1); } - private static final LargeOpSpec largeOpSingleNop = new LargeOpSpec((1 << 14 - 2), 2); + private static final LargeOpSpec largeOpSingleNop = new LargeOpSpec((1 << 14 - 2) - 10, 2); /** * Tests the graceful case, where the estimation for @@ -99,7 +98,7 @@ public class AArch64TestBitAndBranchTest extends LIRTest { checkLIR("testBitTestAndBranchSingleSnippet", checkForBitTestAndBranchOp, 1); } - private static final LargeOpSpec largeOpFourNop = new LargeOpSpec((1 << 14 - 2), 8); + private static final LargeOpSpec largeOpFourNop = new LargeOpSpec((1 << 14 - 2) - 10, 8); /** * Tests the case, where the estimation for @@ -122,6 +121,73 @@ public class AArch64TestBitAndBranchTest extends LIRTest { checkLIR("testBitTestAndBranchFourSnippet", checkForBitTestAndBranchOp, 1); } + private static final float trueTarget = Float.MAX_VALUE; + private static final float falseTarget = Float.MIN_VALUE; + + public static float testLessThanZeroSnippet(long a, long b) { + if (b + a - b < 0) { + return trueTarget - a; + } else { + return falseTarget + a; + } + } + + @Test + public void testLessThanZero() { + test("testLessThanZeroSnippet", 1L, 777L); + test("testLessThanZeroSnippet", 0L, 777L); + test("testLessThanZeroSnippet", -1L, 777L); + checkLIR("testLessThanZeroSnippet", checkForBitTestAndBranchOp, 1); + } + + public static float testLessThanEqualZeroSnippet(long a) { + if (a <= 0) { + return trueTarget - a; + } else { + return falseTarget + a; + } + } + + @Test + public void testLessThanEqualZero() { + test("testLessThanEqualZeroSnippet", 1L); + test("testLessThanEqualZeroSnippet", 0L); + test("testLessThanEqualZeroSnippet", -1L); + checkLIR("testLessThanEqualZeroSnippet", checkForBitTestAndBranchOp, 0); + } + + public static float testGreaterThanZeroSnippet(int a) { + if (a > 0) { + return trueTarget - a; + } else { + return falseTarget + a; + } + } + + @Test + public void testGreaterThanZero() { + test("testGreaterThanZeroSnippet", 1); + test("testGreaterThanZeroSnippet", 0); + test("testGreaterThanZeroSnippet", -1); + checkLIR("testGreaterThanZeroSnippet", checkForBitTestAndBranchOp, 0); + } + + public static float testGreaterThanEqualZeroSnippet(int a) { + if (a >= 0) { + return trueTarget - a; + } else { + return falseTarget + a; + } + } + + @Test + public void testGreaterThanEqualZero() { + test("testGreaterThanEqualZeroSnippet", 1); + test("testGreaterThanEqualZeroSnippet", 0); + test("testGreaterThanEqualZeroSnippet", -1); + checkLIR("testGreaterThanEqualZeroSnippet", checkForBitTestAndBranchOp, 1); + } + private static class LargeOpSpec extends LIRTestSpecification { private final int n; private final int nopCount; @@ -172,7 +238,9 @@ public class AArch64TestBitAndBranchTest extends LIRTest { public class CheckPhase extends LIRPhase { @Override - protected void run(TargetDescription target, LIRGenerationResult lirGenRes, PreAllocationOptimizationContext context) { + protected void run( + TargetDescription target, LIRGenerationResult lirGenRes, + PreAllocationOptimizationContext context) { lir = lirGenRes.getLIR(); } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64ArithmeticLIRGenerator.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64ArithmeticLIRGenerator.java index d2881ea6058..16ec0595ebb 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64ArithmeticLIRGenerator.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64ArithmeticLIRGenerator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,6 +32,7 @@ import static org.graalvm.compiler.lir.LIRValueUtil.isJavaConstant; import static org.graalvm.compiler.lir.aarch64.AArch64BitManipulationOp.BitManipulationOpCode.BSR; import static org.graalvm.compiler.lir.aarch64.AArch64BitManipulationOp.BitManipulationOpCode.CLZ; import static org.graalvm.compiler.lir.aarch64.AArch64BitManipulationOp.BitManipulationOpCode.CTZ; +import static org.graalvm.compiler.lir.aarch64.AArch64BitManipulationOp.BitManipulationOpCode.POPCNT; import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler; import org.graalvm.compiler.core.common.LIRKind; @@ -432,27 +433,30 @@ public class AArch64ArithmeticLIRGenerator extends ArithmeticLIRGenerator implem @Override public Value emitBitCount(Value operand) { - throw GraalError.unimplemented("AArch64 ISA does not offer way to implement this more efficiently than a simple Java algorithm."); + assert ((AArch64Kind) operand.getPlatformKind()).isInteger(); + Variable result = getLIRGen().newVariable(LIRKind.combine(operand).changeType(AArch64Kind.DWORD)); + getLIRGen().append(new AArch64BitManipulationOp(getLIRGen(), POPCNT, result, asAllocatable(operand))); + return result; } @Override public Value emitBitScanReverse(Value value) { Variable result = getLIRGen().newVariable(LIRKind.combine(value).changeType(AArch64Kind.DWORD)); - getLIRGen().append(new AArch64BitManipulationOp(BSR, result, asAllocatable(value))); + getLIRGen().append(new AArch64BitManipulationOp(getLIRGen(), BSR, result, asAllocatable(value))); return result; } @Override public Value emitCountLeadingZeros(Value value) { Variable result = getLIRGen().newVariable(LIRKind.combine(value).changeType(AArch64Kind.DWORD)); - getLIRGen().append(new AArch64BitManipulationOp(CLZ, result, asAllocatable(value))); + getLIRGen().append(new AArch64BitManipulationOp(getLIRGen(), CLZ, result, asAllocatable(value))); return result; } @Override public Value emitCountTrailingZeros(Value value) { Variable result = getLIRGen().newVariable(LIRKind.combine(value).changeType(AArch64Kind.DWORD)); - getLIRGen().append(new AArch64BitManipulationOp(CTZ, result, asAllocatable(value))); + getLIRGen().append(new AArch64BitManipulationOp(getLIRGen(), CTZ, result, asAllocatable(value))); return result; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64NodeMatchRules.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64NodeMatchRules.java index 1c57eaea609..1efd0909271 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64NodeMatchRules.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64NodeMatchRules.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,6 +25,7 @@ package org.graalvm.compiler.core.aarch64; +import jdk.vm.ci.aarch64.AArch64Kind; import jdk.vm.ci.meta.AllocatableValue; import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.Value; @@ -32,6 +33,7 @@ import jdk.internal.vm.compiler.collections.EconomicMap; import jdk.internal.vm.compiler.collections.Equivalence; import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler; import org.graalvm.compiler.core.common.LIRKind; +import org.graalvm.compiler.core.common.calc.CanonicalCondition; import org.graalvm.compiler.core.gen.NodeMatchRules; import org.graalvm.compiler.core.match.ComplexMatchResult; import org.graalvm.compiler.core.match.MatchRule; @@ -44,12 +46,14 @@ import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow; import org.graalvm.compiler.lir.gen.LIRGeneratorTool; import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.DeoptimizingNode; +import org.graalvm.compiler.nodes.FixedNode; import org.graalvm.compiler.nodes.IfNode; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.calc.AddNode; import org.graalvm.compiler.nodes.calc.AndNode; import org.graalvm.compiler.nodes.calc.BinaryNode; +import org.graalvm.compiler.nodes.calc.IntegerLessThanNode; import org.graalvm.compiler.nodes.calc.LeftShiftNode; import org.graalvm.compiler.nodes.calc.NotNode; import org.graalvm.compiler.nodes.calc.OrNode; @@ -59,8 +63,6 @@ import org.graalvm.compiler.nodes.calc.UnsignedRightShiftNode; import org.graalvm.compiler.nodes.calc.XorNode; import org.graalvm.compiler.nodes.memory.Access; -import jdk.vm.ci.aarch64.AArch64Kind; - public class AArch64NodeMatchRules extends NodeMatchRules { private static final EconomicMap, AArch64ArithmeticOp> nodeOpMap; @@ -99,7 +101,8 @@ public class AArch64NodeMatchRules extends NodeMatchRules { return getLIRGeneratorTool().moveSp(value); } - private ComplexMatchResult emitBinaryShift(AArch64ArithmeticOp op, ValueNode value, BinaryNode shift, boolean isShiftNot) { + private ComplexMatchResult emitBinaryShift(AArch64ArithmeticOp op, ValueNode value, BinaryNode shift, + boolean isShiftNot) { AArch64MacroAssembler.ShiftType shiftType = shiftTypeMap.get(shift.getClass()); assert shiftType != null; assert value.getStackKind().isNumericInteger(); @@ -118,6 +121,18 @@ public class AArch64NodeMatchRules extends NodeMatchRules { }; } + private ComplexMatchResult emitBitTestAndBranch(FixedNode trueSuccessor, FixedNode falseSuccessor, + ValueNode value, double trueProbability, int nbits) { + return builder -> { + LabelRef trueDestination = getLIRBlock(trueSuccessor); + LabelRef falseDestination = getLIRBlock(falseSuccessor); + AllocatableValue src = moveSp(gen.asAllocatable(operand(value))); + gen.append(new AArch64ControlFlow.BitTestAndBranchOp(trueDestination, falseDestination, src, + trueProbability, nbits)); + return null; + }; + } + @MatchRule("(Add=binary a (LeftShift=shift b Constant))") @MatchRule("(Add=binary a (RightShift=shift b Constant))") @MatchRule("(Add=binary a (UnsignedRightShift=shift b Constant))") @@ -189,20 +204,27 @@ public class AArch64NodeMatchRules extends NodeMatchRules { if (value.getStackKind().isNumericInteger()) { long constant = a.asJavaConstant().asLong(); if (Long.bitCount(constant) == 1) { - int bitToTest = Long.numberOfTrailingZeros(constant); - return builder -> { - LabelRef trueDestination = getLIRBlock(root.trueSuccessor()); - LabelRef falseDestination = getLIRBlock(root.falseSuccessor()); - AllocatableValue src = moveSp(gen.asAllocatable(operand(value))); - double trueDestinationProbability = root.getTrueSuccessorProbability(); - gen.append(new AArch64ControlFlow.BitTestAndBranchOp(trueDestination, falseDestination, src, trueDestinationProbability, bitToTest)); - return null; - }; + return emitBitTestAndBranch(root.trueSuccessor(), root.falseSuccessor(), value, + root.getTrueSuccessorProbability(), Long.numberOfTrailingZeros(constant)); } } return null; } + /** + * if x < 0 <=> tbz x, sizeOfBits(x) - 1, label. + */ + @MatchRule("(If (IntegerLessThan=lessNode x Constant=y))") + public ComplexMatchResult checkNegativeAndBranch(IfNode root, IntegerLessThanNode lessNode, ValueNode x, ConstantNode y) { + JavaKind xKind = x.getStackKind(); + assert xKind.isNumericInteger(); + if (y.isJavaConstant() && (0 == y.asJavaConstant().asLong()) && lessNode.condition().equals(CanonicalCondition.LT)) { + return emitBitTestAndBranch(root.falseSuccessor(), root.trueSuccessor(), x, + 1.0 - root.getTrueSuccessorProbability(), xKind.getBitCount() - 1); + } + return null; + } + @Override public AArch64LIRGenerator getLIRGeneratorTool() { return (AArch64LIRGenerator) gen; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/GraalOptions.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/GraalOptions.java index 0acfecabe00..c60baa67214 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/GraalOptions.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/GraalOptions.java @@ -282,4 +282,6 @@ public final class GraalOptions { @Option(help = "Use Graal-generated stubs for complicated LIR operations instead of embedding all the emitted code.") public static final OptionKey UseGraalStubs = new OptionKey<>(true); + @Option(help = "Encode and decode snippets and substitutions before parsing to test libgraal code path. This option is ignored in the context of libgraal.") + public static final OptionKey UseEncodedGraphs = new OptionKey<>(false); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/spi/ForeignCallsProvider.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/spi/ForeignCallsProvider.java index 72d60dba54a..be8fc8e6950 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/spi/ForeignCallsProvider.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/spi/ForeignCallsProvider.java @@ -61,4 +61,9 @@ public interface ForeignCallsProvider extends ValueKindFactory { * Gets the linkage for a foreign call. */ ForeignCallLinkage lookupForeignCall(ForeignCallDescriptor descriptor); + + /** + * Return true if the foreign call has a binding. + */ + boolean isAvailable(ForeignCallDescriptor descriptor); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/IntegerStamp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/IntegerStamp.java index 7bc85516b39..22ca0fecc18 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/IntegerStamp.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/IntegerStamp.java @@ -643,7 +643,7 @@ public final class IntegerStamp extends PrimitiveStamp { IntegerStamp b = (IntegerStamp) stamp2; int bits = a.getBits(); - assert bits == b.getBits(); + assert bits == b.getBits() : String.format("stamp1.bits=%d, stamp2.bits=%d", bits, b.getBits()); if (a.lowerBound == a.upperBound && b.lowerBound == b.upperBound) { long value = CodeUtil.convert(a.lowerBound() + b.lowerBound(), a.getBits(), false); @@ -1298,6 +1298,15 @@ public final class IntegerStamp extends PrimitiveStamp { } } + private boolean testNoSignChangeAfterShifting(int bits, long value, int shiftAmount) { + long removedBits = -1L << (bits - shiftAmount - 1); + if (value < 0) { + return (value & removedBits) == removedBits; + } else { + return (value & removedBits) == 0; + } + } + @Override public Stamp foldStamp(Stamp stamp, IntegerStamp shift) { IntegerStamp value = (IntegerStamp) stamp; @@ -1318,13 +1327,15 @@ public final class IntegerStamp extends PrimitiveStamp { return value; } // the mask of bits that will be lost or shifted into the sign bit - long removedBits = -1L << (bits - shiftAmount - 1); - if ((value.lowerBound() & removedBits) == 0 && (value.upperBound() & removedBits) == 0) { + if (testNoSignChangeAfterShifting(bits, value.lowerBound(), shiftAmount) && testNoSignChangeAfterShifting(bits, value.upperBound(), shiftAmount)) { /* * use a better stamp if neither lower nor upper bound can lose * bits */ - return new IntegerStamp(bits, value.lowerBound() << shiftAmount, value.upperBound() << shiftAmount, value.downMask() << shiftAmount, value.upMask() << shiftAmount); + IntegerStamp result = new IntegerStamp(bits, value.lowerBound() << shiftAmount, value.upperBound() << shiftAmount, + (value.downMask() << shiftAmount) & CodeUtil.mask(bits), + (value.upMask() << shiftAmount) & CodeUtil.mask(bits)); + return result; } } if ((shift.lowerBound() >>> shiftBits) == (shift.upperBound() >>> shiftBits)) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/AbstractTypeReader.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/AbstractTypeReader.java new file mode 100644 index 00000000000..fb59fbaa253 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/AbstractTypeReader.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.core.common.util; + +public abstract class AbstractTypeReader implements TypeReader { + @Override + public long getSV() { + return decodeSign(read()); + } + + @Override + public long getUV() { + return read(); + } + + public static long decodeSign(long value) { + return (value >>> 1) ^ -(value & 1); + } + + private long read() { + int b0 = getU1(); + if (b0 < UnsafeArrayTypeWriter.NUM_LOW_CODES) { + return b0; + } else { + return readPacked(b0); + } + } + + private long readPacked(int b0) { + assert b0 >= UnsafeArrayTypeWriter.NUM_LOW_CODES; + long sum = b0; + long shift = UnsafeArrayTypeWriter.HIGH_WORD_SHIFT; + for (int i = 2;; i++) { + long b = getU1(); + sum += b << shift; + if (b < UnsafeArrayTypeWriter.NUM_LOW_CODES || i == UnsafeArrayTypeWriter.MAX_BYTES) { + return sum; + } + shift += UnsafeArrayTypeWriter.HIGH_WORD_SHIFT; + } + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/TypeReader.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/TypeReader.java index c5ff36a3e2b..8bb4713c2bf 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/TypeReader.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/TypeReader.java @@ -60,21 +60,7 @@ public interface TypeReader { * Reads a signed value that has been written using {@link TypeWriter#putSV variable byte size * encoding}. */ - default long getSV() { - long result = 0; - int shift = 0; - long b; - do { - b = getU1(); - result |= (b & 0x7f) << shift; - shift += 7; - } while ((b & 0x80) != 0); - - if ((b & 0x40) != 0 && shift < 64) { - result |= -1L << shift; - } - return result; - } + long getSV(); /** * Reads a signed variable byte size encoded value that is known to fit into the range of int. @@ -87,18 +73,7 @@ public interface TypeReader { * Reads an unsigned value that has been written using {@link TypeWriter#putSV variable byte * size encoding}. */ - default long getUV() { - long result = 0; - int shift = 0; - long b; - do { - b = getU1(); - result |= (b & 0x7f) << shift; - shift += 7; - } while ((b & 0x80) != 0); - - return result; - } + long getUV(); /** * Reads an unsigned variable byte size encoded value that is known to fit into the range of diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/TypeWriter.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/TypeWriter.java index 358dcc49cb2..b99ded06233 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/TypeWriter.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/TypeWriter.java @@ -29,7 +29,6 @@ package org.graalvm.compiler.core.common.util; * bytes. */ public interface TypeWriter { - /** * Returns the number of bytes that have been written, i.e., the byte index of the next byte to * be written. @@ -51,40 +50,18 @@ public interface TypeWriter { /** Writes a signed 4 byte value. */ void putS4(long value); + /** Patches a previously written signed 4 byte value at a given offset. */ + void patchS4(long value, long offset); + /** Writes an unsigned 4 byte value. */ void putU4(long value); /** Writes a signed 8 byte value. */ void putS8(long value); - /** - * Writes a signed value in a variable byte size encoding. - */ - default void putSV(long value) { - long cur = value; - while (true) { - if (cur >= -64 && cur < 64) { - putU1(cur & 0x7f); - return; - } - putU1(0x80 | (cur & 0x7f)); - cur = cur >> 7; - } - } + /** Writes a signed value in a variable byte size encoding. */ + void putSV(long value); - /** - * Writes an unsigned value in a variable byte size encoding. - */ - default void putUV(long value) { - long cur = value; - while (true) { - assert cur >= 0; - if (cur < 128) { - putU1(cur & 0x7f); - return; - } - putU1(0x80 | (cur & 0x7f)); - cur = cur >> 7; - } - } + /** Writes an unsigned value in a variable byte size encoding. */ + void putUV(long value); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/UnsafeArrayTypeReader.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/UnsafeArrayTypeReader.java index ac41ed151dc..576ddb056c3 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/UnsafeArrayTypeReader.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/UnsafeArrayTypeReader.java @@ -40,7 +40,7 @@ import sun.misc.Unsafe; * architectures that support unaligned memory accesses; the value {@code false} is the safe * fallback that works on every hardware. */ -public abstract class UnsafeArrayTypeReader implements TypeReader { +public abstract class UnsafeArrayTypeReader extends AbstractTypeReader { private static final Unsafe UNSAFE = getUnsafe(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/UnsafeArrayTypeWriter.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/UnsafeArrayTypeWriter.java index 8a30d668d77..8237518b6c6 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/UnsafeArrayTypeWriter.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/UnsafeArrayTypeWriter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,6 +32,8 @@ import static org.graalvm.compiler.core.common.util.TypeConversion.asU2; import static org.graalvm.compiler.core.common.util.TypeConversion.asU4; import static org.graalvm.compiler.serviceprovider.GraalUnsafeAccess.getUnsafe; +import org.graalvm.compiler.core.common.calc.UnsignedMath; + import sun.misc.Unsafe; /** @@ -45,11 +47,16 @@ import sun.misc.Unsafe; * fallback that works on every hardware. */ public abstract class UnsafeArrayTypeWriter implements TypeWriter { - private static final Unsafe UNSAFE = getUnsafe(); private static final int MIN_CHUNK_LENGTH = 200; private static final int MAX_CHUNK_LENGTH = 16000; + // Constants for UNSIGNED5 coding of Pack200 + public static final long HIGH_WORD_SHIFT = 6; + public static final long NUM_HIGH_CODES = 1 << HIGH_WORD_SHIFT; // number of high codes (64) + public static final long NUM_LOW_CODES = (1 << Byte.SIZE) - NUM_HIGH_CODES; + public static final long MAX_BYTES = 11; + static class Chunk { protected final byte[] data; protected int size; @@ -118,6 +125,30 @@ public abstract class UnsafeArrayTypeWriter implements TypeWriter { putS4(asU4(value)); } + @Override + public void putS2(long value) { + long offset = writeOffset(Short.BYTES); + putS2(value, writeChunk, offset); + } + + @Override + public void putS4(long value) { + long offset = writeOffset(Integer.BYTES); + putS4(value, writeChunk, offset); + } + + @Override + public void putS8(long value) { + long offset = writeOffset(Long.BYTES); + putS8(value, writeChunk, offset); + } + + protected abstract void putS2(long value, Chunk chunk, long offset); + + protected abstract void putS4(long value, Chunk chunk, long offset); + + protected abstract void putS8(long value, Chunk chunk, long offset); + protected long writeOffset(int writeBytes) { if (writeChunk.size + writeBytes >= writeChunk.data.length) { Chunk newChunk = new Chunk(Math.min(writeChunk.data.length * 2, MAX_CHUNK_LENGTH)); @@ -134,27 +165,76 @@ public abstract class UnsafeArrayTypeWriter implements TypeWriter { return result; } + + @Override + public void patchS4(long value, long offset) { + long chunkStartOffset = 0; + Chunk chunk = firstChunk; + while (chunkStartOffset + chunk.size <= offset) { + chunkStartOffset += chunk.size; + chunk = chunk.next; + } + + long targetOffset = offset - chunkStartOffset; + assert targetOffset + Integer.BYTES <= chunk.size : "out of bounds"; + putS4(value, chunk, Unsafe.ARRAY_BYTE_BASE_OFFSET + targetOffset); + } + + @Override + public void putSV(long value) { + // this is a modified version of the SIGNED5 encoding from Pack200 + write(encodeSign(value)); + } + + @Override + public void putUV(long value) { + // this is a modified version of the UNSIGNED5 encoding from Pack200 + write(value); + } + + private static long encodeSign(long value) { + return (value << 1) ^ (value >> 63); + } + + private void write(long value) { + if (UnsignedMath.belowThan(value, NUM_LOW_CODES)) { + putU1(value); + } else { + writePacked(value); + } + } + + private void writePacked(long value) { + long sum = value; + for (int i = 1; UnsignedMath.aboveOrEqual(sum, NUM_LOW_CODES) && i < MAX_BYTES; i++) { + sum -= NUM_LOW_CODES; + long u1 = NUM_LOW_CODES + (sum & (NUM_HIGH_CODES - 1)); // this is a "high code" + sum >>>= HIGH_WORD_SHIFT; // extracted 6 bits + putU1(u1); + } + + // remainder is either a "low code" or the last byte + assert sum == (sum & 0xFF) : "not a byte"; + putU1(sum & 0xFF); + } } final class UnalignedUnsafeArrayTypeWriter extends UnsafeArrayTypeWriter { private static final Unsafe UNSAFE = getUnsafe(); @Override - public void putS2(long value) { - long offset = writeOffset(Short.BYTES); - UNSAFE.putShort(writeChunk.data, offset, asS2(value)); + protected void putS2(long value, Chunk chunk, long offset) { + UNSAFE.putShort(chunk.data, offset, asS2(value)); } @Override - public void putS4(long value) { - long offset = writeOffset(Integer.BYTES); - UNSAFE.putInt(writeChunk.data, offset, asS4(value)); + protected void putS4(long value, Chunk chunk, long offset) { + UNSAFE.putInt(chunk.data, offset, asS4(value)); } @Override - public void putS8(long value) { - long offset = writeOffset(Long.BYTES); - UNSAFE.putLong(writeChunk.data, offset, value); + protected void putS8(long value, Chunk chunk, long offset) { + UNSAFE.putLong(chunk.data, offset, value); } } @@ -162,31 +242,28 @@ final class AlignedUnsafeArrayTypeWriter extends UnsafeArrayTypeWriter { private static final Unsafe UNSAFE = getUnsafe(); @Override - public void putS2(long value) { - long offset = writeOffset(Short.BYTES); - UNSAFE.putByte(writeChunk.data, offset + 0, (byte) (value >> 0)); - UNSAFE.putByte(writeChunk.data, offset + 1, (byte) (value >> 8)); + protected void putS2(long value, Chunk chunk, long offset) { + UNSAFE.putByte(chunk.data, offset + 0, (byte) (value >> 0)); + UNSAFE.putByte(chunk.data, offset + 1, (byte) (value >> 8)); } @Override - public void putS4(long value) { - long offset = writeOffset(Integer.BYTES); - UNSAFE.putByte(writeChunk.data, offset + 0, (byte) (value >> 0)); - UNSAFE.putByte(writeChunk.data, offset + 1, (byte) (value >> 8)); - UNSAFE.putByte(writeChunk.data, offset + 2, (byte) (value >> 16)); - UNSAFE.putByte(writeChunk.data, offset + 3, (byte) (value >> 24)); + protected void putS4(long value, Chunk chunk, long offset) { + UNSAFE.putByte(chunk.data, offset + 0, (byte) (value >> 0)); + UNSAFE.putByte(chunk.data, offset + 1, (byte) (value >> 8)); + UNSAFE.putByte(chunk.data, offset + 2, (byte) (value >> 16)); + UNSAFE.putByte(chunk.data, offset + 3, (byte) (value >> 24)); } @Override - public void putS8(long value) { - long offset = writeOffset(Long.BYTES); - UNSAFE.putByte(writeChunk.data, offset + 0, (byte) (value >> 0)); - UNSAFE.putByte(writeChunk.data, offset + 1, (byte) (value >> 8)); - UNSAFE.putByte(writeChunk.data, offset + 2, (byte) (value >> 16)); - UNSAFE.putByte(writeChunk.data, offset + 3, (byte) (value >> 24)); - UNSAFE.putByte(writeChunk.data, offset + 4, (byte) (value >> 32)); - UNSAFE.putByte(writeChunk.data, offset + 5, (byte) (value >> 40)); - UNSAFE.putByte(writeChunk.data, offset + 6, (byte) (value >> 48)); - UNSAFE.putByte(writeChunk.data, offset + 7, (byte) (value >> 56)); + protected void putS8(long value, Chunk chunk, long offset) { + UNSAFE.putByte(chunk.data, offset + 0, (byte) (value >> 0)); + UNSAFE.putByte(chunk.data, offset + 1, (byte) (value >> 8)); + UNSAFE.putByte(chunk.data, offset + 2, (byte) (value >> 16)); + UNSAFE.putByte(chunk.data, offset + 3, (byte) (value >> 24)); + UNSAFE.putByte(chunk.data, offset + 4, (byte) (value >> 32)); + UNSAFE.putByte(chunk.data, offset + 5, (byte) (value >> 40)); + UNSAFE.putByte(chunk.data, offset + 6, (byte) (value >> 48)); + UNSAFE.putByte(chunk.data, offset + 7, (byte) (value >> 56)); } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/BasePhaseBinaryGraphTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/BasePhaseBinaryGraphTest.java index 6fb9f2fa20d..a92969caa39 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/BasePhaseBinaryGraphTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/BasePhaseBinaryGraphTest.java @@ -24,11 +24,12 @@ package org.graalvm.compiler.core.test; +import static org.junit.Assert.assertEquals; + import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.phases.BasePhase; import org.graalvm.compiler.printer.BinaryGraphPrinter; -import static org.junit.Assert.assertEquals; import org.junit.Before; import org.junit.Test; @@ -43,7 +44,7 @@ public class BasePhaseBinaryGraphTest { @Before public void createPrinter() throws Exception { - printer = new BinaryGraphPrinter(DebugContext.DISABLED, null); + printer = new BinaryGraphPrinter(DebugContext.disabled(null), null); } @Test diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GraphResetDebugTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GraphResetDebugTest.java index 74260108474..869eae830c0 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GraphResetDebugTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GraphResetDebugTest.java @@ -51,7 +51,7 @@ public class GraphResetDebugTest extends GraalCompilerTest { StructuredGraph graph = parseEager("testSnippet", AllowAssumptions.YES, debug); boolean resetSucceeded = false; try (Scope scope = debug.scope("some scope")) { - graph.resetDebug(DebugContext.DISABLED); + graph.resetDebug(DebugContext.disabled(getInitialOptions())); resetSucceeded = true; } catch (AssertionError expected) { } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/TypeWriterTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/TypeWriterTest.java index 83631eb3200..4f7d9f2e57e 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/TypeWriterTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/TypeWriterTest.java @@ -141,13 +141,13 @@ public class TypeWriterTest extends GraalCompilerTest { test01(false); } - private static void checkSignedSize(TypeWriter writer, long value, int expectedSize) { + private static void checkSignedSize(TypeWriter writer, long value, long expectedSize) { long sizeBefore = writer.getBytesWritten(); writer.putSV(value); Assert.assertEquals(expectedSize, writer.getBytesWritten() - sizeBefore); } - private static void checkUnsignedSize(TypeWriter writer, long value, int expectedSize) { + private static void checkUnsignedSize(TypeWriter writer, long value, long expectedSize) { long sizeBefore = writer.getBytesWritten(); writer.putUV(value); Assert.assertEquals(expectedSize, writer.getBytesWritten() - sizeBefore); @@ -155,23 +155,24 @@ public class TypeWriterTest extends GraalCompilerTest { private static void checkSizes(TypeWriter writer) { checkSignedSize(writer, 0, 1); - checkSignedSize(writer, 63, 1); - checkSignedSize(writer, -64, 1); - checkSignedSize(writer, 64, 2); - checkSignedSize(writer, -65, 2); - checkSignedSize(writer, 8191, 2); - checkSignedSize(writer, -8192, 2); + checkSignedSize(writer, 95, 1); + checkSignedSize(writer, -96, 1); + checkSignedSize(writer, 96, 2); + checkSignedSize(writer, -97, 2); + checkSignedSize(writer, 6239, 2); + checkSignedSize(writer, -6240, 2); checkSignedSize(writer, 8192, 3); checkSignedSize(writer, -8193, 3); - checkSignedSize(writer, Long.MAX_VALUE, 10); - checkSignedSize(writer, Long.MIN_VALUE, 10); + checkSignedSize(writer, Long.MAX_VALUE, UnsafeArrayTypeWriter.MAX_BYTES); + checkSignedSize(writer, Long.MIN_VALUE, UnsafeArrayTypeWriter.MAX_BYTES); checkUnsignedSize(writer, 0, 1); - checkUnsignedSize(writer, 127, 1); - checkUnsignedSize(writer, 128, 2); - checkUnsignedSize(writer, 16383, 2); - checkUnsignedSize(writer, 16384, 3); - checkUnsignedSize(writer, Long.MAX_VALUE, 9); + checkUnsignedSize(writer, 191, 1); + checkUnsignedSize(writer, 192, 2); + checkUnsignedSize(writer, 12479, 2); + checkUnsignedSize(writer, 12480, 3); + checkUnsignedSize(writer, Long.MAX_VALUE, UnsafeArrayTypeWriter.MAX_BYTES); + checkUnsignedSize(writer, Long.MIN_VALUE, UnsafeArrayTypeWriter.MAX_BYTES); } @Test diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/DebugInfoBuilder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/DebugInfoBuilder.java index 1ea53a2e4af..2d95d299c97 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/DebugInfoBuilder.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/DebugInfoBuilder.java @@ -252,7 +252,7 @@ public class DebugInfoBuilder { if (!state.canProduceBytecodeFrame()) { // This typically means a snippet or intrinsic frame state made it to the backend - StackTraceElement ste = state.getCode().asStackTraceElement(state.bci); + String ste = state.getCode() != null ? state.getCode().asStackTraceElement(state.bci).toString() : state.toString(); throw new GraalError("Frame state for %s cannot be converted to a BytecodeFrame since the frame state's code is " + "not the same as the frame state method's code", ste); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/DebugContext.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/DebugContext.java index a618eda3ce1..41a4431236f 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/DebugContext.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/DebugContext.java @@ -311,9 +311,19 @@ public final class DebugContext implements AutoCloseable { } /** - * Shared object used to represent a disabled debug context. + * Singleton used to represent a disabled debug context. */ - public static final DebugContext DISABLED = new DebugContext(NO_DESCRIPTION, NO_GLOBAL_METRIC_VALUES, DEFAULT_LOG_STREAM, new Immutable(), NO_CONFIG_CUSTOMIZERS); + private static final DebugContext DISABLED = new DebugContext(NO_DESCRIPTION, NO_GLOBAL_METRIC_VALUES, DEFAULT_LOG_STREAM, new Immutable(), NO_CONFIG_CUSTOMIZERS); + + /** + * Create a DebugContext with debugging disabled. + */ + public static DebugContext disabled(OptionValues options) { + if (options == null || options.getMap().isEmpty()) { + return DISABLED; + } + return new DebugContext(NO_DESCRIPTION, NO_GLOBAL_METRIC_VALUES, DEFAULT_LOG_STREAM, Immutable.create(options), NO_CONFIG_CUSTOMIZERS); + } /** * Gets the debug context for the current thread. This should only be used when there is no diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/Node.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/Node.java index 2581ac962bf..525f9d37169 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/Node.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/Node.java @@ -192,6 +192,11 @@ public abstract class Node implements Cloneable, Formattable, NodeInterface { * {@code true}. */ boolean injectedStampIsNonNull() default false; + + /** + * If {@code true} then this is lowered into a node that has side effects. + */ + boolean hasSideEffect() default false; } /** diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/NodeList.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/NodeList.java index f8644a2bc2a..704783608c3 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/NodeList.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/NodeList.java @@ -201,16 +201,24 @@ public abstract class NodeList extends AbstractList implement size = other.size; } - public boolean equals(NodeList other) { - if (size != other.size) { - return false; + @Override + public boolean equals(Object other) { + if (other == this) { + return true; } - for (int i = 0; i < size; i++) { - if (nodes[i] != other.nodes[i]) { + if (other instanceof List) { + List otherList = (List) other; + if (size != otherList.size()) { return false; } + for (int i = 0; i < size; i++) { + if (nodes[i] != otherList.get(i)) { + return false; + } + } + return true; } - return true; + return false; } @SuppressWarnings("unchecked") diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64.test/src/org/graalvm/compiler/hotspot/aarch64/test/AArch64UncompressPointerTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64.test/src/org/graalvm/compiler/hotspot/aarch64/test/AArch64UncompressPointerTest.java new file mode 100644 index 00000000000..d209fc5d757 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64.test/src/org/graalvm/compiler/hotspot/aarch64/test/AArch64UncompressPointerTest.java @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, Arm Limited and affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + + +package org.graalvm.compiler.hotspot.aarch64.test; + +import jdk.vm.ci.aarch64.AArch64; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.code.TargetDescription; +import jdk.vm.ci.runtime.JVMCI; +import org.graalvm.compiler.asm.aarch64.AArch64Assembler; +import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler; +import org.graalvm.compiler.core.test.GraalCompilerTest; +import org.graalvm.compiler.hotspot.aarch64.AArch64HotSpotMove; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assume.assumeTrue; + +public class AArch64UncompressPointerTest extends GraalCompilerTest { + + private AArch64MacroAssembler masm1; + private AArch64MacroAssembler masm2; + private Register input; + private Register result; + + @Before + public void checkAArch64() { + assumeTrue("skipping AArch64 specific test", JVMCI.getRuntime().getHostJVMCIBackend().getTarget().arch instanceof AArch64); + } + + @Before + public void setupEnvironment() { + TargetDescription target = JVMCI.getRuntime().getHostJVMCIBackend().getTarget(); + masm1 = new AArch64MacroAssembler(target); + masm2 = new AArch64MacroAssembler(target); + input = AArch64.r10; + result = AArch64.r11; + } + + private void emitUncompressPointer(Register base, int shift) { + AArch64HotSpotMove.UncompressPointer.emitUncompressCode(masm2, input, result, base, shift, true); + } + + private void compareAssembly() { + byte[] expected = masm1.close(false); + byte[] actual = masm2.close(false); + assertArrayEquals(expected, actual); + } + + @Test + public void testUncompressPointerWithBase() { + Register base = AArch64.r12; + int shift = 3; + masm1.add(64, result, base, input, AArch64Assembler.ShiftType.LSL, shift); + emitUncompressPointer(base, shift); + compareAssembly(); + } + + @Test + public void testUncompressPointerWithZeroBase() { + int shift = 3; + masm1.shl(64, result, input, shift); + emitUncompressPointer(null, shift); + compareAssembly(); + } + + @Test + public void testUncompressPointerWithZeroBaseAndShift() { + masm1.or(64, result, AArch64.zr, input); + emitUncompressPointer(null, 0); + compareAssembly(); + } + + static class A { + String str; + + A(String str) { + this.str = str; + } + } + + public static String getObjectField(A a) { + return a.str; + } + + @Test + public void testGetObjectField() { + test("getObjectField", new A("asfghjkjhgfd")); + } + + static String[] strings = {"asf", "egfda", "fsdasere", "eqwred", "fgdadgtre", "qwrrtreety"}; + + public static String getArrayMember(int index) { + return strings[index]; + } + + @Test + public void testGetArrayMember() { + test("getArrayMember", 4); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotBackendFactory.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotBackendFactory.java index 79284e5d3f6..3d62a187b0d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotBackendFactory.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotBackendFactory.java @@ -144,10 +144,10 @@ public class AArch64HotSpotBackendFactory implements HotSpotBackendFactory { bytecodeProvider = new ClassfileBytecodeProvider(metaAccess, snippetReflection); } try (InitTimer rt = timer("create Replacements provider")) { - replacements = createReplacements(graalRuntime.getOptions(), p, snippetReflection, bytecodeProvider); + replacements = createReplacements(p, snippetReflection, bytecodeProvider); } try (InitTimer rt = timer("create GraphBuilderPhase plugins")) { - plugins = createGraphBuilderPlugins(compilerConfiguration, config, constantReflection, foreignCalls, metaAccess, snippetReflection, replacements, wordTypes); + plugins = createGraphBuilderPlugins(compilerConfiguration, config, constantReflection, foreignCalls, metaAccess, snippetReflection, replacements, wordTypes, graalRuntime.getOptions()); replacements.setGraphBuilderPlugins(plugins); } try (InitTimer rt = timer("create Suites provider")) { @@ -156,6 +156,7 @@ public class AArch64HotSpotBackendFactory implements HotSpotBackendFactory { providers = new HotSpotProviders(metaAccess, codeCache, constantReflection, constantFieldProvider, foreignCalls, lowerer, replacements, suites, registers, snippetReflection, wordTypes, plugins); + replacements.setProviders(providers); } try (InitTimer rt = timer("instantiate backend")) { return createBackend(config, graalRuntime, providers); @@ -164,8 +165,8 @@ public class AArch64HotSpotBackendFactory implements HotSpotBackendFactory { protected Plugins createGraphBuilderPlugins(CompilerConfiguration compilerConfiguration, GraalHotSpotVMConfig config, HotSpotConstantReflectionProvider constantReflection, HotSpotHostForeignCallsProvider foreignCalls, HotSpotMetaAccessProvider metaAccess, HotSpotSnippetReflectionProvider snippetReflection, - HotSpotReplacementsImpl replacements, HotSpotWordTypes wordTypes) { - Plugins plugins = HotSpotGraphBuilderPlugins.create(compilerConfiguration, config, wordTypes, metaAccess, constantReflection, snippetReflection, foreignCalls, replacements); + HotSpotReplacementsImpl replacements, HotSpotWordTypes wordTypes, OptionValues options) { + Plugins plugins = HotSpotGraphBuilderPlugins.create(compilerConfiguration, config, wordTypes, metaAccess, constantReflection, snippetReflection, foreignCalls, replacements, options); AArch64GraphBuilderPlugins.register(plugins, replacements.getDefaultReplacementBytecodeProvider(), false, // /* registerMathPlugins */true); return plugins; @@ -179,8 +180,8 @@ public class AArch64HotSpotBackendFactory implements HotSpotBackendFactory { return new HotSpotRegisters(AArch64HotSpotRegisterConfig.threadRegister, AArch64HotSpotRegisterConfig.heapBaseRegister, sp); } - protected HotSpotReplacementsImpl createReplacements(OptionValues options, Providers p, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider) { - return new HotSpotReplacementsImpl(options, p, snippetReflection, bytecodeProvider, p.getCodeCache().getTarget()); + protected HotSpotReplacementsImpl createReplacements(Providers p, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider) { + return new HotSpotReplacementsImpl(p, snippetReflection, bytecodeProvider, p.getCodeCache().getTarget()); } protected HotSpotHostForeignCallsProvider createForeignCalls(HotSpotJVMCIRuntime jvmciRuntime, HotSpotGraalRuntimeProvider runtime, HotSpotMetaAccessProvider metaAccess, diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotMove.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotMove.java index 64795388a38..ec1ae8ca311 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotMove.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotMove.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -194,9 +194,19 @@ public class AArch64HotSpotMove { } public static void emitUncompressCode(AArch64MacroAssembler masm, Register inputRegister, Register resReg, Register baseReg, int shift, boolean nonNull) { + // result = ptr << shift + if (baseReg == null) { + if (shift != 0) { + masm.shl(64, resReg, inputRegister, shift); + } else if (!resReg.equals(inputRegister)) { + masm.movx(resReg, inputRegister); + } + return; + } + // result = base + (ptr << shift) - if (nonNull || baseReg == null) { - masm.add(64, resReg, baseReg == null ? zr : baseReg, inputRegister, AArch64Assembler.ShiftType.LSL, shift); + if (nonNull) { + masm.add(64, resReg, baseReg, inputRegister, AArch64Assembler.ShiftType.LSL, shift); } else { // if ptr is null it has to be null after decompression Label done = new Label(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/StubAVXTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/StubAVXTest.java index dc1b65ca193..5c13d172086 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/StubAVXTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/StubAVXTest.java @@ -234,7 +234,7 @@ public class StubAVXTest extends LIRTest { public void test() { HotSpotProviders providers = (HotSpotProviders) getProviders(); HotSpotForeignCallsProviderImpl foreignCalls = (HotSpotForeignCallsProviderImpl) providers.getForeignCalls(); - HotSpotForeignCallLinkage linkage = foreignCalls.registerStubCall(TEST_STUB, HotSpotForeignCallLinkage.Transition.LEAF_NOFP, HotSpotForeignCallLinkage.Reexecutability.REEXECUTABLE); + HotSpotForeignCallLinkage linkage = foreignCalls.registerStubCall(TEST_STUB, HotSpotForeignCallLinkage.Transition.LEAF_NO_VZERO, HotSpotForeignCallLinkage.Reexecutability.REEXECUTABLE); linkage.setCompiledStub(new TestStub(getInitialOptions(), providers, linkage)); runTest("testStub"); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackendFactory.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackendFactory.java index ec84ef9b88b..35350e69b46 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackendFactory.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackendFactory.java @@ -138,10 +138,10 @@ public class AMD64HotSpotBackendFactory implements HotSpotBackendFactory { bytecodeProvider = new ClassfileBytecodeProvider(metaAccess, snippetReflection); } try (InitTimer rt = timer("create Replacements provider")) { - replacements = createReplacements(options, p, snippetReflection, bytecodeProvider); + replacements = createReplacements(p, snippetReflection, bytecodeProvider); } try (InitTimer rt = timer("create GraphBuilderPhase plugins")) { - plugins = createGraphBuilderPlugins(compilerConfiguration, config, target, constantReflection, foreignCalls, metaAccess, snippetReflection, replacements, wordTypes); + plugins = createGraphBuilderPlugins(compilerConfiguration, config, target, constantReflection, foreignCalls, metaAccess, snippetReflection, replacements, wordTypes, options); replacements.setGraphBuilderPlugins(plugins); } try (InitTimer rt = timer("create Suites provider")) { @@ -159,8 +159,8 @@ public class AMD64HotSpotBackendFactory implements HotSpotBackendFactory { protected Plugins createGraphBuilderPlugins(CompilerConfiguration compilerConfiguration, GraalHotSpotVMConfig config, TargetDescription target, HotSpotConstantReflectionProvider constantReflection, HotSpotHostForeignCallsProvider foreignCalls, HotSpotMetaAccessProvider metaAccess, - HotSpotSnippetReflectionProvider snippetReflection, HotSpotReplacementsImpl replacements, HotSpotWordTypes wordTypes) { - Plugins plugins = HotSpotGraphBuilderPlugins.create(compilerConfiguration, config, wordTypes, metaAccess, constantReflection, snippetReflection, foreignCalls, replacements); + HotSpotSnippetReflectionProvider snippetReflection, HotSpotReplacementsImpl replacements, HotSpotWordTypes wordTypes, OptionValues options) { + Plugins plugins = HotSpotGraphBuilderPlugins.create(compilerConfiguration, config, wordTypes, metaAccess, constantReflection, snippetReflection, foreignCalls, replacements, options); AMD64GraphBuilderPlugins.register(plugins, replacements.getDefaultReplacementBytecodeProvider(), (AMD64) target.arch, false, JAVA_SPECIFICATION_VERSION >= 9); return plugins; } @@ -173,8 +173,8 @@ public class AMD64HotSpotBackendFactory implements HotSpotBackendFactory { return new HotSpotRegisters(AMD64.r15, AMD64.r12, AMD64.rsp); } - protected HotSpotReplacementsImpl createReplacements(OptionValues options, Providers p, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider) { - return new HotSpotReplacementsImpl(options, p, snippetReflection, bytecodeProvider, p.getCodeCache().getTarget()); + protected HotSpotReplacementsImpl createReplacements(Providers p, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider) { + return new HotSpotReplacementsImpl(p, snippetReflection, bytecodeProvider, p.getCodeCache().getTarget()); } protected AMD64HotSpotForeignCallsProvider createForeignCalls(HotSpotJVMCIRuntime jvmciRuntime, HotSpotGraalRuntimeProvider runtime, HotSpotMetaAccessProvider metaAccess, diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotForeignCallsProvider.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotForeignCallsProvider.java index 374bc55f2a1..5467e021629 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotForeignCallsProvider.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotForeignCallsProvider.java @@ -36,7 +36,7 @@ import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Reexecutabi import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Reexecutability.REEXECUTABLE_ONLY_AFTER_EXCEPTION; import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.RegisterEffect.PRESERVES_REGISTERS; import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Transition.LEAF; -import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Transition.LEAF_NOFP; +import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Transition.LEAF_NO_VZERO; import static org.graalvm.compiler.hotspot.replacements.CRC32CSubstitutions.UPDATE_BYTES_CRC32C; import static org.graalvm.compiler.hotspot.replacements.CRC32Substitutions.UPDATE_BYTES_CRC32; import static org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode.BinaryOperation.POW; @@ -89,15 +89,15 @@ public class AMD64HotSpotForeignCallsProvider extends HotSpotHostForeignCallsPro RegisterValue exception = rax.asValue(LIRKind.reference(word)); RegisterValue exceptionPc = rdx.asValue(LIRKind.value(word)); CallingConvention exceptionCc = new CallingConvention(0, ILLEGAL, exception, exceptionPc); - register(new HotSpotForeignCallLinkageImpl(EXCEPTION_HANDLER, 0L, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, exceptionCc, null, any())); - register(new HotSpotForeignCallLinkageImpl(EXCEPTION_HANDLER_IN_CALLER, JUMP_ADDRESS, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, exceptionCc, null, any())); + register(new HotSpotForeignCallLinkageImpl(EXCEPTION_HANDLER, 0L, PRESERVES_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE_ONLY_AFTER_EXCEPTION, exceptionCc, null, any())); + register(new HotSpotForeignCallLinkageImpl(EXCEPTION_HANDLER_IN_CALLER, JUMP_ADDRESS, PRESERVES_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE_ONLY_AFTER_EXCEPTION, exceptionCc, null, any())); if (config.useCRC32Intrinsics) { // This stub does callee saving - registerForeignCall(UPDATE_BYTES_CRC32, config.updateBytesCRC32Stub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, any()); + registerForeignCall(UPDATE_BYTES_CRC32, config.updateBytesCRC32Stub, NativeCall, PRESERVES_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE_ONLY_AFTER_EXCEPTION, any()); } if (config.useCRC32CIntrinsics) { - registerForeignCall(UPDATE_BYTES_CRC32C, config.updateBytesCRC32C, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, any()); + registerForeignCall(UPDATE_BYTES_CRC32C, config.updateBytesCRC32C, NativeCall, PRESERVES_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE_ONLY_AFTER_EXCEPTION, any()); } link(new AMD64ArrayIndexOfStub(AMD64ArrayIndexOf.STUB_INDEX_OF_TWO_CONSECUTIVE_BYTES, options, providers, diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotBackendFactory.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotBackendFactory.java index e0182b9acb9..7afb458575f 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotBackendFactory.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotBackendFactory.java @@ -51,6 +51,7 @@ import org.graalvm.compiler.hotspot.word.HotSpotWordTypes; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.Plugins; import org.graalvm.compiler.nodes.spi.LoweringProvider; import org.graalvm.compiler.nodes.spi.Replacements; +import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.common.AddressLoweringPhase; import org.graalvm.compiler.phases.tiers.CompilerConfiguration; import org.graalvm.compiler.phases.util.Providers; @@ -103,21 +104,22 @@ public class SPARCHotSpotBackendFactory implements HotSpotBackendFactory { Providers p = new Providers(metaAccess, codeCache, constantReflection, constantFieldProvider, foreignCalls, lowerer, null, stampProvider); HotSpotSnippetReflectionProvider snippetReflection = new HotSpotSnippetReflectionProvider(runtime, constantReflection, wordTypes); BytecodeProvider bytecodeProvider = new ClassfileBytecodeProvider(metaAccess, snippetReflection); - HotSpotReplacementsImpl replacements = new HotSpotReplacementsImpl(runtime.getOptions(), p, snippetReflection, bytecodeProvider, target); - Plugins plugins = createGraphBuilderPlugins(compilerConfiguration, config, metaAccess, constantReflection, foreignCalls, snippetReflection, replacements, wordTypes); + HotSpotReplacementsImpl replacements = new HotSpotReplacementsImpl(p, snippetReflection, bytecodeProvider, target); + Plugins plugins = createGraphBuilderPlugins(compilerConfiguration, config, metaAccess, constantReflection, foreignCalls, snippetReflection, replacements, wordTypes, runtime.getOptions()); replacements.setGraphBuilderPlugins(plugins); HotSpotSuitesProvider suites = createSuites(config, runtime, compilerConfiguration, plugins, replacements); HotSpotProviders providers = new HotSpotProviders(metaAccess, codeCache, constantReflection, constantFieldProvider, foreignCalls, lowerer, replacements, suites, registers, snippetReflection, wordTypes, plugins); + replacements.setProviders(providers); return createBackend(config, runtime, providers); } protected Plugins createGraphBuilderPlugins(CompilerConfiguration compilerConfiguration, GraalHotSpotVMConfig config, HotSpotMetaAccessProvider metaAccess, HotSpotConstantReflectionProvider constantReflection, HotSpotForeignCallsProvider foreignCalls, - HotSpotSnippetReflectionProvider snippetReflection, HotSpotReplacementsImpl replacements, HotSpotWordTypes wordTypes) { - Plugins plugins = HotSpotGraphBuilderPlugins.create(compilerConfiguration, config, wordTypes, metaAccess, constantReflection, snippetReflection, foreignCalls, replacements); + HotSpotSnippetReflectionProvider snippetReflection, HotSpotReplacementsImpl replacements, HotSpotWordTypes wordTypes, OptionValues options) { + Plugins plugins = HotSpotGraphBuilderPlugins.create(compilerConfiguration, config, wordTypes, metaAccess, constantReflection, snippetReflection, foreignCalls, replacements, options); SPARCGraphBuilderPlugins.register(plugins, replacements.getDefaultReplacementBytecodeProvider(), false); return plugins; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotForeignCallsProvider.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotForeignCallsProvider.java index ff3a1b81e0f..d79f3cd923d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotForeignCallsProvider.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotForeignCallsProvider.java @@ -35,7 +35,7 @@ import static org.graalvm.compiler.hotspot.HotSpotBackend.EXCEPTION_HANDLER_IN_C import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.JUMP_ADDRESS; import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Reexecutability.REEXECUTABLE_ONLY_AFTER_EXCEPTION; import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.RegisterEffect.PRESERVES_REGISTERS; -import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Transition.LEAF_NOFP; +import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Transition.LEAF_NO_VZERO; import static org.graalvm.compiler.hotspot.replacements.CRC32CSubstitutions.UPDATE_BYTES_CRC32C; import static org.graalvm.compiler.hotspot.replacements.CRC32Substitutions.UPDATE_BYTES_CRC32; import static jdk.internal.vm.compiler.word.LocationIdentity.any; @@ -84,16 +84,16 @@ public class SPARCHotSpotForeignCallsProvider extends HotSpotHostForeignCallsPro RegisterValue incomingExceptionPc = i1.asValue(LIRKind.value(word)); CallingConvention outgoingExceptionCc = new CallingConvention(0, ILLEGAL, outgoingException, outgoingExceptionPc); CallingConvention incomingExceptionCc = new CallingConvention(0, ILLEGAL, incomingException, incomingExceptionPc); - register(new HotSpotForeignCallLinkageImpl(EXCEPTION_HANDLER, 0L, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, outgoingExceptionCc, incomingExceptionCc, any())); - register(new HotSpotForeignCallLinkageImpl(EXCEPTION_HANDLER_IN_CALLER, JUMP_ADDRESS, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, outgoingExceptionCc, + register(new HotSpotForeignCallLinkageImpl(EXCEPTION_HANDLER, 0L, PRESERVES_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE_ONLY_AFTER_EXCEPTION, outgoingExceptionCc, incomingExceptionCc, any())); + register(new HotSpotForeignCallLinkageImpl(EXCEPTION_HANDLER_IN_CALLER, JUMP_ADDRESS, PRESERVES_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE_ONLY_AFTER_EXCEPTION, outgoingExceptionCc, incomingExceptionCc, any())); if (config.useCRC32Intrinsics) { // This stub does callee saving - registerForeignCall(UPDATE_BYTES_CRC32, config.updateBytesCRC32Stub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, any()); + registerForeignCall(UPDATE_BYTES_CRC32, config.updateBytesCRC32Stub, NativeCall, PRESERVES_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE_ONLY_AFTER_EXCEPTION, any()); } if (config.useCRC32CIntrinsics) { - registerForeignCall(UPDATE_BYTES_CRC32C, config.updateBytesCRC32C, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, any()); + registerForeignCall(UPDATE_BYTES_CRC32C, config.updateBytesCRC32C, NativeCall, PRESERVES_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE_ONLY_AFTER_EXCEPTION, any()); } super.initialize(providers, options); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/ArrayCopyIntrinsificationTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/ArrayCopyIntrinsificationTest.java index 2fb73674e18..a0daecee752 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/ArrayCopyIntrinsificationTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/ArrayCopyIntrinsificationTest.java @@ -162,7 +162,7 @@ public class ArrayCopyIntrinsificationTest extends GraalCompilerTest { } /** - * Tests {@link ArrayCopySnippets#arraycopyCheckcastSnippet}. + * Tests {@link ArrayCopySnippets#arraycopyGenericSnippet} with checkcast. */ @Test public void testArrayStoreException() { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CheckGraalIntrinsics.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CheckGraalIntrinsics.java index 4433a1b380b..0d0453cd71d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CheckGraalIntrinsics.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CheckGraalIntrinsics.java @@ -381,8 +381,11 @@ public class CheckGraalIntrinsics extends GraalTest { if (isJDK11OrHigher()) { // Relevant for Java flight recorder add(toBeInvestigated, - "java/util/Base64$Encoder.encodeBlock([BII[BIZ)V", "jdk/jfr/internal/JVM.getEventWriter()Ljava/lang/Object;"); + if (!config.useBase64Intrinsics()) { + add(ignore, + "java/util/Base64$Encoder.encodeBlock([BII[BIZ)V"); + } } if (isJDK12OrHigher()) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CompileTheWorld.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CompileTheWorld.java index f7fdfd10e49..43b0db7d254 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CompileTheWorld.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CompileTheWorld.java @@ -32,7 +32,9 @@ import static org.graalvm.compiler.core.test.ReflectionOptionDescriptors.extract import static org.graalvm.compiler.debug.MemUseTrackerKey.getCurrentThreadAllocatedBytes; import static org.graalvm.compiler.hotspot.test.CompileTheWorld.Options.DESCRIPTORS; import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; +import static sun.misc.Unsafe.ARRAY_BYTE_BASE_OFFSET; +import java.io.ByteArrayOutputStream; import java.io.Closeable; import java.io.File; import java.io.IOException; @@ -73,10 +75,11 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import jdk.internal.vm.compiler.collections.EconomicMap; -import jdk.internal.vm.compiler.collections.UnmodifiableEconomicMap; +import jdk.internal.vm.compiler.collections.UnmodifiableMapCursor; import org.graalvm.compiler.api.replacements.Snippet; import org.graalvm.compiler.bytecode.Bytecodes; import org.graalvm.compiler.core.CompilerThreadFactory; +import org.graalvm.compiler.core.phases.HighTier; import org.graalvm.compiler.core.test.ReflectionOptionDescriptors; import org.graalvm.compiler.debug.DebugOptions; import org.graalvm.compiler.debug.GraalError; @@ -86,11 +89,15 @@ import org.graalvm.compiler.hotspot.CompilationTask; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; import org.graalvm.compiler.hotspot.HotSpotGraalCompiler; import org.graalvm.compiler.hotspot.HotSpotGraalRuntimeProvider; +import org.graalvm.compiler.hotspot.test.CompileTheWorld.LibGraalParams.StackTraceBuffer; import org.graalvm.compiler.options.OptionDescriptors; import org.graalvm.compiler.options.OptionKey; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.options.OptionsParser; +import org.graalvm.compiler.serviceprovider.GraalUnsafeAccess; import org.graalvm.compiler.serviceprovider.JavaVersionUtil; +import jdk.internal.vm.compiler.libgraal.LibGraal; +import jdk.internal.vm.compiler.libgraal.OptionsEncoder; import jdk.vm.ci.hotspot.HotSpotCodeCacheProvider; import jdk.vm.ci.hotspot.HotSpotCompilationRequest; @@ -102,6 +109,7 @@ import jdk.vm.ci.meta.ConstantPool; import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.runtime.JVMCI; import jdk.vm.ci.runtime.JVMCICompiler; +import sun.misc.Unsafe; /** * This class implements compile-the-world functionality with JVMCI. @@ -127,17 +135,19 @@ public final class CompileTheWorld { * Ignored if null. */ public static EconomicMap, Object> parseOptions(String options) { + EconomicMap, Object> values = OptionValues.newOptionMap(); if (options != null) { EconomicMap optionSettings = EconomicMap.create(); for (String optionSetting : options.split("\\s+|#")) { OptionsParser.parseOptionSettingTo(optionSetting, optionSettings); } - EconomicMap, Object> values = OptionValues.newOptionMap(); ServiceLoader loader = ServiceLoader.load(OptionDescriptors.class, OptionDescriptors.class.getClassLoader()); OptionsParser.parseOptions(optionSettings, values, loader); - return values; } - return EconomicMap.create(); + if (!values.containsKey(HighTier.Options.Inline)) { + values.put(HighTier.Options.Inline, false); + } + return values; } private final HotSpotJVMCIRuntime jvmciRuntime; @@ -165,6 +175,13 @@ public final class CompileTheWorld { */ private final int stopAt; + /** + * Max classes to compile. + * + * @see Options#MaxClasses + */ + private final int maxClasses; + /** Only compile methods matching one of the filters in this array if the array is non-null. */ private final MethodFilter[] methodFilters; @@ -186,8 +203,130 @@ public final class CompileTheWorld { private ThreadPoolExecutor threadPool; - private OptionValues currentOptions; - private final UnmodifiableEconomicMap, Object> compilationOptions; + /** + * Values for {@link CompileTheWorld.Options}. + */ + private final OptionValues harnessOptions; + + /** + * Option values used during compilation. + */ + private final OptionValues compilerOptions; + + /** + * Manages native memory buffers for passing arguments into libgraal and receiving return + * values. The native memory buffers are freed when this object is {@linkplain #close() closed}. + */ + static class LibGraalParams implements AutoCloseable { + + static { + LibGraal.registerNativeMethods(HotSpotJVMCIRuntime.runtime(), CompileTheWorld.class); + } + + /** + * Native memory containing {@linkplain OptionsEncoder encoded} {@link OptionValues}. + */ + static class OptionsBuffer { + private long address; + final int size; + final int hash; + + OptionsBuffer(OptionValues options) { + Map map = new HashMap<>(); + UnmodifiableMapCursor, Object> cursor = options.getMap().getEntries(); + while (cursor.advance()) { + final OptionKey key = cursor.getKey(); + Object value = cursor.getValue(); + map.put(key.getName(), value); + } + + byte[] encoded = OptionsEncoder.encode(map); + size = encoded.length; + hash = Arrays.hashCode(encoded); + address = UNSAFE.allocateMemory(encoded.length); + UNSAFE.copyMemory(encoded, ARRAY_BYTE_BASE_OFFSET, null, address, size); + } + + long getAddress() { + if (address == 0) { + throw new IllegalStateException(); + } + return address; + } + + void free() { + if (address != 0) { + UNSAFE.freeMemory(address); + address = 0; + } + } + } + + /** + * Manages native memory for receiving a {@linkplain Throwable#printStackTrace() stack + * trace} from libgraal serialized via {@link ByteArrayOutputStream} to a byte array. + */ + static class StackTraceBuffer { + final int size; + private long address; + + StackTraceBuffer(int size) { + this.size = size; + address = UNSAFE.allocateMemory(size); + } + + void free() { + if (address != 0L) { + UNSAFE.freeMemory(address); + address = 0L; + } + } + + long getAddress() { + if (address == 0) { + throw new IllegalStateException(); + } + return address; + } + } + + final OptionsBuffer options; + + private final List stackTraceBuffers = new ArrayList<>(); + + /** + * Gets a stack trace buffer for the current thread. + */ + StackTraceBuffer getStackTraceBuffer() { + return stackTraceBuffer.get(); + } + + private final ThreadLocal stackTraceBuffer = new ThreadLocal() { + @Override + protected StackTraceBuffer initialValue() { + StackTraceBuffer buffer = new StackTraceBuffer(10_000); + synchronized (stackTraceBuffers) { + stackTraceBuffers.add(buffer); + } + return buffer; + } + }; + + LibGraalParams(OptionValues options) { + this.options = new OptionsBuffer(options); + } + + @Override + public void close() { + options.free(); + synchronized (stackTraceBuffers) { + for (StackTraceBuffer buffer : stackTraceBuffers) { + buffer.free(); + } + stackTraceBuffers.clear(); + } + } + } /** * Creates a compile-the-world instance. @@ -195,73 +334,99 @@ public final class CompileTheWorld { * @param files {@link File#pathSeparator} separated list of Zip/Jar files to compile * @param startAt index of the class file to start compilation at * @param stopAt index of the class file to stop compilation at + * @param maxClasses maximum number of classes to process * @param methodFilters * @param excludeMethodFilters + * @param harnessOptions values for {@link CompileTheWorld.Options} + * @param compilerOptions option values used by the compiler */ - public CompileTheWorld(HotSpotJVMCIRuntime jvmciRuntime, HotSpotGraalCompiler compiler, String files, int startAt, int stopAt, String methodFilters, String excludeMethodFilters, - boolean verbose, OptionValues initialOptions, EconomicMap, Object> compilationOptions) { + public CompileTheWorld(HotSpotJVMCIRuntime jvmciRuntime, + HotSpotGraalCompiler compiler, + String files, + int startAt, + int stopAt, + int maxClasses, + String methodFilters, + String excludeMethodFilters, + boolean verbose, + OptionValues harnessOptions, + OptionValues compilerOptions) { this.jvmciRuntime = jvmciRuntime; this.compiler = compiler; this.inputClassPath = files; - this.startAt = startAt; - this.stopAt = stopAt; + this.startAt = Math.max(startAt, 1); + this.stopAt = Math.max(stopAt, 1); + this.maxClasses = Math.max(maxClasses, 1); this.methodFilters = methodFilters == null || methodFilters.isEmpty() ? null : MethodFilter.parse(methodFilters); this.excludeMethodFilters = excludeMethodFilters == null || excludeMethodFilters.isEmpty() ? null : MethodFilter.parse(excludeMethodFilters); this.verbose = verbose; - this.currentOptions = initialOptions; + this.harnessOptions = harnessOptions; // Copy the initial options and add in any extra options - EconomicMap, Object> compilationOptionsCopy = EconomicMap.create(initialOptions.getMap()); - compilationOptionsCopy.putAll(compilationOptions); + EconomicMap, Object> compilerOptionsMap = EconomicMap.create(compilerOptions.getMap()); // We want to see stack traces when a method fails to compile - CompilationBailoutAsFailure.putIfAbsent(compilationOptionsCopy, true); - CompilationFailureAction.putIfAbsent(compilationOptionsCopy, Print); + CompilationBailoutAsFailure.putIfAbsent(compilerOptionsMap, true); + CompilationFailureAction.putIfAbsent(compilerOptionsMap, Print); // By default only report statistics for the CTW threads themselves - DebugOptions.MetricsThreadFilter.putIfAbsent(compilationOptionsCopy, "^CompileTheWorld"); - this.compilationOptions = compilationOptionsCopy; + DebugOptions.MetricsThreadFilter.putIfAbsent(compilerOptionsMap, "^CompileTheWorld"); + this.compilerOptions = new OptionValues(compilerOptionsMap); } - public CompileTheWorld(HotSpotJVMCIRuntime jvmciRuntime, HotSpotGraalCompiler compiler, OptionValues options) { - this(jvmciRuntime, compiler, Options.Classpath.getValue(options), - Options.StartAt.getValue(options), - Options.StopAt.getValue(options), - Options.MethodFilter.getValue(options), - Options.ExcludeMethodFilter.getValue(options), - Options.Verbose.getValue(options), - options, - parseOptions(Options.Config.getValue(options))); + public CompileTheWorld(HotSpotJVMCIRuntime jvmciRuntime, + HotSpotGraalCompiler compiler, + OptionValues harnessOptions, + OptionValues compilerOptions) { + this(jvmciRuntime, compiler, Options.Classpath.getValue(harnessOptions), + Options.StartAt.getValue(harnessOptions), + Options.StopAt.getValue(harnessOptions), + Options.MaxClasses.getValue(harnessOptions), + Options.MethodFilter.getValue(harnessOptions), + Options.ExcludeMethodFilter.getValue(harnessOptions), + Options.Verbose.getValue(harnessOptions), + harnessOptions, + new OptionValues(compilerOptions, parseOptions(Options.Config.getValue(harnessOptions)))); } /** * Compiles all methods in all classes in {@link #inputClassPath}. If {@link #inputClassPath} * equals {@link #SUN_BOOT_CLASS_PATH} the boot classes are used. */ + @SuppressWarnings("try") public void compile() throws Throwable { - if (SUN_BOOT_CLASS_PATH.equals(inputClassPath)) { - String bcpEntry = null; - if (Java8OrEarlier) { - final String[] entries = System.getProperty(SUN_BOOT_CLASS_PATH).split(File.pathSeparator); - for (int i = 0; i < entries.length && bcpEntry == null; i++) { - String entry = entries[i]; - File entryFile = new File(entry); - if (entryFile.getName().endsWith("rt.jar") && entryFile.isFile()) { - bcpEntry = entry; + try (LibGraalParams libgraal = LibGraal.isAvailable() ? new LibGraalParams(compilerOptions) : null) { + if (SUN_BOOT_CLASS_PATH.equals(inputClassPath)) { + String bcpEntry = null; + if (Java8OrEarlier) { + final String[] entries = System.getProperty(SUN_BOOT_CLASS_PATH).split(File.pathSeparator); + for (int i = 0; i < entries.length && bcpEntry == null; i++) { + String entry = entries[i]; + File entryFile = new File(entry); + if (entryFile.getName().endsWith("rt.jar") && entryFile.isFile()) { + bcpEntry = entry; + } } + if (bcpEntry == null) { + throw new GraalError("Could not find rt.jar on boot class path %s", System.getProperty(SUN_BOOT_CLASS_PATH)); + } + } else { + bcpEntry = JRT_CLASS_PATH_ENTRY; } - if (bcpEntry == null) { - throw new GraalError("Could not find rt.jar on boot class path %s", System.getProperty(SUN_BOOT_CLASS_PATH)); - } + compile(bcpEntry, libgraal); } else { - bcpEntry = JRT_CLASS_PATH_ENTRY; + compile(inputClassPath, libgraal); } - compile(bcpEntry); - } else { - compile(inputClassPath); } } + private AutoCloseable enterCompilation() { + if (!LibGraal.isAvailable()) { + return null; + } + return new LibGraalParams(compilerOptions); + } + public void println() { println(""); } @@ -486,6 +651,19 @@ public final class CompileTheWorld { return true; } + private ClassPathEntry openClassPathEntry(String entry) throws IOException { + if (entry.endsWith(".zip") || entry.endsWith(".jar")) { + return new JarClassPathEntry(entry); + } else if (entry.equals(JRT_CLASS_PATH_ENTRY)) { + return new JRTClassPathEntry(entry, Options.LimitModules.getValue(harnessOptions)); + } else { + if (!new File(entry).isDirectory()) { + return null; + } + return new DirClassPathEntry(entry); + } + } + /** * Compiles all methods in all classes in a given class path. * @@ -493,23 +671,25 @@ public final class CompileTheWorld { * @throws IOException */ @SuppressWarnings("try") - private void compile(String classPath) throws IOException { + private void compile(String classPath, LibGraalParams libgraal) throws IOException { final String[] entries = classPath.split(File.pathSeparator); long start = System.currentTimeMillis(); Map initialThreads = Thread.getAllStackTraces(); - try { - // compile dummy method to get compiler initialized outside of the - // config debug override. - HotSpotResolvedJavaMethod dummyMethod = (HotSpotResolvedJavaMethod) JVMCI.getRuntime().getHostJVMCIBackend().getMetaAccess().lookupJavaMethod( - CompileTheWorld.class.getDeclaredMethod("dummy")); - int entryBCI = JVMCICompiler.INVOCATION_ENTRY_BCI; - boolean useProfilingInfo = false; - boolean installAsDefault = false; - CompilationTask task = new CompilationTask(jvmciRuntime, compiler, new HotSpotCompilationRequest(dummyMethod, entryBCI, 0L), useProfilingInfo, installAsDefault, currentOptions); - task.runCompilation(); - } catch (NoSuchMethodException | SecurityException e1) { - printStackTrace(e1); + if (libgraal == null) { + try { + // compile dummy method to get compiler initialized outside of the + // config debug override. + HotSpotResolvedJavaMethod dummyMethod = (HotSpotResolvedJavaMethod) JVMCI.getRuntime().getHostJVMCIBackend().getMetaAccess().lookupJavaMethod( + CompileTheWorld.class.getDeclaredMethod("dummy")); + int entryBCI = JVMCICompiler.INVOCATION_ENTRY_BCI; + boolean useProfilingInfo = false; + boolean installAsDefault = false; + CompilationTask task = new CompilationTask(jvmciRuntime, compiler, new HotSpotCompilationRequest(dummyMethod, entryBCI, 0L), useProfilingInfo, installAsDefault); + task.runCompilation(compilerOptions); + } catch (NoSuchMethodException | SecurityException e1) { + printStackTrace(e1); + } } /* @@ -517,8 +697,8 @@ public final class CompileTheWorld { * DebugValueThreadFilter to filter on the thread names. */ int threadCount = 1; - if (Options.MultiThreaded.getValue(currentOptions)) { - threadCount = Options.Threads.getValue(currentOptions); + if (Options.MultiThreaded.getValue(harnessOptions)) { + threadCount = Options.Threads.getValue(harnessOptions); if (threadCount == 0) { threadCount = Runtime.getRuntime().availableProcessors(); } @@ -526,26 +706,37 @@ public final class CompileTheWorld { running = true; } - OptionValues savedOptions = currentOptions; - currentOptions = new OptionValues(compilationOptions); threadPool = new ThreadPoolExecutor(threadCount, threadCount, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue(), new CompilerThreadFactory("CompileTheWorld")); - try { - for (int i = 0; i < entries.length; i++) { - final String entry = entries[i]; - - ClassPathEntry cpe; - if (entry.endsWith(".zip") || entry.endsWith(".jar")) { - cpe = new JarClassPathEntry(entry); - } else if (entry.equals(JRT_CLASS_PATH_ENTRY)) { - cpe = new JRTClassPathEntry(entry, Options.LimitModules.getValue(currentOptions)); - } else { - if (!new File(entry).isDirectory()) { - println("CompileTheWorld : Skipped classes in " + entry); - println(); - continue; + int compileStartAt = startAt; + int compileStopAt = stopAt; + int compileStep = 1; + if (maxClasses != Integer.MAX_VALUE) { + int totalClassFileCount = 0; + for (String entry : entries) { + try (ClassPathEntry cpe = openClassPathEntry(entry)) { + if (cpe != null) { + totalClassFileCount += cpe.getClassNames().size(); } - cpe = new DirClassPathEntry(entry); + } + } + + int lastClassFile = totalClassFileCount - 1; + compileStartAt = Math.min(startAt, lastClassFile); + compileStopAt = Math.min(stopAt, lastClassFile); + int range = compileStopAt - compileStartAt + 1; + if (maxClasses < range) { + compileStep = range / maxClasses; + } + } + + for (int i = 0; i < entries.length; i++) { + final String entry = entries[i]; + try (ClassPathEntry cpe = openClassPathEntry(entry)) { + if (cpe == null) { + println("CompileTheWorld : Skipped classes in " + entry); + println(); + continue; } if (methodFilters == null || methodFilters.length == 0) { @@ -565,12 +756,16 @@ public final class CompileTheWorld { for (String className : cpe.getClassNames()) { // Are we done? - if (classFileCounter >= stopAt) { + if (classFileCounter >= compileStopAt) { break; } classFileCounter++; + if (compileStep > 1 && ((classFileCounter - compileStartAt) % compileStep) != 0) { + continue; + } + if (className.startsWith("jdk.management.") || className.startsWith("jdk.internal.cmm.*") || // GR-5881: The class initializer for @@ -606,28 +801,29 @@ public final class CompileTheWorld { } // Are we compiling this class? - if (classFileCounter >= startAt) { - println("CompileTheWorld (%d) : %s", classFileCounter, className); + if (classFileCounter >= compileStartAt) { + long start0 = System.currentTimeMillis(); // Compile each constructor/method in the class. for (Constructor constructor : javaClass.getDeclaredConstructors()) { HotSpotResolvedJavaMethod javaMethod = (HotSpotResolvedJavaMethod) metaAccess.lookupJavaMethod(constructor); if (canBeCompiled(javaMethod, constructor.getModifiers())) { - compileMethod(javaMethod); + compileMethod(javaMethod, libgraal); } } for (Method method : javaClass.getDeclaredMethods()) { HotSpotResolvedJavaMethod javaMethod = (HotSpotResolvedJavaMethod) metaAccess.lookupJavaMethod(method); if (canBeCompiled(javaMethod, method.getModifiers())) { - compileMethod(javaMethod); + compileMethod(javaMethod, libgraal); } } // Also compile the class initializer if it exists HotSpotResolvedJavaMethod clinit = (HotSpotResolvedJavaMethod) metaAccess.lookupJavaType(javaClass).getClassInitializer(); if (clinit != null && canBeCompiled(clinit, clinit.getModifiers())) { - compileMethod(clinit); + compileMethod(clinit, libgraal); } + println("CompileTheWorld (%d) : %s (%d ms)", classFileCounter, className, System.currentTimeMillis() - start0); } } catch (Throwable t) { if (isClassIncluded(className)) { @@ -636,10 +832,7 @@ public final class CompileTheWorld { } } } - cpe.close(); } - } finally { - currentOptions = savedOptions; } if (!running) { @@ -661,11 +854,12 @@ public final class CompileTheWorld { long elapsedTime = System.currentTimeMillis() - start; println(); - if (Options.MultiThreaded.getValue(currentOptions)) { - TTY.println("CompileTheWorld : Done (%d classes, %d methods, %d ms elapsed, %d ms compile time, %d bytes of memory used)", classFileCounter, compiledMethodsCounter.get(), elapsedTime, + int compiledClasses = classFileCounter > compileStartAt ? classFileCounter - compileStartAt : 0; + if (Options.MultiThreaded.getValue(harnessOptions)) { + TTY.println("CompileTheWorld : Done (%d classes, %d methods, %d ms elapsed, %d ms compile time, %d bytes of memory used)", compiledClasses, compiledMethodsCounter.get(), elapsedTime, compileTime.get(), memoryUsed.get()); } else { - TTY.println("CompileTheWorld : Done (%d classes, %d methods, %d ms, %d bytes of memory used)", classFileCounter, compiledMethodsCounter.get(), compileTime.get(), memoryUsed.get()); + TTY.println("CompileTheWorld : Done (%d classes, %d methods, %d ms, %d bytes of memory used)", compiledClasses, compiledMethodsCounter.get(), compileTime.get(), memoryUsed.get()); } // Apart from the main thread, there should be only be daemon threads @@ -712,7 +906,7 @@ public final class CompileTheWorld { } @SuppressWarnings("try") - private void compileMethod(HotSpotResolvedJavaMethod method) throws InterruptedException, ExecutionException { + private void compileMethod(HotSpotResolvedJavaMethod method, LibGraalParams libgraal) throws InterruptedException, ExecutionException { if (methodFilters != null && !MethodFilter.matches(methodFilters, method)) { return; } @@ -723,13 +917,7 @@ public final class CompileTheWorld { @Override public void run() { waitToRun(); - OptionValues savedOptions = currentOptions; - currentOptions = new OptionValues(compilationOptions); - try { - compileMethod(method, classFileCounter); - } finally { - currentOptions = savedOptions; - } + compileMethod(method, classFileCounter, libgraal); } }); if (threadPool.getCorePoolSize() == 1) { @@ -737,23 +925,66 @@ public final class CompileTheWorld { } } + private static final Unsafe UNSAFE = GraalUnsafeAccess.getUnsafe(); + + static native long compileMethodInLibgraal(long isolateThread, + long methodHandle, + boolean useProfilingInfo, + boolean installAsDefault, + long optionsAddress, + int optionsSize, + int optionsHash, + long encodedThrowableBufferAddress, + int encodedThrowableBufferSize); + /** * Compiles a method and gathers some statistics. */ - private void compileMethod(HotSpotResolvedJavaMethod method, int counter) { + private void compileMethod(HotSpotResolvedJavaMethod method, int counter, LibGraalParams libgraal) { try { long start = System.currentTimeMillis(); long allocatedAtStart = getCurrentThreadAllocatedBytes(); - int entryBCI = JVMCICompiler.INVOCATION_ENTRY_BCI; - HotSpotCompilationRequest request = new HotSpotCompilationRequest(method, entryBCI, 0L); // For more stable CTW execution, disable use of profiling information boolean useProfilingInfo = false; boolean installAsDefault = false; - CompilationTask task = new CompilationTask(jvmciRuntime, compiler, request, useProfilingInfo, installAsDefault, currentOptions); - task.runCompilation(); + HotSpotInstalledCode installedCode; + if (libgraal != null) { + HotSpotJVMCIRuntime runtime = HotSpotJVMCIRuntime.runtime(); + long methodHandle = LibGraal.translate(runtime, method); + long isolateThread = LibGraal.getIsolateThread(); + + StackTraceBuffer stackTraceBuffer = libgraal.getStackTraceBuffer(); + + long stackTraceBufferAddress = stackTraceBuffer.getAddress(); + long installedCodeHandle = compileMethodInLibgraal(isolateThread, + methodHandle, + useProfilingInfo, + installAsDefault, + libgraal.options.getAddress(), + libgraal.options.size, + libgraal.options.hash, + stackTraceBufferAddress, + stackTraceBuffer.size); + + installedCode = LibGraal.unhand(runtime, HotSpotInstalledCode.class, installedCodeHandle); + if (installedCode == null) { + int length = UNSAFE.getInt(stackTraceBufferAddress); + byte[] data = new byte[length]; + UNSAFE.copyMemory(null, stackTraceBufferAddress + Integer.BYTES, data, ARRAY_BYTE_BASE_OFFSET, length); + String stackTrace = new String(data).trim(); + println("CompileTheWorld (%d) : Error compiling method: %s", counter, method.format("%H.%n(%p):%r")); + println(stackTrace); + } + + } else { + int entryBCI = JVMCICompiler.INVOCATION_ENTRY_BCI; + HotSpotCompilationRequest request = new HotSpotCompilationRequest(method, entryBCI, 0L); + CompilationTask task = new CompilationTask(jvmciRuntime, compiler, request, useProfilingInfo, installAsDefault); + task.runCompilation(compilerOptions); + installedCode = task.getInstalledCode(); + } // Invalidate the generated code so the code cache doesn't fill up - HotSpotInstalledCode installedCode = task.getInstalledCode(); if (installedCode != null) { installedCode.invalidate(); } @@ -799,13 +1030,12 @@ public final class CompileTheWorld { } static class Options { - // @formatter:off public static final OptionKey Help = new OptionKey<>(false); public static final OptionKey Classpath = new OptionKey<>(CompileTheWorld.SUN_BOOT_CLASS_PATH); public static final OptionKey Verbose = new OptionKey<>(true); /** - * Ignore Graal classes by default to avoid problems associated with compiling - * snippets and method substitutions. + * Ignore Graal classes by default to avoid problems associated with compiling snippets and + * method substitutions. */ public static final OptionKey LimitModules = new OptionKey<>("~jdk.internal.vm.compiler"); public static final OptionKey Iterations = new OptionKey<>(1); @@ -813,10 +1043,12 @@ public final class CompileTheWorld { public static final OptionKey ExcludeMethodFilter = new OptionKey<>(null); public static final OptionKey StartAt = new OptionKey<>(1); public static final OptionKey StopAt = new OptionKey<>(Integer.MAX_VALUE); + public static final OptionKey MaxClasses = new OptionKey<>(Integer.MAX_VALUE); public static final OptionKey Config = new OptionKey<>(null); public static final OptionKey MultiThreaded = new OptionKey<>(false); public static final OptionKey Threads = new OptionKey<>(0); + // @formatter:off static final ReflectionOptionDescriptors DESCRIPTORS = new ReflectionOptionDescriptors(Options.class, "Help", "List options and their help messages and then exit.", "Classpath", "Class path denoting methods to compile. Default is to compile boot classes.", @@ -826,21 +1058,24 @@ public final class CompileTheWorld { "Iterations", "The number of iterations to perform.", "MethodFilter", "Only compile methods matching this filter.", "ExcludeMethodFilter", "Exclude methods matching this filter from compilation.", - "StartAt", "First class to consider for compilation.", - "StopAt", "Last class to consider for compilation.", - "Config", "Option value overrides to use during compile the world. For example, " + - "to disable inlining and partial escape analysis specify 'PartialEscapeAnalysis=false Inline=false'. " + - "The format for each option is the same as on the command line just without the '-Dgraal.' prefix.", + "StartAt", "First class to consider for compilation (default = 1).", + "StopAt", "Last class to consider for compilation (default = ).", + "MaxClasses", "Maximum number of classes to process (default = ). " + + "Ignored if less than (StopAt - StartAt + 1).", + "Config", "Option values to use during compile the world compilations. For example, " + + "to disable partial escape analysis and print compilations specify " + + "'PartialEscapeAnalysis=false PrintCompilation=true'. " + + "Unless explicitly enabled with 'Inline=true' here, inlining is disabled.", "MultiThreaded", "Run using multiple threads for compilation.", "Threads", "Number of threads to use for multithreaded execution. Defaults to Runtime.getRuntime().availableProcessors()."); // @formatter:on } - public static OptionValues loadOptions(OptionValues initialValues) { + public static OptionValues loadHarnessOptions() { EconomicMap, Object> values = OptionValues.newOptionMap(); List loader = singletonList(DESCRIPTORS); OptionsParser.parseOptions(extractEntries(System.getProperties(), "CompileTheWorld.", true), values, loader); - OptionValues options = new OptionValues(initialValues, values); + OptionValues options = new OptionValues(values); if (Options.Help.getValue(options)) { options.printHelp(loader, System.out, "CompileTheWorld."); System.exit(0); @@ -853,14 +1088,14 @@ public final class CompileTheWorld { HotSpotGraalCompiler compiler = (HotSpotGraalCompiler) jvmciRuntime.getCompiler(); HotSpotGraalRuntimeProvider graalRuntime = compiler.getGraalRuntime(); HotSpotCodeCacheProvider codeCache = graalRuntime.getHostProviders().getCodeCache(); - OptionValues options = loadOptions(graalRuntime.getOptions()); + OptionValues harnessOptions = loadHarnessOptions(); - int iterations = Options.Iterations.getValue(options); + int iterations = Options.Iterations.getValue(harnessOptions); for (int i = 0; i < iterations; i++) { codeCache.resetCompilationStatistics(); TTY.println("CompileTheWorld : iteration " + i); - CompileTheWorld ctw = new CompileTheWorld(jvmciRuntime, compiler, options); + CompileTheWorld ctw = new CompileTheWorld(jvmciRuntime, compiler, harnessOptions, graalRuntime.getOptions()); ctw.compile(); } // This is required as non-daemon threads can be started by class initializers diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CompileTheWorldTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CompileTheWorldTest.java index 12e4bf728bb..e6a1a35e52a 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CompileTheWorldTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CompileTheWorldTest.java @@ -27,11 +27,10 @@ package org.graalvm.compiler.hotspot.test; import static org.graalvm.compiler.core.GraalCompilerOptions.CompilationBailoutAsFailure; import static org.graalvm.compiler.core.GraalCompilerOptions.CompilationFailureAction; -import jdk.internal.vm.compiler.collections.EconomicMap; import org.graalvm.compiler.core.CompilationWrapper.ExceptionAction; +import org.graalvm.compiler.core.phases.HighTier; import org.graalvm.compiler.core.test.GraalCompilerTest; import org.graalvm.compiler.hotspot.HotSpotGraalCompiler; -import org.graalvm.compiler.options.OptionKey; import org.graalvm.compiler.options.OptionValues; import org.junit.Test; @@ -50,8 +49,25 @@ public class CompileTheWorldTest extends GraalCompilerTest { HotSpotJVMCIRuntime runtime = HotSpotJVMCIRuntime.runtime(); System.setProperty("CompileTheWorld.LimitModules", "java.base"); OptionValues initialOptions = getInitialOptions(); - EconomicMap, Object> compilationOptions = CompileTheWorld.parseOptions("Inline=false"); - new CompileTheWorld(runtime, (HotSpotGraalCompiler) runtime.getCompiler(), CompileTheWorld.SUN_BOOT_CLASS_PATH, 1, 5, null, null, false, initialOptions, compilationOptions).compile(); + OptionValues harnessOptions = new OptionValues(OptionValues.newOptionMap()); + int startAt = 1; + int stopAt = 5; + int maxClasses = Integer.MAX_VALUE; + String methodFilters = null; + String excludeMethodFilters = null; + boolean verbose = false; + CompileTheWorld ctw = new CompileTheWorld(runtime, + (HotSpotGraalCompiler) runtime.getCompiler(), + CompileTheWorld.SUN_BOOT_CLASS_PATH, + startAt, + stopAt, + maxClasses, + methodFilters, + excludeMethodFilters, + verbose, + harnessOptions, + new OptionValues(initialOptions, HighTier.Options.Inline, false)); + ctw.compile(); assert CompilationBailoutAsFailure.getValue(initialOptions) == originalBailoutAction; assert CompilationFailureAction.getValue(initialOptions) == originalFailureAction; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/GraalOSRTestBase.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/GraalOSRTestBase.java index 9c468808110..315967c21c0 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/GraalOSRTestBase.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/GraalOSRTestBase.java @@ -79,7 +79,7 @@ public abstract class GraalOSRTestBase extends GraalCompilerTest { long jvmciEnv = 0L; HotSpotCompilationRequest request = new HotSpotCompilationRequest((HotSpotResolvedJavaMethod) method, bci, jvmciEnv); HotSpotGraalCompiler compiler = (HotSpotGraalCompiler) runtime.getCompiler(); - CompilationTask task = new CompilationTask(runtime, compiler, request, true, true, debug.getOptions()); + CompilationTask task = new CompilationTask(runtime, compiler, request, true, true); if (method instanceof HotSpotResolvedJavaMethod) { HotSpotGraalRuntimeProvider graalRuntime = compiler.getGraalRuntime(); GraalHotSpotVMConfig config = graalRuntime.getVMConfig(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/HotSpotBase64Test.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/HotSpotBase64Test.java new file mode 100644 index 00000000000..d98e06fef7e --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/HotSpotBase64Test.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.test; + +import static org.junit.Assume.assumeTrue; + +import java.util.Base64; + +import org.graalvm.compiler.api.test.Graal; +import org.graalvm.compiler.hotspot.HotSpotGraalRuntimeProvider; +import org.graalvm.compiler.runtime.RuntimeProvider; +import org.junit.Before; +import org.junit.Test; + +public class HotSpotBase64Test extends HotSpotGraalCompilerTest { + + // Checkstyle: stop + private static final String lipsum = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata "; + // Checkstyle: resume + + @Before + public void sanity() { + HotSpotGraalRuntimeProvider rt = (HotSpotGraalRuntimeProvider) Graal.getRequiredCapability(RuntimeProvider.class); + assumeTrue("Enable test case when the hotspot intrinsic is available", rt.getVMConfig().useBase64Intrinsics()); + } + + @Test + public void testEncode() { + test(getResolvedJavaMethod(Base64.Encoder.class, "encode", byte[].class), Base64.getEncoder(), lipsum.getBytes()); + } + +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/MemoryUsageBenchmark.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/MemoryUsageBenchmark.java index 23dc5b24cb7..4cf90dd534e 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/MemoryUsageBenchmark.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/MemoryUsageBenchmark.java @@ -137,8 +137,8 @@ public class MemoryUsageBenchmark extends HotSpotGraalCompilerTest { HotSpotJVMCIRuntime runtime = HotSpotJVMCIRuntime.runtime(); int entryBCI = JVMCICompiler.INVOCATION_ENTRY_BCI; HotSpotCompilationRequest request = new HotSpotCompilationRequest(method, entryBCI, jvmciEnv); - CompilationTask task = new CompilationTask(runtime, (HotSpotGraalCompiler) runtime.getCompiler(), request, true, false, getInitialOptions()); - task.runCompilation(); + CompilationTask task = new CompilationTask(runtime, (HotSpotGraalCompiler) runtime.getCompiler(), request, true, false); + task.runCompilation(getInitialOptions()); } } @@ -156,8 +156,8 @@ public class MemoryUsageBenchmark extends HotSpotGraalCompilerTest { HotSpotCompilationRequest request = new HotSpotCompilationRequest(method, JVMCICompiler.INVOCATION_ENTRY_BCI, jvmciEnv); HotSpotGraalCompiler compiler = (HotSpotGraalCompiler) runtime.getCompiler(); OptionValues options = getInitialOptions(); - CompilationTask task = new CompilationTask(runtime, compiler, request, true, false, options); - task.runCompilation(); + CompilationTask task = new CompilationTask(runtime, compiler, request, true, false); + task.runCompilation(options); } } } @@ -180,10 +180,10 @@ public class MemoryUsageBenchmark extends HotSpotGraalCompilerTest { public void run() { compileAndTime("simple"); compileAndTime("complex"); - OptionValues options = CompileTheWorld.loadOptions(getInitialOptions()); - if (CompileTheWorld.Options.Classpath.getValue(options) != CompileTheWorld.SUN_BOOT_CLASS_PATH) { + OptionValues harnessOptions = CompileTheWorld.loadHarnessOptions(); + if (CompileTheWorld.Options.Classpath.getValue(harnessOptions) != CompileTheWorld.SUN_BOOT_CLASS_PATH) { HotSpotJVMCIRuntime runtime = HotSpotJVMCIRuntime.runtime(); - CompileTheWorld ctw = new CompileTheWorld(runtime, (HotSpotGraalCompiler) runtime.getCompiler(), options); + CompileTheWorld ctw = new CompileTheWorld(runtime, (HotSpotGraalCompiler) runtime.getCompiler(), harnessOptions, getInitialOptions()); try { ctw.compile(); } catch (Throwable e) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/ObjectHashCodeInliningTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/ObjectHashCodeInliningTest.java new file mode 100644 index 00000000000..4dbcd542a59 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/ObjectHashCodeInliningTest.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.test; + +import jdk.vm.ci.meta.ResolvedJavaMethod; +import org.graalvm.compiler.core.test.GraalCompilerTest; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.extended.ForeignCallNode; +import org.graalvm.compiler.nodes.java.MethodCallTargetNode; +import org.graalvm.compiler.nodes.memory.ReadNode; +import org.junit.Assume; +import org.junit.Test; + +public class ObjectHashCodeInliningTest extends GraalCompilerTest { + + public static int getHash(Object obj) { + return obj.hashCode(); + } + + @Test + public void testInstallCodeInvalidation() { + for (int i = 0; i < 100000; i++) { + getHash(i % 1000 == 0 ? new Object() : ""); + } + + ResolvedJavaMethod method = getResolvedJavaMethod("getHash"); + StructuredGraph graph = parseForCompile(method); + for (MethodCallTargetNode callTargetNode : graph.getNodes(MethodCallTargetNode.TYPE)) { + if ("Object.hashCode".equals(callTargetNode.targetName())) { + Assume.assumeTrue(callTargetNode.getProfile() != null); + } + } + compile(method, graph); + } + + private static boolean containsForeignCallToIdentityHashCode(StructuredGraph graph) { + for (ForeignCallNode foreignCallNode : graph.getNodes().filter(ForeignCallNode.class)) { + if ("identity_hashcode".equals(foreignCallNode.getDescriptor().getName())) { + return true; + } + } + return false; + } + + private static boolean containsReadStringHash(StructuredGraph graph) { + for (ReadNode readNode : graph.getNodes().filter(ReadNode.class)) { + if ("String.hash".equals(readNode.getLocationIdentity().toString())) { + return true; + } + } + return false; + } + + @Override + protected boolean checkHighTierGraph(StructuredGraph graph) { + return containsForeignCallToIdentityHashCode(graph) && containsReadStringHash(graph); + } + +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/BootstrapWatchDog.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/BootstrapWatchDog.java index 28d973abd6a..0978795b400 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/BootstrapWatchDog.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/BootstrapWatchDog.java @@ -178,6 +178,10 @@ final class BootstrapWatchDog extends Thread { StackTraceElement[] stackTraceNow = t.getStackTrace(); TTY.printf("Printing stack trace for current compilation of %s lasting more than %d seconds:%n%s", fmt(request1.getMethod()), EPOCH, fmt(stackTraceNow)); + + // Fortify: Null Dereference false positive + assert stacksAtTimeout != null; + if (Arrays.equals(stacksAtTimeout.get(t), stackTraceNow)) { TTY.printf("\t** Identical stack trace %d seconds ago, implying a hung compilation **%n", EPOCH); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/CompilationTask.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/CompilationTask.java index 8e89cfe2c16..f740210fb2d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/CompilationTask.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/CompilationTask.java @@ -40,7 +40,6 @@ import org.graalvm.compiler.code.CompilationResult; import org.graalvm.compiler.core.CompilationPrinter; import org.graalvm.compiler.core.CompilationWrapper; import org.graalvm.compiler.core.common.CompilationIdentifier; -import org.graalvm.compiler.debug.Assertions; import org.graalvm.compiler.debug.CounterKey; import org.graalvm.compiler.debug.DebugCloseable; import org.graalvm.compiler.debug.DebugContext; @@ -93,7 +92,6 @@ public class CompilationTask { private final boolean useProfilingInfo; private final boolean shouldRetainLocalVariables; - private final OptionValues options; final class HotSpotCompilationWrapper extends CompilationWrapper { private final EventProvider.CompilationEvent compilationEvent; @@ -150,9 +148,8 @@ public class CompilationTask { if (bailout.isPermanent()) { // Respect current action if it has been explicitly set. if (!CompilationBailoutAsFailure.hasBeenSet(values)) { - // Get more info for permanent bailouts during bootstrap - // or when assertions are enabled. - if (Assertions.assertionsEnabled() || compiler.getGraalRuntime().isBootstrapping()) { + // Get more info for permanent bailouts during bootstrap. + if (compiler.getGraalRuntime().isBootstrapping()) { return Diagnose; } @@ -165,9 +162,8 @@ public class CompilationTask { // Respect current action if it has been explicitly set. if (!CompilationFailureAction.hasBeenSet(values)) { - // Automatically exit on failure during bootstrap - // or when assertions are enabled. - if (Assertions.assertionsEnabled() || compiler.getGraalRuntime().isBootstrapping()) { + // Automatically exit on failure during bootstrap. + if (compiler.getGraalRuntime().isBootstrapping()) { return ExitVM; } } @@ -180,14 +176,14 @@ public class CompilationTask { HotSpotResolvedJavaMethod method = getMethod(); int entryBCI = getEntryBCI(); final boolean isOSR = entryBCI != JVMCICompiler.INVOCATION_ENTRY_BCI; - CompilationStatistics stats = CompilationStatistics.create(options, method, isOSR); + CompilationStatistics stats = CompilationStatistics.create(debug.getOptions(), method, isOSR); - final CompilationPrinter printer = CompilationPrinter.begin(options, compilationId, method, entryBCI); + final CompilationPrinter printer = CompilationPrinter.begin(debug.getOptions(), compilationId, method, entryBCI); try (DebugContext.Scope s = debug.scope("Compiling", new DebugDumpScope(getIdString(), true))) { // Begin the compilation event. compilationEvent.begin(); - result = compiler.compile(method, entryBCI, useProfilingInfo, shouldRetainLocalVariables, compilationId, options, debug); + result = compiler.compile(method, entryBCI, useProfilingInfo, shouldRetainLocalVariables, compilationId, debug); } catch (Throwable e) { throw debug.handle(e); } finally { @@ -211,20 +207,21 @@ public class CompilationTask { } - public CompilationTask(HotSpotJVMCIRuntime jvmciRuntime, HotSpotGraalCompiler compiler, HotSpotCompilationRequest request, boolean useProfilingInfo, boolean installAsDefault, - OptionValues options) { - this(jvmciRuntime, compiler, request, useProfilingInfo, false, installAsDefault, options); + public CompilationTask(HotSpotJVMCIRuntime jvmciRuntime, HotSpotGraalCompiler compiler, HotSpotCompilationRequest request, boolean useProfilingInfo, boolean installAsDefault) { + this(jvmciRuntime, compiler, request, useProfilingInfo, false, installAsDefault); } public CompilationTask(HotSpotJVMCIRuntime jvmciRuntime, HotSpotGraalCompiler compiler, HotSpotCompilationRequest request, boolean useProfilingInfo, boolean shouldRetainLocalVariables, - boolean installAsDefault, OptionValues options) { + boolean installAsDefault) { this.jvmciRuntime = jvmciRuntime; this.compiler = compiler; this.compilationId = new HotSpotCompilationIdentifier(request); this.useProfilingInfo = useProfilingInfo; this.shouldRetainLocalVariables = shouldRetainLocalVariables; this.installAsDefault = installAsDefault; + } + public OptionValues filterOptions(OptionValues options) { /* * Disable inlining if HotSpot has it disabled unless it's been explicitly set in Graal. */ @@ -243,7 +240,7 @@ public class CompilationTask { newOptions = new OptionValues(options, m); } } - this.options = newOptions; + return newOptions; } public HotSpotResolvedJavaMethod getMethod() { @@ -309,7 +306,8 @@ public class CompilationTask { */ public static final TimerKey CodeInstallationTime = DebugContext.timer("CodeInstallation"); - public HotSpotCompilationRequestResult runCompilation() { + public HotSpotCompilationRequestResult runCompilation(OptionValues initialOptions) { + OptionValues options = filterOptions(initialOptions); SnippetReflectionProvider snippetReflection = compiler.getGraalRuntime().getHostProviders().getSnippetReflection(); try (DebugContext debug = DebugContext.create(options, new GraalDebugHandlersFactory(snippetReflection))) { return runCompilation(debug); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/CompilerConfigurationFactory.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/CompilerConfigurationFactory.java index 94e448713b7..9ab8eb56b73 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/CompilerConfigurationFactory.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/CompilerConfigurationFactory.java @@ -219,6 +219,8 @@ public abstract class CompilerConfigurationFactory implements Comparable= source.getStartOffset(); + assert !siteListIterator.hasNext() || (site != null && site.pcOffset >= source.getStartOffset()); if (site != null && source.getStartOffset() <= site.pcOffset && site.pcOffset <= source.getEndOffset()) { // Conflicting source mapping, skip it. continue; } else { // Since the sites are sorted there can not be any more sites in this interval. } - assert !siteListIterator.hasNext() || site.pcOffset > source.getEndOffset(); + assert !siteListIterator.hasNext() || (site != null && site.pcOffset > source.getEndOffset()); // Good source mapping. Create an infopoint and add it to the list. NodeSourcePosition sourcePosition = source.getSourcePosition(); assert sourcePosition.verify(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotForeignCallLinkage.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotForeignCallLinkage.java index cc22f311444..c3f1ce2dfc0 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotForeignCallLinkage.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotForeignCallLinkage.java @@ -53,11 +53,14 @@ public interface HotSpotForeignCallLinkage extends ForeignCallLinkage, InvokeTar */ enum Transition { /** - * A call to a leaf function that is guaranteed to not use floating point registers and will - * never have its caller stack inspected by the VM. That is, {@code JavaFrameAnchor} - * management around the call can be omitted. + * A call to a leaf function that is guaranteed to not use floating point registers. + * Consequently, floating point registers cleanup will be waived. On AMD64, this means the + * compiler will no longer emit vzeroupper instruction around the foreign call, which it + * normally does for unknown foreign calls to avoid potential SSE-AVX transition penalty. + * Besides, this foreign call will never have its caller stack inspected by the VM. That is, + * {@code JavaFrameAnchor} management around the call can be omitted. */ - LEAF_NOFP, + LEAF_NO_VZERO, /** * A call to a leaf function that might use floating point registers but will never have its diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotForeignCallLinkageImpl.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotForeignCallLinkageImpl.java index 1e8b2fe4a44..a5211018798 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotForeignCallLinkageImpl.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotForeignCallLinkageImpl.java @@ -283,7 +283,7 @@ public class HotSpotForeignCallLinkageImpl extends HotSpotForeignCallTarget impl @Override public boolean mayContainFP() { - return transition != Transition.LEAF_NOFP; + return transition != Transition.LEAF_NO_VZERO; } @Override diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalCompiler.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalCompiler.java index 922e9950221..4c9bcd73385 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalCompiler.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalCompiler.java @@ -25,7 +25,6 @@ package org.graalvm.compiler.hotspot; import static org.graalvm.compiler.core.common.GraalOptions.OptAssumptions; -import org.graalvm.compiler.serviceprovider.GraalUnsafeAccess; import java.io.ByteArrayOutputStream; import java.io.PrintStream; @@ -59,6 +58,7 @@ import org.graalvm.compiler.phases.PhaseSuite; import org.graalvm.compiler.phases.tiers.HighTierContext; import org.graalvm.compiler.phases.tiers.Suites; import org.graalvm.compiler.printer.GraalDebugHandlersFactory; +import org.graalvm.compiler.serviceprovider.GraalUnsafeAccess; import jdk.vm.ci.code.CompilationRequest; import jdk.vm.ci.code.CompilationRequestResult; @@ -109,7 +109,7 @@ public class HotSpotGraalCompiler implements GraalJVMCICompiler { } @SuppressWarnings("try") - CompilationRequestResult compileMethod(CompilationRequest request, boolean installAsDefault, OptionValues options) { + CompilationRequestResult compileMethod(CompilationRequest request, boolean installAsDefault, OptionValues initialOptions) { if (graalRuntime.isShutdown()) { return HotSpotCompilationRequestResult.failure(String.format("Shutdown entered"), false); } @@ -117,7 +117,7 @@ public class HotSpotGraalCompiler implements GraalJVMCICompiler { ResolvedJavaMethod method = request.getMethod(); if (graalRuntime.isBootstrapping()) { - if (DebugOptions.BootstrapInitializeOnly.getValue(options)) { + if (DebugOptions.BootstrapInitializeOnly.getValue(initialOptions)) { return HotSpotCompilationRequestResult.failure(String.format("Skip compilation because %s is enabled", DebugOptions.BootstrapInitializeOnly.getName()), true); } if (bootstrapWatchDog != null) { @@ -128,13 +128,14 @@ public class HotSpotGraalCompiler implements GraalJVMCICompiler { } } HotSpotCompilationRequest hsRequest = (HotSpotCompilationRequest) request; + CompilationTask task = new CompilationTask(jvmciRuntime, this, hsRequest, true, shouldRetainLocalVariables(hsRequest.getJvmciEnv()), installAsDefault); + OptionValues options = task.filterOptions(initialOptions); try (CompilationWatchDog w1 = CompilationWatchDog.watch(method, hsRequest.getId(), options); BootstrapWatchDog.Watch w2 = bootstrapWatchDog == null ? null : bootstrapWatchDog.watch(request); CompilationAlarm alarm = CompilationAlarm.trackCompilationPeriod(options);) { if (compilationCounters != null) { compilationCounters.countCompilation(method); } - CompilationTask task = new CompilationTask(jvmciRuntime, this, hsRequest, true, shouldRetainLocalVariables(hsRequest.getJvmciEnv()), installAsDefault, options); CompilationRequestResult r = null; try (DebugContext debug = graalRuntime.openDebugContext(options, task.getCompilationIdentifier(), method, getDebugHandlersFactories(), DebugContext.DEFAULT_LOG_STREAM); Activation a = debug.activate()) { @@ -187,7 +188,7 @@ public class HotSpotGraalCompiler implements GraalJVMCICompiler { public CompilationResult compileHelper(CompilationResultBuilderFactory crbf, CompilationResult result, StructuredGraph graph, ResolvedJavaMethod method, int entryBCI, boolean useProfilingInfo, boolean shouldRetainLocalVariables, OptionValues options) { - + assert options == graph.getOptions(); HotSpotBackend backend = graalRuntime.getHostBackend(); HotSpotProviders providers = backend.getProviders(); final boolean isOSR = entryBCI != JVMCICompiler.INVOCATION_ENTRY_BCI; @@ -218,15 +219,11 @@ public class HotSpotGraalCompiler implements GraalJVMCICompiler { return result; } - public CompilationResult compile(ResolvedJavaMethod method, int entryBCI, boolean useProfilingInfo, CompilationIdentifier compilationId, OptionValues options, DebugContext debug) { - return compile(method, entryBCI, useProfilingInfo, false, compilationId, options, debug); - } - - public CompilationResult compile(ResolvedJavaMethod method, int entryBCI, boolean useProfilingInfo, boolean shouldRetainLocalVariables, CompilationIdentifier compilationId, OptionValues options, + public CompilationResult compile(ResolvedJavaMethod method, int entryBCI, boolean useProfilingInfo, boolean shouldRetainLocalVariables, CompilationIdentifier compilationId, DebugContext debug) { - StructuredGraph graph = createGraph(method, entryBCI, useProfilingInfo, compilationId, options, debug); + StructuredGraph graph = createGraph(method, entryBCI, useProfilingInfo, compilationId, debug.getOptions(), debug); CompilationResult result = new CompilationResult(compilationId); - return compileHelper(CompilationResultBuilderFactory.Default, result, graph, method, entryBCI, useProfilingInfo, shouldRetainLocalVariables, options); + return compileHelper(CompilationResultBuilderFactory.Default, result, graph, method, entryBCI, useProfilingInfo, shouldRetainLocalVariables, debug.getOptions()); } protected OptimisticOptimizations getOptimisticOpts(ProfilingInfo profilingInfo, OptionValues options) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotReplacementsImpl.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotReplacementsImpl.java index 8f5a70d0149..692ed4f0742 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotReplacementsImpl.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotReplacementsImpl.java @@ -26,7 +26,9 @@ package org.graalvm.compiler.hotspot; import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; -import static org.graalvm.compiler.replacements.ReplacementsImpl.Options.UseEncodedSnippets; +import static org.graalvm.compiler.core.common.GraalOptions.UseEncodedGraphs; +import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.INLINE_AFTER_PARSING; +import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.ROOT_COMPILATION; import java.util.Set; @@ -43,6 +45,7 @@ import org.graalvm.compiler.nodes.Invoke; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderPlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext; import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin; import org.graalvm.compiler.nodes.graphbuilderconf.MethodSubstitutionPlugin; import org.graalvm.compiler.options.OptionValues; @@ -59,12 +62,12 @@ import jdk.vm.ci.meta.ResolvedJavaMethod; * them. */ public class HotSpotReplacementsImpl extends ReplacementsImpl { - public HotSpotReplacementsImpl(OptionValues options, Providers providers, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider, TargetDescription target) { - super(options, new GraalDebugHandlersFactory(snippetReflection), providers, snippetReflection, bytecodeProvider, target); + public HotSpotReplacementsImpl(Providers providers, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider, TargetDescription target) { + super(new GraalDebugHandlersFactory(snippetReflection), providers, snippetReflection, bytecodeProvider, target); } - protected HotSpotReplacementsImpl(HotSpotReplacementsImpl replacements, Providers providers) { - super(replacements.options, new GraalDebugHandlersFactory(replacements.snippetReflection), providers, replacements.snippetReflection, + HotSpotReplacementsImpl(HotSpotReplacementsImpl replacements, Providers providers) { + super(new GraalDebugHandlersFactory(replacements.snippetReflection), providers, replacements.snippetReflection, replacements.getDefaultReplacementBytecodeProvider(), replacements.target); } @@ -73,14 +76,15 @@ public class HotSpotReplacementsImpl extends ReplacementsImpl { return method.getAnnotation(HotSpotOperation.class) != null ? HotSpotWordOperationPlugin.class : super.getIntrinsifyingPlugin(method); } - public void registerMethodSubstitution(ResolvedJavaMethod method, ResolvedJavaMethod original) { + @Override + public void registerMethodSubstitution(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original, IntrinsicContext.CompilationContext context, OptionValues options) { if (!IS_IN_NATIVE_IMAGE) { - if (IS_BUILDING_NATIVE_IMAGE || UseEncodedSnippets.getValue(options)) { + if (IS_BUILDING_NATIVE_IMAGE || UseEncodedGraphs.getValue(options)) { synchronized (HotSpotReplacementsImpl.class) { if (snippetEncoder == null) { snippetEncoder = new SymbolicSnippetEncoder(this); } - snippetEncoder.registerMethodSubstitution(method, original); + snippetEncoder.registerMethodSubstitution(plugin, original, context, options); } } } @@ -88,18 +92,44 @@ public class HotSpotReplacementsImpl extends ReplacementsImpl { @Override public StructuredGraph getIntrinsicGraph(ResolvedJavaMethod method, CompilationIdentifier compilationId, DebugContext debug) { - if (IS_IN_NATIVE_IMAGE) { + boolean useEncodedGraphs = UseEncodedGraphs.getValue(debug.getOptions()); + if (IS_IN_NATIVE_IMAGE || useEncodedGraphs) { HotSpotReplacementsImpl replacements = (HotSpotReplacementsImpl) providers.getReplacements(); InvocationPlugin plugin = replacements.getGraphBuilderPlugins().getInvocationPlugins().lookupInvocation(method); if (plugin instanceof MethodSubstitutionPlugin) { MethodSubstitutionPlugin msp = (MethodSubstitutionPlugin) plugin; - return replacements.getMethodSubstitution(msp, method); + if (useEncodedGraphs) { + replacements.registerMethodSubstitution(msp, method, ROOT_COMPILATION, debug.getOptions()); + } + StructuredGraph methodSubstitution = replacements.getMethodSubstitution(msp, method, ROOT_COMPILATION, StructuredGraph.AllowAssumptions.YES, debug.getOptions()); + methodSubstitution.resetDebug(debug); + return methodSubstitution; } return null; } return super.getIntrinsicGraph(method, compilationId, debug); } + @Override + public StructuredGraph getSubstitution(ResolvedJavaMethod targetMethod, int invokeBci, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition, OptionValues options) { + boolean useEncodedGraphs = UseEncodedGraphs.getValue(options); + if (IS_IN_NATIVE_IMAGE || useEncodedGraphs) { + InvocationPlugin plugin = getGraphBuilderPlugins().getInvocationPlugins().lookupInvocation(targetMethod); + if (plugin instanceof MethodSubstitutionPlugin && (!plugin.inlineOnly() || invokeBci >= 0)) { + MethodSubstitutionPlugin msPlugin = (MethodSubstitutionPlugin) plugin; + if (!IS_IN_NATIVE_IMAGE && useEncodedGraphs) { + registerMethodSubstitution(msPlugin, targetMethod, INLINE_AFTER_PARSING, options); + } + // This assumes the normal path creates the graph using + // GraphBuilderConfiguration.getSnippetDefault with omits exception edges + StructuredGraph subst = getMethodSubstitution(msPlugin, targetMethod, INLINE_AFTER_PARSING, StructuredGraph.AllowAssumptions.NO, options); + return subst; + } + } + + return super.getSubstitution(targetMethod, invokeBci, trackNodeSourcePosition, replaceePosition, options); + } + @Override public void notifyNotInlined(GraphBuilderContext b, ResolvedJavaMethod method, Invoke invoke) { if (b.parsingIntrinsic() && snippetEncoder != null) { @@ -118,16 +148,16 @@ public class HotSpotReplacementsImpl extends ReplacementsImpl { private boolean snippetRegistrationClosed; @Override - public void registerSnippet(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean trackNodeSourcePosition) { + public void registerSnippet(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean trackNodeSourcePosition, OptionValues options) { if (!IS_IN_NATIVE_IMAGE) { assert !snippetRegistrationClosed : "Cannot register snippet after registration is closed: " + method.format("%H.%n(%p)"); assert registeredSnippets.add(method) : "Cannot register snippet twice: " + method.format("%H.%n(%p)"); - if (IS_BUILDING_NATIVE_IMAGE || UseEncodedSnippets.getValue(options)) { + if (IS_BUILDING_NATIVE_IMAGE || UseEncodedGraphs.getValue(options)) { synchronized (HotSpotReplacementsImpl.class) { if (snippetEncoder == null) { snippetEncoder = new SymbolicSnippetEncoder(this); } - snippetEncoder.registerSnippet(method, original, receiver, trackNodeSourcePosition); + snippetEncoder.registerSnippet(method, original, receiver, trackNodeSourcePosition, options); } } } @@ -138,7 +168,7 @@ public class HotSpotReplacementsImpl extends ReplacementsImpl { snippetRegistrationClosed = true; } - static SymbolicSnippetEncoder.EncodedSnippets getEncodedSnippets() { + private static SymbolicSnippetEncoder.EncodedSnippets getEncodedSnippets() { return encodedSnippets; } @@ -153,57 +183,66 @@ public class HotSpotReplacementsImpl extends ReplacementsImpl { HotSpotReplacementsImpl.encodedSnippets = encodedSnippets; } - public boolean encode() { + public boolean encode(OptionValues options) { SymbolicSnippetEncoder encoder = HotSpotReplacementsImpl.snippetEncoder; if (encoder != null) { - return encoder.encode(); + return encoder.encode(options); } return false; } private static volatile SymbolicSnippetEncoder.EncodedSnippets encodedSnippets; - @NativeImageReinitialize static SymbolicSnippetEncoder snippetEncoder; + @NativeImageReinitialize private static SymbolicSnippetEncoder snippetEncoder; @Override - public StructuredGraph getSnippet(ResolvedJavaMethod method, ResolvedJavaMethod recursiveEntry, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) { - StructuredGraph graph = getEncodedSnippet(method, args); + public StructuredGraph getSnippet(ResolvedJavaMethod method, ResolvedJavaMethod recursiveEntry, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition, + OptionValues options) { + StructuredGraph graph = getEncodedSnippet(method, args, StructuredGraph.AllowAssumptions.NO, options); if (graph != null) { return graph; } assert !IS_IN_NATIVE_IMAGE : "should be using encoded snippets"; - return super.getSnippet(method, recursiveEntry, args, trackNodeSourcePosition, replaceePosition); + return super.getSnippet(method, recursiveEntry, args, trackNodeSourcePosition, replaceePosition, options); } - public StructuredGraph getEncodedSnippet(ResolvedJavaMethod method, Object[] args) { - if (IS_IN_NATIVE_IMAGE || UseEncodedSnippets.getValue(options)) { + private StructuredGraph getEncodedSnippet(ResolvedJavaMethod method, Object[] args, StructuredGraph.AllowAssumptions allowAssumptions, OptionValues options) { + boolean useEncodedGraphs = UseEncodedGraphs.getValue(options); + if (IS_IN_NATIVE_IMAGE || useEncodedGraphs) { synchronized (HotSpotReplacementsImpl.class) { - if (!IS_IN_NATIVE_IMAGE && UseEncodedSnippets.getValue(options)) { - snippetEncoder.encode(); + if (!IS_IN_NATIVE_IMAGE) { + snippetEncoder.encode(options); } if (getEncodedSnippets() == null) { throw GraalError.shouldNotReachHere("encoded snippets not found"); } - StructuredGraph graph = getEncodedSnippets().getEncodedSnippet(method, this, args); + StructuredGraph graph = getEncodedSnippets().getEncodedSnippet(method, this, args, allowAssumptions, options); if (graph == null) { throw GraalError.shouldNotReachHere("snippet not found: " + method.format("%H.%n(%p)")); } return graph; } - } else if (registeredSnippets != null) { - assert registeredSnippets.contains(method) : "Asking for snippet method that was never registered: " + method.format("%H.%n(%p)"); + } else { + assert registeredSnippets == null || registeredSnippets.contains(method) : "Asking for snippet method that was never registered: " + method.format("%H.%n(%p)"); } return null; } - public StructuredGraph getMethodSubstitution(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original) { - if (IS_IN_NATIVE_IMAGE || UseEncodedSnippets.getValue(options)) { + @Override + public StructuredGraph getMethodSubstitution(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original, IntrinsicContext.CompilationContext context, + StructuredGraph.AllowAssumptions allowAssumptions, OptionValues options) { + boolean useEncodedGraphs = UseEncodedGraphs.getValue(options); + if (IS_IN_NATIVE_IMAGE || useEncodedGraphs) { + if (!IS_IN_NATIVE_IMAGE) { + snippetEncoder.encode(options); + } + if (getEncodedSnippets() == null) { throw GraalError.shouldNotReachHere("encoded snippets not found"); } - return getEncodedSnippets().getMethodSubstitutionGraph(plugin, original, this); + return getEncodedSnippets().getMethodSubstitutionGraph(plugin, original, this, context, allowAssumptions, options); } return null; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/IsGraalPredicate.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/IsGraalPredicate.java index 8c688464d94..a0bab444081 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/IsGraalPredicate.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/IsGraalPredicate.java @@ -63,4 +63,9 @@ class IsGraalPredicate extends IsGraalPredicateBase { Module module = declaringClass.getModule(); return jvmciModule == module || graalModule == module || compilerConfigurationModule == module; } + + @Override + HotSpotJVMCICompilerFactory.CompilationLevelAdjustment getCompilationLevelAdjustment() { + return HotSpotJVMCICompilerFactory.CompilationLevelAdjustment.ByHolder; + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/SymbolicSnippetEncoder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/SymbolicSnippetEncoder.java index 8b5c9baff35..f65d37fa43d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/SymbolicSnippetEncoder.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/SymbolicSnippetEncoder.java @@ -27,8 +27,9 @@ package org.graalvm.compiler.hotspot; import static jdk.vm.ci.runtime.JVMCI.getRuntime; import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; +import static org.graalvm.compiler.core.common.GraalOptions.UseEncodedGraphs; import static org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin.InlineInfo.createIntrinsicInlineInfo; -import static org.graalvm.compiler.replacements.ReplacementsImpl.Options.UseEncodedSnippets; +import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.INLINE_AFTER_PARSING; import java.util.ArrayList; import java.util.Arrays; @@ -48,6 +49,7 @@ import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; import org.graalvm.compiler.api.runtime.GraalJVMCICompiler; import org.graalvm.compiler.api.runtime.GraalRuntime; import org.graalvm.compiler.bytecode.BytecodeProvider; +import org.graalvm.compiler.bytecode.ResolvedJavaMethodBytecode; import org.graalvm.compiler.core.common.type.AbstractObjectStamp; import org.graalvm.compiler.core.common.type.Stamp; import org.graalvm.compiler.core.common.type.StampPair; @@ -86,7 +88,6 @@ import org.graalvm.compiler.nodes.graphbuilderconf.NodePlugin; import org.graalvm.compiler.nodes.graphbuilderconf.ParameterPlugin; import org.graalvm.compiler.nodes.java.AccessFieldNode; import org.graalvm.compiler.nodes.java.MethodCallTargetNode; -import org.graalvm.compiler.nodes.spi.DelegatingReplacements; import org.graalvm.compiler.nodes.virtual.VirtualObjectNode; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.OptimisticOptimizations; @@ -98,12 +99,12 @@ import org.graalvm.compiler.replacements.ReplacementsImpl; import org.graalvm.compiler.replacements.SnippetCounter; import org.graalvm.compiler.replacements.SnippetIntegerHistogram; -import jdk.vm.ci.code.Architecture; import jdk.vm.ci.code.TargetDescription; import jdk.vm.ci.hotspot.HotSpotJVMCIRuntime; import jdk.vm.ci.hotspot.HotSpotResolvedJavaField; import jdk.vm.ci.hotspot.HotSpotResolvedJavaMethod; import jdk.vm.ci.hotspot.HotSpotResolvedJavaType; +import jdk.vm.ci.hotspot.HotSpotSignature; import jdk.vm.ci.meta.Constant; import jdk.vm.ci.meta.ConstantReflectionProvider; import jdk.vm.ci.meta.JavaConstant; @@ -123,13 +124,13 @@ import jdk.vm.ci.meta.UnresolvedJavaType; * method references into a symbolic form that can be resolved at graph decode time using * {@link SymbolicJVMCIReference}. */ -public class SymbolicSnippetEncoder extends DelegatingReplacements { +public class SymbolicSnippetEncoder { /** * This is a customized HotSpotReplacementsImpl intended only for parsing snippets and method * substitutions for graph encoding. */ - private final HotSpotSnippetReplacementsImpl replacements; + private final HotSpotSnippetReplacementsImpl snippetReplacements; /** * The set of all snippet methods that have been encoded. @@ -142,11 +143,13 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { */ private final Map originalMethods = new ConcurrentHashMap<>(); + private final HotSpotReplacementsImpl originalReplacements; + /** * The current count of graphs encoded. Used to detect when new graphs have been enqueued for * encoding. */ - int encodedGraphs = 0; + private int encodedGraphs = 0; /** * All the graphs parsed so far. @@ -175,13 +178,13 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { return InlineInfo.DO_NOT_INLINE_NO_EXCEPTION; } - if (getIntrinsifyingPlugin(method) != null) { + if (snippetReplacements.getIntrinsifyingPlugin(method) != null) { delayedInvocationPluginMethods.add(method); return InlineInfo.DO_NOT_INLINE_NO_EXCEPTION; } // Force inlining when parsing replacements - return createIntrinsicInlineInfo(method, null, getDefaultReplacementBytecodeProvider()); + return createIntrinsicInlineInfo(method, snippetReplacements.getDefaultReplacementBytecodeProvider()); } @Override @@ -219,12 +222,12 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { return true; } if (field.getType().getName().equals(snippetCounterName)) { - b.addPush(JavaKind.Object, ConstantNode.forConstant(replacements.snippetReflection.forObject(SnippetCounter.DISABLED_COUNTER), b.getMetaAccess())); + b.addPush(JavaKind.Object, ConstantNode.forConstant(snippetReplacements.snippetReflection.forObject(SnippetCounter.DISABLED_COUNTER), b.getMetaAccess())); return true; } if (field.getType().getName().equals(snippetIntegerHistogramName)) { - b.addPush(JavaKind.Object, ConstantNode.forConstant(replacements.snippetReflection.forObject(SnippetIntegerHistogram.DISABLED_COUNTER), b.getMetaAccess())); + b.addPush(JavaKind.Object, ConstantNode.forConstant(snippetReplacements.snippetReflection.forObject(SnippetIntegerHistogram.DISABLED_COUNTER), b.getMetaAccess())); return true; } return false; @@ -240,8 +243,7 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { } SymbolicSnippetEncoder(HotSpotReplacementsImpl replacements) { - super(replacements); - + this.originalReplacements = replacements; GraphBuilderConfiguration.Plugins plugins = replacements.getGraphBuilderPlugins(); SnippetInvocationPlugins invocationPlugins = new SnippetInvocationPlugins(plugins.getInvocationPlugins()); GraphBuilderConfiguration.Plugins copy = new GraphBuilderConfiguration.Plugins(plugins, invocationPlugins); @@ -249,22 +251,20 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { copy.appendInlineInvokePlugin(new SnippetInlineInvokePlugin()); copy.appendNodePlugin(new SnippetCounterPlugin()); HotSpotProviders providers = (HotSpotProviders) replacements.getProviders().copyWith(new HotSpotSubstrateConstantReflectionProvider(replacements.getProviders().getConstantReflection())); - this.replacements = new HotSpotSnippetReplacementsImpl(replacements, providers.copyWith(copy)); - this.replacements.setGraphBuilderPlugins(copy); - } - - @Override - public GraphBuilderConfiguration.Plugins getGraphBuilderPlugins() { - return replacements.getGraphBuilderPlugins(); + this.snippetReplacements = new HotSpotSnippetReplacementsImpl(replacements, providers.copyWith(copy)); + this.snippetReplacements.setGraphBuilderPlugins(copy); } /** * Compiles the snippet and stores the graph. */ - public void registerMethodSubstitution(ResolvedJavaMethod method, ResolvedJavaMethod original) { + synchronized void registerMethodSubstitution(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original, IntrinsicContext.CompilationContext context, OptionValues options) { + ResolvedJavaMethod method = plugin.getSubstitute(snippetReplacements.getProviders().getMetaAccess()); assert method.getAnnotation(MethodSubstitution.class) != null : "MethodSubstitution must be annotated with @" + MethodSubstitution.class.getSimpleName(); - buildGraph(method, original, null, false, false); + StructuredGraph subst = buildGraph(method, original, null, true, false, context, options); snippetMethods.add(method); + originalMethods.put(methodKey(method), methodKey(original)); + preparedSnippetGraphs.put(plugin.toString() + context, subst); } static class EncodedSnippets { @@ -282,54 +282,26 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { this.originalMethods = originalMethods; } - public StructuredGraph getMethodSubstitutionGraph(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original, ReplacementsImpl replacements) { - Integer startOffset = snippetStartOffsets.get(plugin.toString()); + StructuredGraph getMethodSubstitutionGraph(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original, ReplacementsImpl replacements, IntrinsicContext.CompilationContext context, + StructuredGraph.AllowAssumptions allowAssumptions, OptionValues options) { + Integer startOffset = snippetStartOffsets.get(plugin.toString() + context); if (startOffset == null) { - throw GraalError.shouldNotReachHere("plugin graph not found: " + plugin); + throw GraalError.shouldNotReachHere("plugin graph not found: " + plugin + " with " + context); } - return decodeGraph(original, null, startOffset, replacements); + ResolvedJavaType accessingClass = replacements.getProviders().getMetaAccess().lookupJavaType(plugin.getDeclaringClass()); + return decodeGraph(original, accessingClass, startOffset, replacements, context, allowAssumptions, options); } @SuppressWarnings("try") - private StructuredGraph decodeGraph(ResolvedJavaMethod method, Object[] args, int startOffset, ReplacementsImpl replacements) { - OptionValues options = replacements.getOptions(); - SnippetReflectionProvider snippetReflection = replacements.snippetReflection; - ParameterPlugin parameterPlugin = null; + private StructuredGraph decodeGraph(ResolvedJavaMethod method, ResolvedJavaType accessingClass, int startOffset, ReplacementsImpl replacements, + IntrinsicContext.CompilationContext context, StructuredGraph.AllowAssumptions allowAssumptions, OptionValues options) { Providers providers = replacements.getProviders(); - if (args != null) { - parameterPlugin = new ConstantBindingParameterPlugin(args, providers.getMetaAccess(), snippetReflection); - } - - EncodedGraph encodedGraph = new SymbolicEncodedGraph(snippetEncoding, startOffset, snippetObjects, snippetNodeClasses, method.getDeclaringClass(), - originalMethods.get(methodKey(method))); - try (DebugContext debug = replacements.openDebugContext("SVMSnippet_", method)) { - StructuredGraph result = new StructuredGraph.Builder(options, debug).method(method).setIsSubstitution(true).build(); - PEGraphDecoder graphDecoder = new PEGraphDecoder( - providers.getCodeCache().getTarget().arch, - result, - providers, - null, // loopExplosionPlugin - replacements.getGraphBuilderPlugins().getInvocationPlugins(), - new InlineInvokePlugin[0], - parameterPlugin, - null, // nodePlugins - null, // callInlinedMethod - null // sourceLanguagePositionProvider - ) { - @Override - protected EncodedGraph lookupEncodedGraph(ResolvedJavaMethod lookupMethod, - ResolvedJavaMethod originalMethod, - BytecodeProvider intrinsicBytecodeProvider, - boolean isSubstitution, - boolean trackNodeSourcePosition) { - if (lookupMethod.equals(method)) { - return encodedGraph; - } else { - throw GraalError.shouldNotReachHere(method.format("%H.%n(%p)")); - } - } - }; + EncodedGraph encodedGraph = new SymbolicEncodedGraph(snippetEncoding, startOffset, snippetObjects, snippetNodeClasses, + methodKey(method), accessingClass, method.getDeclaringClass()); + try (DebugContext debug = replacements.openDebugContext("SVMSnippet_", method, options)) { + StructuredGraph result = new StructuredGraph.Builder(options, debug, allowAssumptions).method(method).setIsSubstitution(true).build(); + PEGraphDecoder graphDecoder = new SubstitutionGraphDecoder(providers, result, replacements, null, method, context, encodedGraph); graphDecoder.decode(method, result.isSubstitution(), encodedGraph.trackNodeSourcePosition()); @@ -338,7 +310,7 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { } } - StructuredGraph getEncodedSnippet(ResolvedJavaMethod method, ReplacementsImpl replacements, Object[] args) { + StructuredGraph getEncodedSnippet(ResolvedJavaMethod method, ReplacementsImpl replacements, Object[] args, StructuredGraph.AllowAssumptions allowAssumptions, OptionValues options) { Integer startOffset = null; if (snippetStartOffsets != null) { startOffset = snippetStartOffsets.get(methodKey(method)); @@ -351,22 +323,57 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { } } - SymbolicEncodedGraph encodedGraph = new SymbolicEncodedGraph(snippetEncoding, startOffset, snippetObjects, snippetNodeClasses, method.getDeclaringClass(), - originalMethods.get(methodKey(method))); - return decodeSnippetGraph(encodedGraph, method, replacements, args, HotSpotJVMCIRuntime.runtime().getHostJVMCIBackend().getTarget().arch); + SymbolicEncodedGraph encodedGraph = new SymbolicEncodedGraph(snippetEncoding, startOffset, snippetObjects, snippetNodeClasses, + originalMethods.get(methodKey(method)), method.getDeclaringClass()); + return decodeSnippetGraph(encodedGraph, method, replacements, args, allowAssumptions, options); } } - private StructuredGraph buildGraph(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean requireInlining, boolean trackNodeSourcePosition) { + private static class SubstitutionGraphDecoder extends PEGraphDecoder { + private final ResolvedJavaMethod method; + private final EncodedGraph encodedGraph; + private IntrinsicContext intrinsic; + + SubstitutionGraphDecoder(Providers providers, StructuredGraph result, ReplacementsImpl replacements, ParameterPlugin parameterPlugin, ResolvedJavaMethod method, + IntrinsicContext.CompilationContext context, EncodedGraph encodedGraph) { + super(providers.getCodeCache().getTarget().arch, result, providers, null, + replacements.getGraphBuilderPlugins().getInvocationPlugins(), new InlineInvokePlugin[0], parameterPlugin, + null, null, null); + this.method = method; + this.encodedGraph = encodedGraph; + intrinsic = new IntrinsicContext(method, null, replacements.getDefaultReplacementBytecodeProvider(), context, false); + } + + @Override + protected EncodedGraph lookupEncodedGraph(ResolvedJavaMethod lookupMethod, + MethodSubstitutionPlugin plugin, + BytecodeProvider intrinsicBytecodeProvider, + boolean isSubstitution, + boolean trackNodeSourcePosition) { + if (lookupMethod.equals(method)) { + return encodedGraph; + } else { + throw GraalError.shouldNotReachHere(method.format("%H.%n(%p)")); + } + } + + @Override + protected IntrinsicContext getIntrinsic() { + return intrinsic; + } + } + + private StructuredGraph buildGraph(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean requireInlining, boolean trackNodeSourcePosition, + IntrinsicContext.CompilationContext context, OptionValues options) { assert method.hasBytecodes() : "Snippet must not be abstract or native"; Object[] args = null; if (receiver != null) { args = new Object[method.getSignature().getParameterCount(true)]; args[0] = receiver; } - try (DebugContext debug = openDebugContext("Snippet_", method)) { - StructuredGraph graph = replacements.makeGraph(debug, replacements.getDefaultReplacementBytecodeProvider(), method, args, original, trackNodeSourcePosition, null); + try (DebugContext debug = openDebugContext("Snippet_", method, options)) { + StructuredGraph graph = snippetReplacements.makeGraph(debug, snippetReplacements.getDefaultReplacementBytecodeProvider(), method, args, original, trackNodeSourcePosition, null, context); // Check if all methods which should be inlined are really inlined. for (MethodCallTargetNode callTarget : graph.getNodes(MethodCallTargetNode.TYPE)) { @@ -382,48 +389,24 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { } @SuppressWarnings("try") - static StructuredGraph decodeSnippetGraph(SymbolicEncodedGraph encodedGraph, ResolvedJavaMethod method, ReplacementsImpl replacements, Object[] args, Architecture architecture) { + private static StructuredGraph decodeSnippetGraph(SymbolicEncodedGraph encodedGraph, ResolvedJavaMethod method, ReplacementsImpl replacements, Object[] args, + StructuredGraph.AllowAssumptions allowAssumptions, OptionValues options) { Providers providers = replacements.getProviders(); ParameterPlugin parameterPlugin = null; if (args != null) { parameterPlugin = new ConstantBindingParameterPlugin(args, providers.getMetaAccess(), replacements.snippetReflection); } - try (DebugContext debug = replacements.openDebugContext("SVMSnippet_", method)) { + try (DebugContext debug = replacements.openDebugContext("SVMSnippet_", method, options)) { // @formatter:off - StructuredGraph result = new StructuredGraph.Builder(replacements.getOptions(), debug) - .method(method) - .trackNodeSourcePosition(encodedGraph.trackNodeSourcePosition()) - .setIsSubstitution(true) - .build(); + StructuredGraph result = new StructuredGraph.Builder(options, debug, allowAssumptions) + .method(method) + .trackNodeSourcePosition(encodedGraph.trackNodeSourcePosition()) + .setIsSubstitution(true) + .build(); // @formatter:on try (DebugContext.Scope scope = debug.scope("DecodeSnippetGraph", result)) { - PEGraphDecoder graphDecoder = new PEGraphDecoder( - architecture, - result, - providers, - null, - replacements.getGraphBuilderPlugins().getInvocationPlugins(), - new InlineInvokePlugin[0], - parameterPlugin, - null, - null, - null) { - @Override - protected EncodedGraph lookupEncodedGraph( - ResolvedJavaMethod lookupMethod, - ResolvedJavaMethod originalMethod, - BytecodeProvider intrinsicBytecodeProvider, - boolean isSubstitution, - boolean track) { - if (lookupMethod.equals(method)) { - assert !track || encodedGraph.trackNodeSourcePosition(); - return encodedGraph; - } else { - throw GraalError.shouldNotReachHere(method.format("%H.%n(%p)")); - } - } - }; + PEGraphDecoder graphDecoder = new SubstitutionGraphDecoder(providers, result, replacements, parameterPlugin, method, INLINE_AFTER_PARSING, encodedGraph); graphDecoder.decode(method, result.isSubstitution(), encodedGraph.trackNodeSourcePosition()); debug.dump(DebugContext.VERBOSE_LEVEL, result, "After decoding"); @@ -437,14 +420,12 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { } @SuppressWarnings("try") - private boolean verifySnippetEncodeDecode(ResolvedJavaMethod method, ResolvedJavaMethod original, boolean trackNodeSourcePosition, StructuredGraph structuredGraph) { + private boolean verifySnippetEncodeDecode(ResolvedJavaMethod method, ResolvedJavaMethod original, boolean trackNodeSourcePosition, StructuredGraph graph) { // Verify the encoding and decoding process - EncodedGraph encodedGraph = GraphEncoder.encodeSingleGraph(structuredGraph, HotSpotJVMCIRuntime.runtime().getHostJVMCIBackend().getTarget().arch); + EncodedGraph encodedGraph = GraphEncoder.encodeSingleGraph(graph, HotSpotJVMCIRuntime.runtime().getHostJVMCIBackend().getTarget().arch); - Architecture arch = HotSpotJVMCIRuntime.runtime().getHostJVMCIBackend().getTarget().arch; - - try (DebugContext debug = replacements.openDebugContext("VerifySnippetEncodeDecode_", method)) { - HotSpotProviders originalProvider = (HotSpotProviders) replacements.getProviders(); + try (DebugContext debug = snippetReplacements.openDebugContext("VerifySnippetEncodeDecode_", method, graph.getOptions())) { + HotSpotProviders originalProvider = (HotSpotProviders) snippetReplacements.getProviders(); SnippetReflectionProvider snippetReflection = originalProvider.getSnippetReflection(); SymbolicSnippetEncoder.HotSpotSubstrateConstantReflectionProvider constantReflection = new SymbolicSnippetEncoder.HotSpotSubstrateConstantReflectionProvider( @@ -452,29 +433,29 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { HotSpotProviders newProviders = new HotSpotProviders(originalProvider.getMetaAccess(), originalProvider.getCodeCache(), constantReflection, originalProvider.getConstantFieldProvider(), originalProvider.getForeignCalls(), originalProvider.getLowerer(), null, originalProvider.getSuites(), originalProvider.getRegisters(), snippetReflection, originalProvider.getWordTypes(), originalProvider.getGraphBuilderPlugins()); - HotSpotSnippetReplacementsImpl filteringReplacements = new HotSpotSnippetReplacementsImpl(getOptions(), newProviders, snippetReflection, - originalProvider.getReplacements().getDefaultReplacementBytecodeProvider(), - originalProvider.getCodeCache().getTarget()); + HotSpotSnippetReplacementsImpl filteringReplacements = new HotSpotSnippetReplacementsImpl(newProviders, snippetReflection, + originalProvider.getReplacements().getDefaultReplacementBytecodeProvider(), originalProvider.getCodeCache().getTarget()); filteringReplacements.setGraphBuilderPlugins(originalProvider.getReplacements().getGraphBuilderPlugins()); - try (DebugContext.Scope scaope = debug.scope("VerifySnippetEncodeDecode", structuredGraph)) { + try (DebugContext.Scope scaope = debug.scope("VerifySnippetEncodeDecode", graph)) { for (int i = 0; i < encodedGraph.getNumObjects(); i++) { filterSnippetObject(encodedGraph.getObject(i)); } StructuredGraph snippet = filteringReplacements.makeGraph(debug, filteringReplacements.getDefaultReplacementBytecodeProvider(), method, null, original, trackNodeSourcePosition, null); SymbolicEncodedGraph symbolicGraph = new SymbolicEncodedGraph(encodedGraph, method.getDeclaringClass(), original != null ? methodKey(original) : null); - StructuredGraph decodedSnippet = decodeSnippetGraph(symbolicGraph, method, replacements, null, arch); + StructuredGraph decodedSnippet = decodeSnippetGraph(symbolicGraph, original != null ? original : method, originalReplacements, null, + StructuredGraph.AllowAssumptions.ifNonNull(graph.getAssumptions()), graph.getOptions()); String snippetString = getCanonicalGraphString(snippet, true, false); String decodedSnippetString = getCanonicalGraphString(decodedSnippet, true, false); if (snippetString.equals(decodedSnippetString)) { debug.log("Snippet decode for %s produces exactly same graph", method); - debug.dump(DebugContext.INFO_LEVEL, decodedSnippet, "Decoded snippet graph for %s", method); + debug.dump(DebugContext.VERBOSE_LEVEL, decodedSnippet, "Decoded snippet graph for %s", method); } else { debug.log("Snippet decode for %s produces different graph", method); debug.log("%s", compareGraphStrings(snippet, snippetString, decodedSnippet, decodedSnippetString)); - debug.dump(DebugContext.INFO_LEVEL, snippet, "Snippet graph for %s", method); - debug.dump(DebugContext.INFO_LEVEL, structuredGraph, "Encoded snippet graph for %s", method); - debug.dump(DebugContext.INFO_LEVEL, decodedSnippet, "Decoded snippet graph for %s", method); + debug.dump(DebugContext.VERBOSE_LEVEL, snippet, "Snippet graph for %s", method); + debug.dump(DebugContext.VERBOSE_LEVEL, graph, "Encoded snippet graph for %s", method); + debug.dump(DebugContext.VERBOSE_LEVEL, decodedSnippet, "Decoded snippet graph for %s", method); } } catch (Throwable t) { throw debug.handle(t); @@ -487,10 +468,10 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { * If there are new graphs waiting to be encoded, reencode all the graphs and return the result. */ @SuppressWarnings("try") - synchronized EncodedSnippets maybeEncodeSnippets() { + private synchronized EncodedSnippets maybeEncodeSnippets(OptionValues options) { Map graphs = this.preparedSnippetGraphs; if (encodedGraphs != graphs.size()) { - DebugContext debug = openDebugContext("SnippetEncoder", null); + DebugContext debug = openDebugContext("SnippetEncoder", null, options); try (DebugContext.Scope scope = debug.scope("SnippetSupportEncode")) { encodedGraphs = graphs.size(); for (StructuredGraph graph : graphs.values()) { @@ -504,23 +485,23 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { return null; } - @Override - public void registerSnippet(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean trackNodeSourcePosition) { - if (IS_BUILDING_NATIVE_IMAGE || UseEncodedSnippets.getValue(getOptions())) { + synchronized void registerSnippet(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean trackNodeSourcePosition, OptionValues options) { + if (IS_BUILDING_NATIVE_IMAGE || UseEncodedGraphs.getValue(options)) { assert method.getAnnotation(Snippet.class) != null : "Snippet must be annotated with @" + Snippet.class.getSimpleName(); String key = methodKey(method); if (!preparedSnippetGraphs.containsKey(key)) { if (original != null) { originalMethods.put(key, methodKey(original)); } - StructuredGraph snippet = buildGraph(method, original, receiver, true, trackNodeSourcePosition); + StructuredGraph snippet = buildGraph(method, original, receiver, true, trackNodeSourcePosition, INLINE_AFTER_PARSING, options); snippetMethods.add(method); preparedSnippetGraphs.put(key, snippet); } } + } - EncodedSnippets encodeSnippets(DebugContext debug) { + private synchronized EncodedSnippets encodeSnippets(DebugContext debug) { GraphEncoder encoder = new GraphEncoder(HotSpotJVMCIRuntime.runtime().getHostJVMCIBackend().getTarget().arch, debug); for (StructuredGraph graph : preparedSnippetGraphs.values()) { encoder.prepare(graph); @@ -552,8 +533,8 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { * Encode any outstanding graphs and return true if any work was done. */ @SuppressWarnings("try") - public boolean encode() { - EncodedSnippets encodedSnippets = maybeEncodeSnippets(); + public boolean encode(OptionValues options) { + EncodedSnippets encodedSnippets = maybeEncodeSnippets(options); if (encodedSnippets != null) { HotSpotReplacementsImpl.setEncodedSnippets(encodedSnippets); return true; @@ -561,38 +542,65 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { return false; } - private DebugContext openDebugContext(String idPrefix, ResolvedJavaMethod method) { - return replacements.openDebugContext(idPrefix, method); + private DebugContext openDebugContext(String idPrefix, ResolvedJavaMethod method, OptionValues options) { + return snippetReplacements.openDebugContext(idPrefix, method, options); } static class SymbolicEncodedGraph extends EncodedGraph { - private final ResolvedJavaType accessingClass; + private final ResolvedJavaType[] accessingClasses; private final String originalMethod; - SymbolicEncodedGraph(byte[] encoding, int startOffset, Object[] objects, NodeClass[] types, ResolvedJavaType accessingClass, String originalMethod) { + SymbolicEncodedGraph(byte[] encoding, int startOffset, Object[] objects, NodeClass[] types, String originalMethod, ResolvedJavaType... accessingClasses) { super(encoding, startOffset, objects, types, null, null, null, false, false); - this.accessingClass = accessingClass; + this.accessingClasses = accessingClasses; this.originalMethod = originalMethod; } SymbolicEncodedGraph(EncodedGraph encodedGraph, ResolvedJavaType declaringClass, String originalMethod) { - this(encodedGraph.getEncoding(), encodedGraph.getStartOffset(), encodedGraph.getObjects(), encodedGraph.getNodeClasses(), declaringClass, originalMethod); + this(encodedGraph.getEncoding(), encodedGraph.getStartOffset(), encodedGraph.getObjects(), encodedGraph.getNodeClasses(), + originalMethod, declaringClass); } @Override public Object getObject(int i) { Object o = objects[i]; + Object replacement = null; if (o instanceof SymbolicJVMCIReference) { - objects[i] = o = ((SymbolicJVMCIReference) o).resolve(accessingClass); + for (ResolvedJavaType type : accessingClasses) { + try { + replacement = ((SymbolicJVMCIReference) o).resolve(type); + break; + } catch (NoClassDefFoundError | AssertionError e) { + } + } } else if (o instanceof UnresolvedJavaType) { - objects[i] = o = ((UnresolvedJavaType) o).resolve(accessingClass); + for (ResolvedJavaType type : accessingClasses) { + try { + replacement = ((UnresolvedJavaType) o).resolve(type); + break; + } catch (NoClassDefFoundError | AssertionError e) { + } + } } else if (o instanceof UnresolvedJavaMethod) { throw new InternalError(o.toString()); } else if (o instanceof UnresolvedJavaField) { - objects[i] = o = ((UnresolvedJavaField) o).resolve(accessingClass); + for (ResolvedJavaType type : accessingClasses) { + try { + replacement = ((UnresolvedJavaField) o).resolve(type); + break; + } catch (NoClassDefFoundError | AssertionError e) { + } + } } else if (o instanceof GraalCapability) { - objects[i] = o = ((GraalCapability) o).resolve(((GraalJVMCICompiler) getRuntime().getCompiler()).getGraalRuntime()); + replacement = ((GraalCapability) o).resolve(((GraalJVMCICompiler) getRuntime().getCompiler()).getGraalRuntime()); + } else { + return o; + } + if (replacement != null) { + objects[i] = o = replacement; + } else { + throw new GraalError("Can't resolve " + o); } return o; } @@ -632,7 +640,7 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { final String methodName; final String signature; - SymbolicResolvedJavaMethod(HotSpotResolvedJavaMethod method) { + SymbolicResolvedJavaMethod(ResolvedJavaMethod method) { this.type = UnresolvedJavaType.create(method.getDeclaringClass().getName()); this.methodName = method.getName(); this.signature = method.getSignature().toMethodDescriptor(); @@ -650,6 +658,9 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { @Override public ResolvedJavaMethod resolve(ResolvedJavaType accessingClass) { ResolvedJavaType resolvedType = type.resolve(accessingClass); + if (resolvedType == null) { + throw new InternalError("Could not resolve " + this + " in context of " + accessingClass.toJavaName()); + } for (ResolvedJavaMethod method : methodName.equals("") ? resolvedType.getDeclaredConstructors() : resolvedType.getDeclaredMethods()) { if (method.getName().equals(methodName) && method.getSignature().toMethodDescriptor().equals(signature)) { return method; @@ -665,7 +676,7 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { final UnresolvedJavaType signature; private final boolean isStatic; - SymbolicResolvedJavaField(HotSpotResolvedJavaField field) { + SymbolicResolvedJavaField(ResolvedJavaField field) { this.declaringType = UnresolvedJavaType.create(field.getDeclaringClass().getName()); this.name = field.getName(); this.signature = UnresolvedJavaType.create(field.getType().getName()); @@ -697,6 +708,19 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { } } + static class SymbolicResolvedJavaMethodBytecode implements SymbolicJVMCIReference { + SymbolicResolvedJavaMethod method; + + SymbolicResolvedJavaMethodBytecode(ResolvedJavaMethodBytecode bytecode) { + method = new SymbolicResolvedJavaMethod(bytecode.getMethod()); + } + + @Override + public ResolvedJavaMethodBytecode resolve(ResolvedJavaType accessingClass) { + return new ResolvedJavaMethodBytecode(method.resolve(accessingClass)); + } + } + static class SymbolicStampPair implements SymbolicJVMCIReference { Object trustedStamp; Object uncheckdStamp; @@ -820,13 +844,13 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { * Objects embedded in encoded graphs might need to converted into a symbolic form so convert * the object or pass it through. */ - static Object filterSnippetObject(Object o) { + private static Object filterSnippetObject(Object o) { if (o instanceof HotSpotResolvedJavaMethod) { return new SymbolicResolvedJavaMethod((HotSpotResolvedJavaMethod) o); } else if (o instanceof HotSpotResolvedJavaField) { return new SymbolicResolvedJavaField((HotSpotResolvedJavaField) o); } else if (o instanceof HotSpotResolvedJavaType) { - return UnresolvedJavaType.create(((HotSpotResolvedJavaType) o).getName()); + return UnresolvedJavaType.create(((ResolvedJavaType) o).getName()); } else if (o instanceof NodeSourcePosition) { // Filter these out for now. These can't easily be handled because these positions // description snippet methods which might not be available in the runtime. @@ -843,11 +867,15 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { if (((StampPair) o).getTrustedStamp() instanceof AbstractObjectStamp) { return new SymbolicStampPair((StampPair) o); } + } else if (o instanceof ResolvedJavaMethodBytecode) { + return new SymbolicResolvedJavaMethodBytecode((ResolvedJavaMethodBytecode) o); + } else if (o instanceof HotSpotSignature) { + throw new GraalError(o.toString()); } return o; } - static String compareGraphStrings(StructuredGraph expectedGraph, String expectedString, StructuredGraph actualGraph, String actualString) { + private static String compareGraphStrings(StructuredGraph expectedGraph, String expectedString, StructuredGraph actualGraph, String actualString) { if (!expectedString.equals(actualString)) { String[] expectedLines = expectedString.split("\n"); String[] actualLines = actualString.split("\n"); @@ -883,7 +911,7 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { } } - static String getCanonicalGraphString(StructuredGraph graph, boolean excludeVirtual, boolean checkConstants) { + private static String getCanonicalGraphString(StructuredGraph graph, boolean excludeVirtual, boolean checkConstants) { SchedulePhase schedule = new SchedulePhase(SchedulePhase.SchedulingStrategy.EARLIEST); schedule.apply(graph); StructuredGraph.ScheduleResult scheduleResult = graph.getLastSchedule(); @@ -968,8 +996,8 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { super(replacements, providers); } - HotSpotSnippetReplacementsImpl(OptionValues options, Providers providers, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider, TargetDescription target) { - super(options, providers, snippetReflection, bytecodeProvider, target); + HotSpotSnippetReplacementsImpl(Providers providers, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider, TargetDescription target) { + super(providers, snippetReflection, bytecodeProvider, target); } @Override @@ -1009,14 +1037,25 @@ public class SymbolicSnippetEncoder extends DelegatingReplacements { @Override public boolean canDeferPlugin(GeneratedInvocationPlugin plugin) { + // Fold is always deferred but NodeIntrinsics may have to wait if all their arguments + // aren't constant yet. return plugin.getSource().equals(Fold.class) || plugin.getSource().equals(Node.NodeIntrinsic.class); } + @Override + protected boolean canInlinePartialIntrinsicExit() { + return false; + } + @Override protected boolean tryInvocationPlugin(CallTargetNode.InvokeKind invokeKind, ValueNode[] args, ResolvedJavaMethod targetMethod, JavaKind resultType, JavaType returnType) { if (intrinsicContext != null && intrinsicContext.isCallToOriginal(targetMethod)) { return false; } + if (targetMethod.getAnnotation(Fold.class) != null) { + // Always defer Fold until decode time but NodeIntrinsics may fold if they are able. + return false; + } return super.tryInvocationPlugin(invokeKind, args, targetMethod, resultType, returnType); } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java index 1955d772ebf..c3946e8f5c6 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java @@ -161,7 +161,7 @@ import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.replacements.DefaultJavaLoweringProvider; import org.graalvm.compiler.replacements.arraycopy.ArrayCopyNode; import org.graalvm.compiler.replacements.arraycopy.ArrayCopySnippets; -import org.graalvm.compiler.replacements.arraycopy.ArrayCopyWithSlowPathNode; +import org.graalvm.compiler.replacements.arraycopy.ArrayCopyWithDelayedLoweringNode; import org.graalvm.compiler.replacements.nodes.AssertionNode; import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import jdk.internal.vm.compiler.word.LocationIdentity; @@ -234,6 +234,10 @@ public class DefaultHotSpotLoweringProvider extends DefaultJavaLoweringProvider foreignCallSnippets = new ForeignCallSnippets.Templates(options, factories, providers, target); } + public ArrayCopySnippets.Templates getArraycopySnippets() { + return arraycopySnippets; + } + public MonitorSnippets.Templates getMonitorSnippets() { return monitorSnippets; } @@ -332,8 +336,8 @@ public class DefaultHotSpotLoweringProvider extends DefaultJavaLoweringProvider } } else if (n instanceof ArrayCopyNode) { arraycopySnippets.lower((ArrayCopyNode) n, tool); - } else if (n instanceof ArrayCopyWithSlowPathNode) { - arraycopySnippets.lower((ArrayCopyWithSlowPathNode) n, tool); + } else if (n instanceof ArrayCopyWithDelayedLoweringNode) { + arraycopySnippets.lower((ArrayCopyWithDelayedLoweringNode) n, tool); } else if (n instanceof G1PreWriteBarrier) { writeBarrierSnippets.lower((G1PreWriteBarrier) n, registers, tool); } else if (n instanceof G1PostWriteBarrier) { @@ -468,11 +472,15 @@ public class DefaultHotSpotLoweringProvider extends DefaultJavaLoweringProvider if (invoke.callTarget() instanceof MethodCallTargetNode) { MethodCallTargetNode callTarget = (MethodCallTargetNode) invoke.callTarget(); NodeInputList parameters = callTarget.arguments(); - ValueNode receiver = parameters.size() <= 0 ? null : parameters.get(0); - if (!callTarget.isStatic() && receiver.stamp(NodeView.DEFAULT) instanceof ObjectStamp && !StampTool.isPointerNonNull(receiver)) { - ValueNode nonNullReceiver = createNullCheckedValue(receiver, invoke.asNode(), tool); - parameters.set(0, nonNullReceiver); - receiver = nonNullReceiver; + ValueNode receiver = parameters.isEmpty() ? null : parameters.get(0); + + if (!callTarget.isStatic()) { + assert receiver != null : "non-static call must have a receiver"; + if (receiver.stamp(NodeView.DEFAULT) instanceof ObjectStamp && !StampTool.isPointerNonNull(receiver)) { + ValueNode nonNullReceiver = createNullCheckedValue(receiver, invoke.asNode(), tool); + parameters.set(0, nonNullReceiver); + receiver = nonNullReceiver; + } } JavaType[] signature = callTarget.targetMethod().getSignature().toParameterTypes(callTarget.isStatic() ? null : callTarget.targetMethod().getDeclaringClass()); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotClassInitializationPlugin.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotClassInitializationPlugin.java index a5492d7d47a..682140b0e82 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotClassInitializationPlugin.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotClassInitializationPlugin.java @@ -24,13 +24,11 @@ package org.graalvm.compiler.hotspot.meta; -import java.lang.reflect.Method; import java.util.function.Supplier; import org.graalvm.compiler.core.common.type.ObjectStamp; import org.graalvm.compiler.core.common.type.Stamp; import org.graalvm.compiler.core.common.type.StampFactory; -import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.hotspot.nodes.aot.InitializeKlassNode; import org.graalvm.compiler.hotspot.nodes.aot.ResolveConstantNode; import org.graalvm.compiler.nodes.ConstantNode; @@ -40,6 +38,7 @@ import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.graphbuilderconf.ClassInitializationPlugin; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; +import jdk.vm.ci.hotspot.HotSpotConstantPool; import jdk.vm.ci.hotspot.HotSpotResolvedObjectType; import jdk.vm.ci.meta.ConstantPool; import jdk.vm.ci.meta.ResolvedJavaMethod; @@ -85,44 +84,17 @@ public final class HotSpotClassInitializationPlugin implements ClassInitializati return false; } - private static final Class hscp; - private static final Method loadReferencedTypeIIZMH; - - static { - Method m = null; - Class c = null; - try { - c = Class.forName("jdk.vm.ci.hotspot.HotSpotConstantPool").asSubclass(ConstantPool.class); - m = c.getDeclaredMethod("loadReferencedType", int.class, int.class, boolean.class); - } catch (Exception e) { - throw GraalError.shouldNotReachHere(e); - } - loadReferencedTypeIIZMH = m; - hscp = c; - } - - private static boolean isHotSpotConstantPool(ConstantPool cp) { - // jdk.vm.ci.hotspot.HotSpotConstantPool is final, so we can - // directly compare Classes. - return cp.getClass() == hscp; - } - @Override public boolean supportsLazyInitialization(ConstantPool cp) { - if (loadReferencedTypeIIZMH != null && isHotSpotConstantPool(cp)) { - return true; - } - return false; + // jdk.vm.ci.hotspot.HotSpotConstantPool is final, so we can + // directly compare Classes. + return (cp instanceof HotSpotConstantPool); } @Override public void loadReferencedType(GraphBuilderContext builder, ConstantPool cp, int cpi, int opcode) { - if (loadReferencedTypeIIZMH != null && isHotSpotConstantPool(cp)) { - try { - loadReferencedTypeIIZMH.invoke(cp, cpi, opcode, false); - } catch (Throwable t) { - throw GraalError.shouldNotReachHere(t); - } + if (cp instanceof HotSpotConstantPool) { + ((HotSpotConstantPool) cp).loadReferencedType(cpi, opcode, false); } else { cp.loadReferencedType(cpi, opcode); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotForeignCallsProviderImpl.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotForeignCallsProviderImpl.java index fe6785cd5e4..afc538bc373 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotForeignCallsProviderImpl.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotForeignCallsProviderImpl.java @@ -35,6 +35,7 @@ import java.util.List; import jdk.internal.vm.compiler.collections.EconomicMap; import org.graalvm.compiler.core.common.LIRKind; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; +import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage; import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Reexecutability; import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.RegisterEffect; @@ -207,10 +208,18 @@ public abstract class HotSpotForeignCallsProviderImpl implements HotSpotForeignC public HotSpotForeignCallLinkage lookupForeignCall(ForeignCallDescriptor descriptor) { assert foreignCalls != null : descriptor; HotSpotForeignCallLinkage callTarget = foreignCalls.get(descriptor); + if (callTarget == null) { + throw GraalError.shouldNotReachHere("missing implementation for runtime call: " + descriptor); + } callTarget.finalizeAddress(runtime.getHostBackend()); return callTarget; } + @Override + public boolean isAvailable(ForeignCallDescriptor descriptor) { + return foreignCalls.containsKey(descriptor); + } + @Override public boolean isReexecutable(ForeignCallDescriptor descriptor) { assert foreignCalls.containsKey(descriptor) : "unknown foreign call: " + descriptor; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java index fe0fab5faf6..e00ac5b6b11 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java @@ -25,6 +25,7 @@ package org.graalvm.compiler.hotspot.meta; import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC; +import static org.graalvm.compiler.hotspot.HotSpotBackend.BASE64_ENCODE_BLOCK; import static org.graalvm.compiler.hotspot.HotSpotBackend.GHASH_PROCESS_BLOCKS; import static org.graalvm.compiler.hotspot.meta.HotSpotAOTProfilingPlugin.Options.TieredAOT; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.JAVA_THREAD_THREAD_OBJECT_LOCATION; @@ -123,9 +124,11 @@ public class HotSpotGraphBuilderPlugins { * @param constantReflection * @param snippetReflection * @param foreignCalls + * @param options */ public static Plugins create(CompilerConfiguration compilerConfiguration, GraalHotSpotVMConfig config, HotSpotWordTypes wordTypes, MetaAccessProvider metaAccess, - ConstantReflectionProvider constantReflection, SnippetReflectionProvider snippetReflection, ForeignCallsProvider foreignCalls, ReplacementsImpl replacements) { + ConstantReflectionProvider constantReflection, SnippetReflectionProvider snippetReflection, ForeignCallsProvider foreignCalls, ReplacementsImpl replacements, + OptionValues options) { InvocationPlugins invocationPlugins = new HotSpotInvocationPlugins(config, compilerConfiguration); Plugins plugins = new Plugins(invocationPlugins); @@ -135,7 +138,6 @@ public class HotSpotGraphBuilderPlugins { plugins.appendTypePlugin(nodePlugin); plugins.appendNodePlugin(nodePlugin); - OptionValues options = replacements.getOptions(); if (!GeneratePIC.getValue(options)) { plugins.appendNodePlugin(new MethodHandlePlugin(constantReflection.getMethodHandleAccess(), true)); } @@ -172,6 +174,7 @@ public class HotSpotGraphBuilderPlugins { registerSHAPlugins(invocationPlugins, config, replacementBytecodeProvider); registerGHASHPlugins(invocationPlugins, config, metaAccess, foreignCalls); registerCounterModePlugins(invocationPlugins, config, replacementBytecodeProvider); + registerBase64Plugins(invocationPlugins, config, metaAccess, foreignCalls); registerUnsafePlugins(invocationPlugins, replacementBytecodeProvider); StandardGraphBuilderPlugins.registerInvocationPlugins(metaAccess, snippetReflection, invocationPlugins, replacementBytecodeProvider, true, false); registerArrayPlugins(invocationPlugins, replacementBytecodeProvider); @@ -563,6 +566,38 @@ public class HotSpotGraphBuilderPlugins { } } + private static void registerBase64Plugins(InvocationPlugins plugins, GraalHotSpotVMConfig config, MetaAccessProvider metaAccess, ForeignCallsProvider foreignCalls) { + if (config.useBase64Intrinsics()) { + Registration r = new Registration(plugins, "java.util.Base64$Encoder"); + r.register7("encodeBlock", + Receiver.class, + byte[].class, + int.class, + int.class, + byte[].class, + int.class, + boolean.class, + new InvocationPlugin() { + @Override + public boolean apply(GraphBuilderContext b, + ResolvedJavaMethod targetMethod, + Receiver receiver, + ValueNode src, + ValueNode sp, + ValueNode sl, + ValueNode dst, + ValueNode dp, + ValueNode isURL) { + int byteArrayBaseOffset = metaAccess.getArrayBaseOffset(JavaKind.Byte); + ComputeObjectAddressNode srcAddress = b.add(new ComputeObjectAddressNode(src, ConstantNode.forInt(byteArrayBaseOffset))); + ComputeObjectAddressNode dstAddress = b.add(new ComputeObjectAddressNode(dst, ConstantNode.forInt(byteArrayBaseOffset))); + b.add(new ForeignCallNode(foreignCalls, BASE64_ENCODE_BLOCK, srcAddress, sp, sl, dstAddress, dp, isURL)); + return true; + } + }); + } + } + private static void registerCRC32Plugins(InvocationPlugins plugins, GraalHotSpotVMConfig config, BytecodeProvider bytecodeProvider) { if (config.useCRC32Intrinsics) { Registration r = new Registration(plugins, CRC32.class, bytecodeProvider); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java index 25a9db510b8..0a4367ca7da 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java @@ -29,6 +29,7 @@ import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC; import static org.graalvm.compiler.core.target.Backend.ARITHMETIC_DREM; import static org.graalvm.compiler.core.target.Backend.ARITHMETIC_FREM; import static org.graalvm.compiler.hotspot.HotSpotBackend.BACKEDGE_EVENT; +import static org.graalvm.compiler.hotspot.HotSpotBackend.BASE64_ENCODE_BLOCK; import static org.graalvm.compiler.hotspot.HotSpotBackend.COUNTERMODE_IMPL_CRYPT; import static org.graalvm.compiler.hotspot.HotSpotBackend.DECRYPT; import static org.graalvm.compiler.hotspot.HotSpotBackend.DECRYPT_BLOCK; @@ -72,7 +73,7 @@ import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Reexecutabi import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.RegisterEffect.DESTROYS_REGISTERS; import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.RegisterEffect.PRESERVES_REGISTERS; import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Transition.LEAF; -import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Transition.LEAF_NOFP; +import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Transition.LEAF_NO_VZERO; import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Transition.SAFEPOINT; import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Transition.STACK_INSPECTABLE_LEAF; import static org.graalvm.compiler.hotspot.HotSpotHostBackend.DEOPTIMIZATION_HANDLER; @@ -217,7 +218,7 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall String name = kind + (aligned ? "Aligned" : "") + (disjoint ? "Disjoint" : "") + (uninit ? "Uninit" : "") + "Arraycopy" + (killAny ? "KillAny" : ""); ForeignCallDescriptor desc = new ForeignCallDescriptor(name, void.class, Word.class, Word.class, Word.class); LocationIdentity killed = killAny ? LocationIdentity.any() : NamedLocationIdentity.getArrayLocation(kind); - registerForeignCall(desc, routine, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, NOT_REEXECUTABLE, killed); + registerForeignCall(desc, routine, NativeCall, DESTROYS_REGISTERS, LEAF_NO_VZERO, NOT_REEXECUTABLE, killed); return desc; } @@ -232,7 +233,7 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall // return: 0 = success, n = number of copied elements xor'd with -1. ForeignCallDescriptor desc = new ForeignCallDescriptor(name, int.class, Word.class, Word.class, Word.class, Word.class, Word.class); LocationIdentity killed = NamedLocationIdentity.any(); - registerForeignCall(desc, routine, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, NOT_REEXECUTABLE, killed); + registerForeignCall(desc, routine, NativeCall, DESTROYS_REGISTERS, LEAF_NO_VZERO, NOT_REEXECUTABLE, killed); checkcastArraycopyDescriptors[uninit ? 1 : 0] = desc; } @@ -272,25 +273,25 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall public void initialize(HotSpotProviders providers, OptionValues options) { GraalHotSpotVMConfig c = runtime.getVMConfig(); - registerForeignCall(DEOPTIMIZATION_HANDLER, c.handleDeoptStub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE, NO_LOCATIONS); - registerForeignCall(UNCOMMON_TRAP_HANDLER, c.uncommonTrapStub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE, NO_LOCATIONS); - registerForeignCall(IC_MISS_HANDLER, c.inlineCacheMissStub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(DEOPTIMIZATION_HANDLER, c.handleDeoptStub, NativeCall, PRESERVES_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(UNCOMMON_TRAP_HANDLER, c.uncommonTrapStub, NativeCall, PRESERVES_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(IC_MISS_HANDLER, c.inlineCacheMissStub, NativeCall, PRESERVES_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE, NO_LOCATIONS); if (c.enableStackReservedZoneAddress != 0) { assert c.throwDelayedStackOverflowErrorEntry != 0 : "both must exist"; - registerForeignCall(ENABLE_STACK_RESERVED_ZONE, c.enableStackReservedZoneAddress, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE, NO_LOCATIONS); - registerForeignCall(THROW_DELAYED_STACKOVERFLOW_ERROR, c.throwDelayedStackOverflowErrorEntry, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(ENABLE_STACK_RESERVED_ZONE, c.enableStackReservedZoneAddress, NativeCall, DESTROYS_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(THROW_DELAYED_STACKOVERFLOW_ERROR, c.throwDelayedStackOverflowErrorEntry, NativeCall, DESTROYS_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE, NO_LOCATIONS); } - registerForeignCall(JAVA_TIME_MILLIS, c.javaTimeMillisAddress, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE, NO_LOCATIONS); - registerForeignCall(JAVA_TIME_NANOS, c.javaTimeNanosAddress, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(JAVA_TIME_MILLIS, c.javaTimeMillisAddress, NativeCall, DESTROYS_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(JAVA_TIME_NANOS, c.javaTimeNanosAddress, NativeCall, DESTROYS_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE, NO_LOCATIONS); registerMathStubs(c, providers, options); registerForeignCall(ARITHMETIC_FREM, c.fremAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); registerForeignCall(ARITHMETIC_DREM, c.dremAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); - registerForeignCall(LOAD_AND_CLEAR_EXCEPTION, c.loadAndClearExceptionAddress, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, any()); + registerForeignCall(LOAD_AND_CLEAR_EXCEPTION, c.loadAndClearExceptionAddress, NativeCall, DESTROYS_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE_ONLY_AFTER_EXCEPTION, any()); registerForeignCall(EXCEPTION_HANDLER_FOR_PC, c.exceptionHandlerForPcAddress, NativeCall, DESTROYS_REGISTERS, SAFEPOINT, REEXECUTABLE, any()); registerForeignCall(EXCEPTION_HANDLER_FOR_RETURN_ADDRESS, c.exceptionHandlerForReturnAddressAddress, NativeCall, DESTROYS_REGISTERS, SAFEPOINT, REEXECUTABLE, any()); @@ -318,7 +319,7 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall link(new ExceptionHandlerStub(options, providers, foreignCalls.get(EXCEPTION_HANDLER))); link(new UnwindExceptionToCallerStub(options, providers, registerStubCall(UNWIND_EXCEPTION_TO_CALLER, SAFEPOINT, REEXECUTABLE_ONLY_AFTER_EXCEPTION, any()))); - link(new VerifyOopStub(options, providers, registerStubCall(VERIFY_OOP, LEAF_NOFP, REEXECUTABLE, NO_LOCATIONS))); + link(new VerifyOopStub(options, providers, registerStubCall(VERIFY_OOP, LEAF_NO_VZERO, REEXECUTABLE, NO_LOCATIONS))); EnumMap exceptionRuntimeCalls = DefaultHotSpotLoweringProvider.RuntimeCalls.runtimeCalls; link(new ArrayStoreExceptionStub(options, providers, registerStubCall(exceptionRuntimeCalls.get(BytecodeExceptionKind.ARRAY_STORE), SAFEPOINT, REEXECUTABLE, any()))); @@ -338,14 +339,14 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall linkForeignCall(options, providers, LOG_PRINTF, c.logPrintfAddress, PREPEND_THREAD, LEAF, REEXECUTABLE, NO_LOCATIONS); linkForeignCall(options, providers, LOG_OBJECT, c.logObjectAddress, PREPEND_THREAD, LEAF, REEXECUTABLE, NO_LOCATIONS); linkForeignCall(options, providers, LOG_PRIMITIVE, c.logPrimitiveAddress, PREPEND_THREAD, LEAF, REEXECUTABLE, NO_LOCATIONS); - linkForeignCall(options, providers, VM_ERROR, c.vmErrorAddress, PREPEND_THREAD, LEAF_NOFP, REEXECUTABLE, NO_LOCATIONS); - linkForeignCall(options, providers, OSR_MIGRATION_END, c.osrMigrationEndAddress, DONT_PREPEND_THREAD, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NO_LOCATIONS); - linkForeignCall(options, providers, G1WBPRECALL, c.writeBarrierPreAddress, PREPEND_THREAD, LEAF_NOFP, REEXECUTABLE, NO_LOCATIONS); - linkForeignCall(options, providers, G1WBPOSTCALL, c.writeBarrierPostAddress, PREPEND_THREAD, LEAF_NOFP, REEXECUTABLE, NO_LOCATIONS); - linkForeignCall(options, providers, VALIDATE_OBJECT, c.validateObject, PREPEND_THREAD, LEAF_NOFP, REEXECUTABLE, NO_LOCATIONS); + linkForeignCall(options, providers, VM_ERROR, c.vmErrorAddress, PREPEND_THREAD, LEAF_NO_VZERO, REEXECUTABLE, NO_LOCATIONS); + linkForeignCall(options, providers, OSR_MIGRATION_END, c.osrMigrationEndAddress, DONT_PREPEND_THREAD, LEAF_NO_VZERO, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NO_LOCATIONS); + linkForeignCall(options, providers, G1WBPRECALL, c.writeBarrierPreAddress, PREPEND_THREAD, LEAF_NO_VZERO, REEXECUTABLE, NO_LOCATIONS); + linkForeignCall(options, providers, G1WBPOSTCALL, c.writeBarrierPostAddress, PREPEND_THREAD, LEAF_NO_VZERO, REEXECUTABLE, NO_LOCATIONS); + linkForeignCall(options, providers, VALIDATE_OBJECT, c.validateObject, PREPEND_THREAD, LEAF_NO_VZERO, REEXECUTABLE, NO_LOCATIONS); if (GeneratePIC.getValue(options)) { - registerForeignCall(WRONG_METHOD_HANDLER, c.handleWrongMethodStub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(WRONG_METHOD_HANDLER, c.handleWrongMethodStub, NativeCall, PRESERVES_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE, NO_LOCATIONS); CompilerRuntimeHotSpotVMConfig cr = new CompilerRuntimeHotSpotVMConfig(HotSpotJVMCIRuntime.runtime().getConfigStore()); linkForeignCall(options, providers, RESOLVE_STRING_BY_SYMBOL, cr.resolveStringBySymbol, PREPEND_THREAD, SAFEPOINT, REEXECUTABLE, TLAB_TOP_LOCATION, TLAB_END_LOCATION); linkForeignCall(options, providers, RESOLVE_DYNAMIC_INVOKE, cr.resolveDynamicInvoke, PREPEND_THREAD, SAFEPOINT, REEXECUTABLE, any()); @@ -375,11 +376,11 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall registerCheckcastArraycopyDescriptor(true, c.checkcastArraycopyUninit); registerCheckcastArraycopyDescriptor(false, c.checkcastArraycopy); - registerForeignCall(GENERIC_ARRAYCOPY, c.genericArraycopy, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); - registerForeignCall(UNSAFE_ARRAYCOPY, c.unsafeArraycopy, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); + registerForeignCall(GENERIC_ARRAYCOPY, c.genericArraycopy, NativeCall, DESTROYS_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); + registerForeignCall(UNSAFE_ARRAYCOPY, c.unsafeArraycopy, NativeCall, DESTROYS_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); if (c.useMultiplyToLenIntrinsic()) { - registerForeignCall(MULTIPLY_TO_LEN, c.multiplyToLen, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, + registerForeignCall(MULTIPLY_TO_LEN, c.multiplyToLen, NativeCall, DESTROYS_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Int)); } @@ -398,19 +399,22 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall if (c.useGHASHIntrinsics()) { registerForeignCall(GHASH_PROCESS_BLOCKS, c.ghashProcessBlocks, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); } + if (c.useBase64Intrinsics()) { + registerForeignCall(BASE64_ENCODE_BLOCK, c.base64EncodeBlock, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); + } if (c.useMulAddIntrinsic()) { - registerForeignCall(MUL_ADD, c.mulAdd, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Int)); + registerForeignCall(MUL_ADD, c.mulAdd, NativeCall, DESTROYS_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Int)); } if (c.useMontgomeryMultiplyIntrinsic()) { - registerForeignCall(MONTGOMERY_MULTIPLY, c.montgomeryMultiply, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, + registerForeignCall(MONTGOMERY_MULTIPLY, c.montgomeryMultiply, NativeCall, DESTROYS_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Int)); } if (c.useMontgomerySquareIntrinsic()) { - registerForeignCall(MONTGOMERY_SQUARE, c.montgomerySquare, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, + registerForeignCall(MONTGOMERY_SQUARE, c.montgomerySquare, NativeCall, DESTROYS_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Int)); } if (c.useSquareToLenIntrinsic()) { - registerForeignCall(SQUARE_TO_LEN, c.squareToLen, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Int)); + registerForeignCall(SQUARE_TO_LEN, c.squareToLen, NativeCall, DESTROYS_REGISTERS, LEAF_NO_VZERO, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Int)); } if (c.useAESIntrinsics) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/aot/AOTInliningPolicy.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/aot/AOTInliningPolicy.java index 984e9d7a6ea..5248f0abe7e 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/aot/AOTInliningPolicy.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/aot/AOTInliningPolicy.java @@ -65,8 +65,9 @@ public class AOTInliningPolicy extends GreedyInliningPolicy { } @Override - public Decision isWorthInlining(Replacements replacements, MethodInvocation invocation, int inliningDepth, boolean fullyProcessed) { - final boolean isTracing = GraalOptions.TraceInlining.getValue(replacements.getOptions()); + public Decision isWorthInlining(Replacements replacements, MethodInvocation invocation, InlineInfo calleeInfo, int inliningDepth, boolean fullyProcessed) { + OptionValues options = calleeInfo.graph().getOptions(); + final boolean isTracing = GraalOptions.TraceInlining.getValue(options); final InlineInfo info = invocation.callee(); for (int i = 0; i < info.numberOfMethods(); ++i) { @@ -79,7 +80,6 @@ public class AOTInliningPolicy extends GreedyInliningPolicy { final double probability = invocation.probability(); final double relevance = invocation.relevance(); - OptionValues options = info.graph().getOptions(); if (InlineEverything.getValue(options)) { InliningUtil.traceInlinedMethod(info, inliningDepth, fullyProcessed, "inline everything"); return InliningPolicy.Decision.YES.withReason(isTracing, "inline everything"); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotReplacementsUtil.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotReplacementsUtil.java index 23b6a2246fe..81fa5ef94a3 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotReplacementsUtil.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotReplacementsUtil.java @@ -160,7 +160,7 @@ public class HotSpotReplacementsUtil { return field.getOffset(); } } - throw new GraalError("missing field " + fieldName); + throw new GraalError("missing field " + fieldName + " in type " + type); } public static HotSpotJVMCIRuntime runtime() { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/MonitorSnippets.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/MonitorSnippets.java index d3fe8b76158..863c5efb381 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/MonitorSnippets.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/MonitorSnippets.java @@ -831,7 +831,8 @@ public class MonitorSnippets implements Snippets { invoke.setStateAfter(graph.start().stateAfter()); graph.addAfterFixed(graph.start(), invoke); - StructuredGraph inlineeGraph = providers.getReplacements().getSnippet(initCounter.getMethod(), null, invoke.graph().trackNodeSourcePosition(), invoke.getNodeSourcePosition()); + StructuredGraph inlineeGraph = providers.getReplacements().getSnippet(initCounter.getMethod(), null, null, invoke.graph().trackNodeSourcePosition(), invoke.getNodeSourcePosition(), + invoke.getOptions()); InliningUtil.inline(invoke, inlineeGraph, false, null); List rets = graph.getNodes(ReturnNode.TYPE).snapshot(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/ObjectCloneNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/ObjectCloneNode.java index e91a3eb11ce..d879667359d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/ObjectCloneNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/ObjectCloneNode.java @@ -89,7 +89,7 @@ public final class ObjectCloneNode extends BasicObjectCloneNode { StructuredGraph snippetGraph = null; DebugContext debug = getDebug(); try (DebugContext.Scope s = debug.scope("ArrayCloneSnippet", snippetMethod)) { - snippetGraph = replacements.getSnippet(snippetMethod, null, graph().trackNodeSourcePosition(), this.getNodeSourcePosition()); + snippetGraph = replacements.getSnippet(snippetMethod, null, null, graph().trackNodeSourcePosition(), this.getNodeSourcePosition(), debug.getOptions()); } catch (Throwable e) { throw debug.handle(e); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/SnippetStub.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/SnippetStub.java index bfbdcc41105..7874332700c 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/SnippetStub.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/SnippetStub.java @@ -84,7 +84,7 @@ public abstract class SnippetStub extends Stub implements Snippets { } protected void registerSnippet() { - providers.getReplacements().registerSnippet(method, null, null, false); + providers.getReplacements().registerSnippet(method, null, null, false, options); } @Override @@ -115,7 +115,7 @@ public abstract class SnippetStub extends Stub implements Snippets { } protected StructuredGraph buildInitialGraph(DebugContext debug, CompilationIdentifier compilationId, Object[] args) { - return providers.getReplacements().getSnippet(method, args, false, null).copyWithIdentifier(compilationId, debug); + return providers.getReplacements().getSnippet(method, null, args, false, null, options).copyWithIdentifier(compilationId, debug); } protected boolean checkConstArg(int index, String expectedName) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/Stub.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/Stub.java index 15fbf21c00c..9b64df99010 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/Stub.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/Stub.java @@ -183,7 +183,7 @@ public abstract class Stub { Description description = new Description(linkage, "Stub_" + nextStubId.incrementAndGet()); return DebugContext.create(options, description, outer.getGlobalMetrics(), DEFAULT_LOG_STREAM, singletonList(new GraalDebugHandlersFactory(providers.getSnippetReflection()))); } - return DebugContext.DISABLED; + return DebugContext.disabled(options); } /** diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BciBlockMapping.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BciBlockMapping.java index d2e905c96db..d0642e4137d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BciBlockMapping.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BciBlockMapping.java @@ -900,8 +900,7 @@ public final class BciBlockMapping { assert next == newBlocks.length - 1; // Add unwind block. - int deoptBci = code.getMethod().isSynchronized() ? BytecodeFrame.UNWIND_BCI : BytecodeFrame.AFTER_EXCEPTION_BCI; - ExceptionDispatchBlock unwindBlock = new ExceptionDispatchBlock(deoptBci); + ExceptionDispatchBlock unwindBlock = new ExceptionDispatchBlock(BytecodeFrame.AFTER_EXCEPTION_BCI); unwindBlock.setId(newBlocks.length - 1); newBlocks[newBlocks.length - 1] = unwindBlock; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BytecodeParser.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BytecodeParser.java index 7b099fd14bc..a2e88218bdc 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BytecodeParser.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BytecodeParser.java @@ -27,6 +27,7 @@ package org.graalvm.compiler.java; import static java.lang.String.format; import static java.lang.reflect.Modifier.STATIC; import static java.lang.reflect.Modifier.SYNCHRONIZED; +import static jdk.vm.ci.code.BytecodeFrame.UNKNOWN_BCI; import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateRecompile; import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateReprofile; import static jdk.vm.ci.meta.DeoptimizationAction.None; @@ -256,7 +257,7 @@ import static org.graalvm.compiler.java.BytecodeParserOptions.TraceBytecodeParse import static org.graalvm.compiler.java.BytecodeParserOptions.TraceInlineDuringParsing; import static org.graalvm.compiler.java.BytecodeParserOptions.TraceParserPlugins; import static org.graalvm.compiler.java.BytecodeParserOptions.UseGuardedIntrinsics; -import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LUDICROUSLY_FAST_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LUDICROUSLY_SLOW_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.INLINE_DURING_PARSING; @@ -271,6 +272,7 @@ import java.util.function.Supplier; import jdk.internal.vm.compiler.collections.EconomicMap; import jdk.internal.vm.compiler.collections.Equivalence; +import jdk.internal.vm.compiler.collections.UnmodifiableEconomicMap; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.api.replacements.Snippet; import org.graalvm.compiler.bytecode.Bytecode; @@ -383,6 +385,7 @@ import org.graalvm.compiler.nodes.extended.AnchoringNode; import org.graalvm.compiler.nodes.extended.BranchProbabilityNode; import org.graalvm.compiler.nodes.extended.BytecodeExceptionNode; import org.graalvm.compiler.nodes.extended.BytecodeExceptionNode.BytecodeExceptionKind; +import org.graalvm.compiler.nodes.extended.ForeignCallNode; import org.graalvm.compiler.nodes.extended.GuardingNode; import org.graalvm.compiler.nodes.extended.IntegerSwitchNode; import org.graalvm.compiler.nodes.extended.LoadArrayComponentHubNode; @@ -488,10 +491,11 @@ public class BytecodeParser implements GraphBuilderContext { protected static final CounterKey EXPLICIT_EXCEPTIONS = DebugContext.counter("ExplicitExceptions"); /** - * A scoped object for tasks to be performed after parsing an intrinsic such as processing + * A scoped object for tasks to be performed after inlining during parsing such as processing * {@linkplain BytecodeFrame#isPlaceholderBci(int) placeholder} frames states. */ - static class IntrinsicScope implements AutoCloseable { + static class InliningScope implements AutoCloseable { + final ResolvedJavaMethod callee; FrameState stateBefore; final Mark mark; final BytecodeParser parser; @@ -502,44 +506,41 @@ public class BytecodeParser implements GraphBuilderContext { * * @param parser the parsing context of the intrinsic */ - IntrinsicScope(BytecodeParser parser) { + InliningScope(BytecodeParser parser) { this.parser = parser; assert parser.parent == null; assert parser.bci() == 0; mark = null; + callee = null; } /** - * Creates a scope for parsing an intrinsic during graph builder inlining. + * Creates a scope for graph builder inlining. * * @param parser the parsing context of the (non-intrinsic) method calling the intrinsic * @param args the arguments to the call */ - IntrinsicScope(BytecodeParser parser, JavaKind[] argSlotKinds, ValueNode[] args) { + InliningScope(BytecodeParser parser, ResolvedJavaMethod callee, ValueNode[] args) { + this.callee = callee; assert !parser.parsingIntrinsic(); this.parser = parser; mark = parser.getGraph().getMark(); + JavaKind[] argSlotKinds = callee.getSignature().toParameterKinds(!callee.isStatic()); stateBefore = parser.frameState.create(parser.bci(), parser.getNonIntrinsicAncestor(), false, argSlotKinds, args); } @Override public void close() { - IntrinsicContext intrinsic = parser.intrinsicContext; - if (intrinsic != null && intrinsic.isPostParseInlined()) { - return; - } - - processPlaceholderFrameStates(intrinsic); + processPlaceholderFrameStates(false); } /** * Fixes up the {@linkplain BytecodeFrame#isPlaceholderBci(int) placeholder} frame states * added to the graph while parsing/inlining the intrinsic for which this object exists. */ - private void processPlaceholderFrameStates(IntrinsicContext intrinsic) { + protected void processPlaceholderFrameStates(boolean isCompilationRoot) { StructuredGraph graph = parser.getGraph(); graph.getDebug().dump(DebugContext.DETAILED_LEVEL, graph, "Before processPlaceholderFrameStates in %s", parser.method); - boolean sawInvalidFrameState = false; for (Node node : graph.getNewNodes(mark)) { if (node instanceof FrameState) { FrameState frameState = (FrameState) node; @@ -547,7 +548,7 @@ public class BytecodeParser implements GraphBuilderContext { if (frameState.bci == BytecodeFrame.AFTER_BCI) { if (parser.getInvokeReturnType() == null) { // A frame state in a root compiled intrinsic. - assert intrinsic.isCompilationRoot(); + assert isCompilationRoot; FrameState newFrameState = graph.add(new FrameState(BytecodeFrame.INVALID_FRAMESTATE_BCI)); frameState.replaceAndDelete(newFrameState); } else { @@ -557,7 +558,7 @@ public class BytecodeParser implements GraphBuilderContext { if (frameState.stackSize() != 0) { ValueNode returnVal = frameState.stackAt(0); if (!ReturnToCallerData.containsReturnValue(returnDataList, returnVal)) { - throw new GraalError("AFTER_BCI frame state within an intrinsic has a non-return value on the stack: %s", returnVal); + throw new GraalError("AFTER_BCI frame state within a sub-parse has a non-return value on the stack: %s", returnVal); } // Swap the top-of-stack value with the return value @@ -569,15 +570,7 @@ public class BytecodeParser implements GraphBuilderContext { newFrameState.setNodeSourcePosition(frameState.getNodeSourcePosition()); frameStateBuilder.push(returnKind, tos); } else if (returnKind != JavaKind.Void) { - // If the intrinsic returns a non-void value, then any frame - // state with an empty stack is invalid as it cannot - // be used to deoptimize to just after the call returns. - // These invalid frame states are expected to be removed - // by later compilation stages. - FrameState newFrameState = graph.add(new FrameState(BytecodeFrame.INVALID_FRAMESTATE_BCI)); - newFrameState.setNodeSourcePosition(frameState.getNodeSourcePosition()); - frameState.replaceAndDelete(newFrameState); - sawInvalidFrameState = true; + handleReturnMismatch(graph, frameState); } else { // An intrinsic for a void method. FrameState newFrameState = frameStateBuilder.create(parser.stream.nextBCI(), null); @@ -592,7 +585,7 @@ public class BytecodeParser implements GraphBuilderContext { if (stateBefore != frameState) { frameState.replaceAndDelete(stateBefore); } - } else if (frameState.bci == BytecodeFrame.AFTER_EXCEPTION_BCI) { + } else if (frameState.bci == BytecodeFrame.AFTER_EXCEPTION_BCI || (frameState.bci == BytecodeFrame.UNWIND_BCI && !callee.isSynchronized())) { // This is a frame state for the entry point to an exception // dispatcher in an intrinsic. For example, the invoke denoting // a partial intrinsic exit will have an edge to such a @@ -603,41 +596,102 @@ public class BytecodeParser implements GraphBuilderContext { // namely the exception object. assert frameState.rethrowException(); ValueNode exceptionValue = frameState.stackAt(0); - ExceptionObjectNode exceptionObject = (ExceptionObjectNode) GraphUtil.unproxify(exceptionValue); FrameStateBuilder dispatchState = parser.frameState.copy(); dispatchState.clearStack(); dispatchState.push(JavaKind.Object, exceptionValue); dispatchState.setRethrowException(true); - FrameState newFrameState = dispatchState.create(parser.bci(), exceptionObject); - frameState.replaceAndDelete(newFrameState); - newFrameState.setNodeSourcePosition(frameState.getNodeSourcePosition()); + for (Node usage : frameState.usages()) { + FrameState newFrameState = dispatchState.create(parser.bci(), (StateSplit) usage); + frameState.replaceAndDelete(newFrameState); + newFrameState.setNodeSourcePosition(frameState.getNodeSourcePosition()); + } + } else if (frameState.bci == BytecodeFrame.UNWIND_BCI) { + if (graph.getGuardsStage().allowsFloatingGuards()) { + throw GraalError.shouldNotReachHere("Cannot handle this UNWIND_BCI"); + } + // hope that by construction, there are no fixed guard after this unwind + // and before an other state split } else { - assert frameState.bci == BytecodeFrame.INVALID_FRAMESTATE_BCI; + assert frameState.bci == BytecodeFrame.INVALID_FRAMESTATE_BCI : frameState.bci; } } } } + graph.getDebug().dump(DebugContext.DETAILED_LEVEL, graph, "After processPlaceholderFrameStates in %s", parser.method); + } + + @SuppressWarnings("unused") + protected void handleReturnMismatch(StructuredGraph g, FrameState fs) { + throw GraalError.shouldNotReachHere("Unexpected return kind mismatch in " + parser.method + " at FS " + fs); + } + } + + static class IntrinsicScope extends InliningScope { + boolean sawInvalidFrameState; + + IntrinsicScope(BytecodeParser parser) { + super(parser); + } + + IntrinsicScope(BytecodeParser parser, ResolvedJavaMethod callee, ValueNode[] args) { + super(parser, callee, args); + } + + @Override + public void close() { + IntrinsicContext intrinsic = parser.intrinsicContext; + boolean isRootCompilation; + if (intrinsic != null) { + if (intrinsic.isPostParseInlined()) { + return; + } + isRootCompilation = intrinsic.isCompilationRoot(); + } else { + isRootCompilation = false; + } + processPlaceholderFrameStates(isRootCompilation); if (sawInvalidFrameState) { JavaKind returnKind = parser.getInvokeReturnType().getJavaKind(); FrameStateBuilder frameStateBuilder = parser.frameState; ValueNode returnValue = frameStateBuilder.pop(returnKind); + StructuredGraph graph = parser.lastInstr.graph(); StateSplitProxyNode proxy = graph.add(new StateSplitProxyNode(returnValue)); parser.lastInstr.setNext(proxy); frameStateBuilder.push(returnKind, proxy); proxy.setStateAfter(parser.createFrameState(parser.stream.nextBCI(), proxy)); parser.lastInstr = proxy; } - graph.getDebug().dump(DebugContext.DETAILED_LEVEL, graph, "After processPlaceholderFrameStates in %s", parser.method); + } + + @Override + protected void handleReturnMismatch(StructuredGraph g, FrameState fs) { + // If the intrinsic returns a non-void value, then any frame + // state with an empty stack is invalid as it cannot + // be used to deoptimize to just after the call returns. + // These invalid frame states are expected to be removed + // by later compilation stages. + FrameState newFrameState = g.add(new FrameState(BytecodeFrame.INVALID_FRAMESTATE_BCI)); + newFrameState.setNodeSourcePosition(fs.getNodeSourcePosition()); + fs.replaceAndDelete(newFrameState); + sawInvalidFrameState = true; } } private static class Target { - FixedNode fixed; - FrameStateBuilder state; + final FixedNode entry; + final FixedNode originalEntry; + final FrameStateBuilder state; - Target(FixedNode fixed, FrameStateBuilder state) { - this.fixed = fixed; + Target(FixedNode entry, FrameStateBuilder state) { + this.entry = entry; this.state = state; + this.originalEntry = null; + } + + Target(FixedNode entry, FrameStateBuilder state, FixedNode originalEntry) { + this.entry = entry; + this.state = state; + this.originalEntry = originalEntry; } } @@ -1748,7 +1802,7 @@ public class BytecodeParser implements GraphBuilderContext { // edge. Finally, we know that this intrinsic is parsed for late inlining, // so the bci must be set to unknown, so that the inliner patches it later. assert intrinsicContext.isPostParseInlined(); - invokeBci = BytecodeFrame.UNKNOWN_BCI; + invokeBci = UNKNOWN_BCI; profile = null; edgeAction = graph.method().getAnnotation(Snippet.class) == null ? ExceptionEdgeAction.INCLUDE_AND_HANDLE : ExceptionEdgeAction.OMIT; } @@ -2064,7 +2118,14 @@ public class BytecodeParser implements GraphBuilderContext { AbstractBeginNode intrinsicBranch = graph.add(new BeginNode()); AbstractBeginNode nonIntrinsicBranch = graph.add(new BeginNode()); - append(new IfNode(compare, intrinsicBranch, nonIntrinsicBranch, FAST_PATH_PROBABILITY)); + // In the adjustment above, we filter out receiver types that select the intrinsic as + // virtual call target. This means the recorded types in the adjusted profile will + // definitely not call into the intrinsic. Note that the following branch probability is + // still not precise -- the previously-not-recorded receiver types in the original + // profile might or might not call into the intrinsic. Yet we accumulate them into the + // probability of the intrinsic branch, assuming that the not-recorded types will only + // be a small fraction. + append(new IfNode(compare, intrinsicBranch, nonIntrinsicBranch, profile != null ? profile.getNotRecordedProbability() : LIKELY_PROBABILITY)); lastInstr = intrinsicBranch; return new IntrinsicGuard(currentLastInstr, intrinsicReceiver, mark, nonIntrinsicBranch, profile); } else { @@ -2223,7 +2284,7 @@ public class BytecodeParser implements GraphBuilderContext { for (InlineInvokePlugin plugin : graphBuilderConfig.getPlugins().getInlineInvokePlugins()) { InlineInfo inlineInfo = plugin.shouldInlineInvoke(this, targetMethod, args); if (inlineInfo != null) { - if (inlineInfo.getMethodToInline() != null) { + if (inlineInfo.allowsInlining()) { if (inline(targetMethod, inlineInfo.getMethodToInline(), inlineInfo.getIntrinsicBytecodeProvider(), args)) { return SUCCESSFULLY_INLINED; } @@ -2277,6 +2338,134 @@ public class BytecodeParser implements GraphBuilderContext { return false; } + /** + * Inline a method substitution graph. This is necessary for libgraal as substitutions only + * exist as encoded graphs and can't be parsed directly into the caller. + */ + @Override + @SuppressWarnings("try") + public boolean intrinsify(ResolvedJavaMethod targetMethod, StructuredGraph substituteGraph, InvocationPlugin.Receiver receiver, ValueNode[] args) { + if (receiver != null) { + receiver.get(); + } + + InvokeWithExceptionNode withException = null; + FixedWithNextNode replacee = lastInstr; + try (DebugContext.Scope a = debug.scope("instantiate", substituteGraph)) { + // Inline the snippet nodes, replacing parameters with the given args in the process + StartNode entryPointNode = substituteGraph.start(); + FixedNode firstCFGNode = entryPointNode.next(); + StructuredGraph replaceeGraph = replacee.graph(); + Mark mark = replaceeGraph.getMark(); + try (InliningScope inlineScope = new IntrinsicScope(this, targetMethod, args)) { + + EconomicMap replacementsMap = EconomicMap.create(Equivalence.IDENTITY); + for (ParameterNode param : substituteGraph.getNodes().filter(ParameterNode.class)) { + replacementsMap.put(param, args[param.index()]); + } + replacementsMap.put(entryPointNode, AbstractBeginNode.prevBegin(replacee)); + + debug.dump(DebugContext.DETAILED_LEVEL, replaceeGraph, "Before inlining method substitution %s", substituteGraph.method()); + UnmodifiableEconomicMap duplicates = inlineMethodSubstitution(replaceeGraph, substituteGraph, replacementsMap); + + FixedNode firstCFGNodeDuplicate = (FixedNode) duplicates.get(firstCFGNode); + replacee.setNext(firstCFGNodeDuplicate); + debug.dump(DebugContext.DETAILED_LEVEL, replaceeGraph, "After inlining method substitution %s", substituteGraph.method()); + + // Handle partial intrinsic exits + for (Node node : graph.getNewNodes(mark)) { + if (node instanceof Invoke) { + Invoke invoke = (Invoke) node; + if (invoke.bci() == BytecodeFrame.UNKNOWN_BCI) { + invoke.replaceBci(bci()); + } + if (node instanceof InvokeWithExceptionNode) { + // The graphs for MethodSubsitutions are produced assuming that + // exceptions + // must be dispatched. If the calling context doesn't want exception + // then + // convert back into a normal InvokeNode. + assert withException == null : "only one invoke expected"; + withException = (InvokeWithExceptionNode) node; + BytecodeParser intrinsicCallSiteParser = getNonIntrinsicAncestor(); + if (intrinsicCallSiteParser != null && intrinsicCallSiteParser.getActionForInvokeExceptionEdge(null) == ExceptionEdgeAction.OMIT) { + InvokeNode newInvoke = graph.add(new InvokeNode(withException)); + newInvoke.setStateDuring(withException.stateDuring()); + newInvoke.setStateAfter(withException.stateAfter()); + withException.killExceptionEdge(); + AbstractBeginNode next = withException.killKillingBegin(); + FixedWithNextNode pred = (FixedWithNextNode) withException.predecessor(); + pred.setNext(newInvoke); + withException.setNext(null); + newInvoke.setNext(next); + withException.replaceAndDelete(newInvoke); + } else { + // Disconnnect exception edge + withException.killExceptionEdge(); + } + } + } else if (node instanceof ForeignCallNode) { + ForeignCallNode call = (ForeignCallNode) node; + if (call.getBci() == BytecodeFrame.UNKNOWN_BCI) { + call.setBci(bci()); + if (call.stateAfter() != null && call.stateAfter().bci == BytecodeFrame.INVALID_FRAMESTATE_BCI) { + call.setStateAfter(inlineScope.stateBefore); + } + } + } + } + + ArrayList calleeReturnDataList = new ArrayList<>(); + for (ReturnNode n : substituteGraph.getNodes().filter(ReturnNode.class)) { + ReturnNode returnNode = (ReturnNode) duplicates.get(n); + FixedWithNextNode predecessor = (FixedWithNextNode) returnNode.predecessor(); + calleeReturnDataList.add(new ReturnToCallerData(returnNode.result(), predecessor)); + predecessor.setNext(null); + returnNode.safeDelete(); + } + + // Merge multiple returns + processCalleeReturn(targetMethod, inlineScope, calleeReturnDataList); + + // Exiting this scope causes processing of the placeholder frame states. + } + + if (withException != null && withException.isAlive()) { + // Connect exception edge into main graph + AbstractBeginNode exceptionEdge = handleException(null, bci(), false); + withException.setExceptionEdge(exceptionEdge); + } + + debug.dump(DebugContext.DETAILED_LEVEL, replaceeGraph, "After lowering %s with %s", replacee, this); + return true; + } catch (Throwable t) { + throw debug.handle(t); + } + } + + private static UnmodifiableEconomicMap inlineMethodSubstitution(StructuredGraph replaceeGraph, StructuredGraph snippet, + EconomicMap replacementsMap) { + try (InliningLog.UpdateScope scope = replaceeGraph.getInliningLog().openUpdateScope((oldNode, newNode) -> { + InliningLog log = replaceeGraph.getInliningLog(); + if (oldNode == null) { + log.trackNewCallsite(newNode); + } + })) { + StartNode entryPointNode = snippet.start(); + ArrayList nodes = new ArrayList<>(snippet.getNodeCount()); + for (Node node : snippet.getNodes()) { + if (node != entryPointNode && node != entryPointNode.stateAfter()) { + nodes.add(node); + } + } + UnmodifiableEconomicMap duplicates = replaceeGraph.addDuplicates(nodes, snippet, snippet.getNodeCount(), replacementsMap); + if (scope != null) { + replaceeGraph.getInliningLog().addLog(duplicates, snippet.getInliningLog()); + } + return duplicates; + } + } + @Override public boolean intrinsify(BytecodeProvider intrinsicBytecodeProvider, ResolvedJavaMethod targetMethod, ResolvedJavaMethod substitute, InvocationPlugin.Receiver receiver, ValueNode[] args) { if (receiver != null) { @@ -2441,7 +2630,8 @@ public class BytecodeParser implements GraphBuilderContext { FixedWithNextNode calleeBeforeUnwindNode = null; ValueNode calleeUnwindValue = null; - try (IntrinsicScope s = calleeIntrinsicContext != null && !parsingIntrinsic() ? new IntrinsicScope(this, targetMethod.getSignature().toParameterKinds(!targetMethod.isStatic()), args) : null) { + try (InliningScope s = parsingIntrinsic() ? null : (calleeIntrinsicContext != null ? new IntrinsicScope(this, targetMethod, args) + : new InliningScope(this, targetMethod, args))) { BytecodeParser parser = graphBuilderInstance.createBytecodeParser(graph, this, targetMethod, INVOCATION_ENTRY_BCI, calleeIntrinsicContext); FrameStateBuilder startFrameState = new FrameStateBuilder(parser, parser.code, graph, graphBuilderConfig.retainLocalVariables()); if (!targetMethod.isStatic()) { @@ -2450,35 +2640,9 @@ public class BytecodeParser implements GraphBuilderContext { startFrameState.initializeFromArgumentsArray(args); parser.build(this.lastInstr, startFrameState); - if (parser.returnDataList == null) { - /* Callee does not return. */ - lastInstr = null; - } else { - ValueNode calleeReturnValue; - MergeNode returnMergeNode = null; - if (s != null) { - s.returnDataList = parser.returnDataList; - } - if (parser.returnDataList.size() == 1) { - /* Callee has a single return, we can continue parsing at that point. */ - ReturnToCallerData singleReturnData = parser.returnDataList.get(0); - lastInstr = singleReturnData.beforeReturnNode; - calleeReturnValue = singleReturnData.returnValue; - } else { - assert parser.returnDataList.size() > 1; - /* Callee has multiple returns, we need to insert a control flow merge. */ - returnMergeNode = graph.add(new MergeNode()); - calleeReturnValue = ValueMergeUtil.mergeValueProducers(returnMergeNode, parser.returnDataList, returnData -> returnData.beforeReturnNode, returnData -> returnData.returnValue); - } + List calleeReturnDataList = parser.returnDataList; - if (calleeReturnValue != null) { - frameState.push(targetMethod.getSignature().getReturnKind().getStackKind(), calleeReturnValue); - } - if (returnMergeNode != null) { - returnMergeNode.setStateAfter(createFrameState(stream.nextBCI(), returnMergeNode)); - lastInstr = finishInstruction(returnMergeNode, frameState); - } - } + processCalleeReturn(targetMethod, s, calleeReturnDataList); /* * Propagate any side effects into the caller when parsing intrinsics. */ @@ -2509,6 +2673,40 @@ public class BytecodeParser implements GraphBuilderContext { } } + private ValueNode processCalleeReturn(ResolvedJavaMethod targetMethod, InliningScope inliningScope, List calleeReturnDataList) { + if (calleeReturnDataList == null) { + /* Callee does not return. */ + lastInstr = null; + } else { + ValueNode calleeReturnValue; + MergeNode returnMergeNode = null; + if (inliningScope != null) { + inliningScope.returnDataList = calleeReturnDataList; + } + if (calleeReturnDataList.size() == 1) { + /* Callee has a single return, we can continue parsing at that point. */ + ReturnToCallerData singleReturnData = calleeReturnDataList.get(0); + lastInstr = singleReturnData.beforeReturnNode; + calleeReturnValue = singleReturnData.returnValue; + } else { + assert calleeReturnDataList.size() > 1; + /* Callee has multiple returns, we need to insert a control flow merge. */ + returnMergeNode = graph.add(new MergeNode()); + calleeReturnValue = ValueMergeUtil.mergeValueProducers(returnMergeNode, calleeReturnDataList, returnData -> returnData.beforeReturnNode, returnData -> returnData.returnValue); + } + + if (calleeReturnValue != null) { + frameState.push(targetMethod.getSignature().getReturnKind().getStackKind(), calleeReturnValue); + } + if (returnMergeNode != null) { + returnMergeNode.setStateAfter(createFrameState(stream.nextBCI(), returnMergeNode)); + lastInstr = finishInstruction(returnMergeNode, frameState); + } + return calleeReturnValue; + } + return null; + } + public MethodCallTargetNode createMethodCallTarget(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, StampPair returnStamp, JavaTypeProfile profile) { return new MethodCallTargetNode(invokeKind, targetMethod, args, returnStamp, profile); } @@ -2566,7 +2764,7 @@ public class BytecodeParser implements GraphBuilderContext { /* * This must be the return value from within a partial intrinsification. */ - assert !BytecodeFrame.isPlaceholderBci(stateAfter.bci); + assert !BytecodeFrame.isPlaceholderBci(stateAfter.bci) || intrinsicContext.isDeferredInvoke(stateSplit); } } else { assert stateAfter == null; @@ -2628,6 +2826,9 @@ public class BytecodeParser implements GraphBuilderContext { append(new FinalFieldBarrierNode(entryBCI == INVOCATION_ENTRY_BCI ? originalReceiver : null)); } synchronizedEpilogue(BytecodeFrame.AFTER_BCI, x, kind); + if (method.isSynchronized()) { + finishPrepare(lastInstr, BytecodeFrame.AFTER_BCI, frameState); + } } protected MonitorEnterNode createMonitorEnterNode(ValueNode x, MonitorIdNode monitorId) { @@ -2641,7 +2842,7 @@ public class BytecodeParser implements GraphBuilderContext { monitorEnter.setStateAfter(createFrameState(bci, monitorEnter)); } - protected void genMonitorExit(ValueNode x, ValueNode escapedReturnValue, int bci) { + protected void genMonitorExit(ValueNode x, ValueNode escapedValue, int bci) { if (frameState.lockDepth(false) == 0) { throw bailout("unbalanced monitors: too many exits"); } @@ -2650,7 +2851,7 @@ public class BytecodeParser implements GraphBuilderContext { if (GraphUtil.originalValue(lockedObject) != GraphUtil.originalValue(x)) { throw bailout(String.format("unbalanced monitors: mismatch at monitorexit, %s != %s", GraphUtil.originalValue(x), GraphUtil.originalValue(lockedObject))); } - MonitorExitNode monitorExit = append(new MonitorExitNode(lockedObject, monitorId, escapedReturnValue)); + MonitorExitNode monitorExit = append(new MonitorExitNode(lockedObject, monitorId, escapedValue)); monitorExit.setStateAfter(createFrameState(bci, monitorExit)); } @@ -2760,7 +2961,7 @@ public class BytecodeParser implements GraphBuilderContext { } } - private Target checkLoopExit(FixedNode target, BciBlock targetBlock, FrameStateBuilder state) { + private Target checkLoopExit(Target target, BciBlock targetBlock) { if (currentBlock != null) { long exits = currentBlock.loops & ~targetBlock.loops; if (exits != 0) { @@ -2790,7 +2991,7 @@ public class BytecodeParser implements GraphBuilderContext { if (targetBlock instanceof ExceptionDispatchBlock) { bci = ((ExceptionDispatchBlock) targetBlock).deoptBci; } - FrameStateBuilder newState = state.copy(); + FrameStateBuilder newState = target.state.copy(); for (BciBlock loop : exitLoops) { LoopBeginNode loopBegin = (LoopBeginNode) getFirstInstruction(loop); LoopExitNode loopExit = graph.add(new LoopExitNode(loopBegin)); @@ -2807,11 +3008,49 @@ public class BytecodeParser implements GraphBuilderContext { loopExit.setStateAfter(newState.create(bci, loopExit)); } - lastLoopExit.setNext(target); - return new Target(firstLoopExit, newState); + // Fortify: Suppress Null Dereference false positive + assert lastLoopExit != null; + + if (target.originalEntry == null) { + lastLoopExit.setNext(target.entry); + return new Target(firstLoopExit, newState, target.entry); + } else { + target.originalEntry.replaceAtPredecessor(firstLoopExit); + lastLoopExit.setNext(target.originalEntry); + return new Target(target.entry, newState, target.originalEntry); + } } } - return new Target(target, state); + return target; + } + + private Target checkUnwind(FixedNode target, BciBlock targetBlock, FrameStateBuilder state) { + if (targetBlock != blockMap.getUnwindBlock()) { + return new Target(target, state); + } + FrameStateBuilder newState = state; + newState = newState.copy(); + newState.setRethrowException(false); + if (!method.isSynchronized()) { + return new Target(target, newState); + } + FixedWithNextNode originalLast = lastInstr; + FrameStateBuilder originalState = frameState; + BeginNode holder = new BeginNode(); + lastInstr = graph.add(holder); + frameState = newState; + assert frameState.stackSize() == 1; + ValueNode exception = frameState.peekObject(); + synchronizedEpilogue(BytecodeFrame.AFTER_EXCEPTION_BCI, exception, JavaKind.Void); + lastInstr.setNext(target); + + lastInstr = originalLast; + frameState = originalState; + + FixedNode result = holder.next(); + holder.setNext(null); + holder.safeDelete(); + return new Target(result, newState, target); } private FrameStateBuilder getEntryState(BciBlock block) { @@ -2856,29 +3095,23 @@ public class BytecodeParser implements GraphBuilderContext { * placeholder that later can be replaced with a MergeNode when we see this block * again. */ - FixedNode targetNode; if (canReuseInstruction && (block.getPredecessorCount() == 1 || !controlFlowSplit) && !block.isLoopHeader() && (currentBlock.loops & ~block.loops) == 0) { setFirstInstruction(block, lastInstr); lastInstr = null; } else { setFirstInstruction(block, graph.add(new BeginNode())); } - targetNode = getFirstInstruction(block); - Target target = checkLoopExit(targetNode, block, state); - FixedNode result = target.fixed; + Target target = checkUnwind(getFirstInstruction(block), block, state); + target = checkLoopExit(target, block); + FixedNode result = target.entry; FrameStateBuilder currentEntryState = target.state == state ? (canReuseState ? state : state.copy()) : target.state; setEntryState(block, currentEntryState); currentEntryState.clearNonLiveLocals(block, liveness, true); - debug.log("createTarget %s: first visit, result: %s", block, targetNode); + debug.log("createTarget %s: first visit, result: %s", block, result); return result; } - // We already saw this block before, so we have to merge states. - if (!getEntryState(block).isCompatibleWith(state)) { - throw bailout(String.format("stacks do not match on merge from %d into %s; bytecodes would not verify:%nexpect: %s%nactual: %s", bci(), block, getEntryState(block), state)); - } - if (getFirstInstruction(block) instanceof LoopBeginNode) { assert (block.isLoopHeader() && currentBlock.getId() >= block.getId()) : "must be backward branch"; /* @@ -2887,8 +3120,8 @@ public class BytecodeParser implements GraphBuilderContext { */ LoopBeginNode loopBegin = (LoopBeginNode) getFirstInstruction(block); LoopEndNode loopEnd = graph.add(new LoopEndNode(loopBegin)); - Target target = checkLoopExit(loopEnd, block, state); - FixedNode result = target.fixed; + Target target = checkLoopExit(new Target(loopEnd, state), block); + FixedNode result = target.entry; getEntryState(block).merge(loopBegin, target.state); debug.log("createTarget %s: merging backward branch to loop header %s, result: %s", block, loopBegin, result); @@ -2927,8 +3160,8 @@ public class BytecodeParser implements GraphBuilderContext { // The EndNode for the newly merged edge. EndNode newEnd = graph.add(new EndNode()); - Target target = checkLoopExit(newEnd, block, state); - FixedNode result = target.fixed; + Target target = checkLoopExit(checkUnwind(newEnd, block, state), block); + FixedNode result = target.entry; getEntryState(block).merge(mergeNode, target.state); mergeNode.addForwardEnd(newEnd); @@ -2993,13 +3226,15 @@ public class BytecodeParser implements GraphBuilderContext { } private void handleUnwindBlock(ExceptionDispatchBlock block) { + if (frameState.lockDepth(false) != 0) { + throw bailout("unbalanced monitors: too few exits exiting frame"); + } + assert !frameState.rethrowException(); + finishPrepare(lastInstr, block.deoptBci, frameState); if (parent == null) { - finishPrepare(lastInstr, block.deoptBci, frameState); - frameState.setRethrowException(false); createUnwind(); } else { - ValueNode exception = frameState.pop(JavaKind.Object); - this.unwindValue = exception; + this.unwindValue = frameState.pop(JavaKind.Object); this.beforeUnwindNode = this.lastInstr; } } @@ -3018,7 +3253,6 @@ public class BytecodeParser implements GraphBuilderContext { @SuppressWarnings("try") private void createUnwind() { assert frameState.stackSize() == 1 : frameState; - synchronizedEpilogue(BytecodeFrame.AFTER_EXCEPTION_BCI, null, null); try (DebugCloseable context = openNodeContext(frameState, BytecodeFrame.UNWIND_BCI)) { ValueNode exception = frameState.pop(JavaKind.Object); append(new UnwindNode(exception)); @@ -3029,12 +3263,13 @@ public class BytecodeParser implements GraphBuilderContext { private void synchronizedEpilogue(int bci, ValueNode currentReturnValue, JavaKind currentReturnValueKind) { try (DebugCloseable context = openNodeContext(frameState, bci)) { if (method.isSynchronized()) { - if (currentReturnValue != null) { + if (currentReturnValueKind != JavaKind.Void) { + // we are making a state that should look like the state after the return: + // push the return value on the stack frameState.push(currentReturnValueKind, currentReturnValue); } genMonitorExit(methodSynchronizedObject, currentReturnValue, bci); assert !frameState.rethrowException(); - finishPrepare(lastInstr, bci, frameState); } if (frameState.lockDepth(false) != 0) { throw bailout("unbalanced monitors: too few exits exiting frame"); @@ -4557,7 +4792,7 @@ public class BytecodeParser implements GraphBuilderContext { */ if (resolvedField.isSynthetic() && resolvedField.getName().startsWith("$assertionsDisabled")) { if (parsingIntrinsic()) { - throw new GraalError("Cannot use an assertion within the context of an intrinsic."); + throw new GraalError("Cannot use an assertion within the context of an intrinsic: " + resolvedField); } else if (graphBuilderConfig.omitAssertions()) { frameState.push(field.getJavaKind(), ConstantNode.forBoolean(true, graph)); return; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/FrameStateBuilder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/FrameStateBuilder.java index 07ffe35ff3a..02ef27057da 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/FrameStateBuilder.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/FrameStateBuilder.java @@ -48,6 +48,7 @@ import org.graalvm.compiler.core.common.PermanentBailoutException; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.core.common.type.StampPair; import org.graalvm.compiler.debug.DebugContext; +import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.NodeSourcePosition; import org.graalvm.compiler.java.BciBlockMapping.BciBlock; import org.graalvm.compiler.nodeinfo.Verbosity; @@ -390,6 +391,10 @@ public final class FrameStateBuilder implements SideEffectsState { assert code.equals(other.code) && graph == other.graph && localsSize() == other.localsSize() : "Can only compare frame states of the same method"; assert lockedObjects.length == monitorIds.length && other.lockedObjects.length == other.monitorIds.length : "mismatch between lockedObjects and monitorIds"; + if (rethrowException != other.rethrowException) { + return false; + } + if (stackSize() != other.stackSize()) { return false; } @@ -413,7 +418,7 @@ public final class FrameStateBuilder implements SideEffectsState { } public void merge(AbstractMergeNode block, FrameStateBuilder other) { - assert isCompatibleWith(other); + GraalError.guarantee(isCompatibleWith(other), "stacks do not match on merge; bytecodes would not verify:%nexpect: %s%nactual: %s", block, other); for (int i = 0; i < localsSize(); i++) { locals[i] = merge(locals[i], other.locals[i], block); @@ -803,6 +808,12 @@ public final class FrameStateBuilder implements SideEffectsState { return result; } + public ValueNode peekObject() { + ValueNode x = xpeek(); + assert verifyKind(JavaKind.Object, x); + return x; + } + /** * Pop the specified number of slots off of this stack and return them as an array of * instructions. diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/jdk/System_currentTimeMillis02.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/jdk/System_currentTimeMillis02.java index b7b99c9dc4c..280a43a177f 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/jdk/System_currentTimeMillis02.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/jdk/System_currentTimeMillis02.java @@ -24,34 +24,28 @@ package org.graalvm.compiler.jtt.jdk; +import org.graalvm.compiler.api.directives.GraalDirectives; +import org.graalvm.compiler.jtt.JTTTest; import org.junit.Test; -import org.graalvm.compiler.jtt.JTTTest; - -/* +/** + * Checks that the time between 2 successive calls to {@link System#currentTimeMillis()} is less + * than 100 milliseconds at least once in 5_000_000 attempts. */ public class System_currentTimeMillis02 extends JTTTest { - static void m(long[] times) { - times[1] = System.currentTimeMillis() - times[0]; - } - public static boolean test() { - long[] times = new long[2]; // { start, delta } - times[0] = System.currentTimeMillis(); - times[1] = 0; - // force compilation: - for (int i = 0; i < 5000; i++) { - m(times); + for (int i = 0; i < 5_000_000; i++) { + long elapsed = System.currentTimeMillis() - System.currentTimeMillis(); + if (elapsed < 100) { + return true; + } } - times[0] = System.currentTimeMillis(); - times[1] = 0; - for (int i = 0; times[1] == 0 && i < 5000000; i++) { - m(times); - // do nothing. + if (!GraalDirectives.inCompiledCode()) { + // We don't care about the result for the interpreter, C1 or C2 + return true; } - // better get at least 100 millisecond resolution. - return times[1] >= 1 && times[1] < 100; + return false; } @Test diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/jdk/System_nanoTime02.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/jdk/System_nanoTime02.java index e6744da95c3..207de820e8c 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/jdk/System_nanoTime02.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/jdk/System_nanoTime02.java @@ -24,38 +24,32 @@ package org.graalvm.compiler.jtt.jdk; +import org.graalvm.compiler.api.directives.GraalDirectives; +import org.graalvm.compiler.jtt.JTTTest; import org.junit.Test; -import org.graalvm.compiler.jtt.JTTTest; - -/* +/** + * Checks that the time between 2 successive calls to {@link System#nanoTime()} is less than 30 + * microseconds at least once in 5_000_000 attempts. */ public class System_nanoTime02 extends JTTTest { public static boolean test() { - long minDelta = Long.MAX_VALUE; - - // the first call to System.nanoTime might take a long time due to call resolution - for (int c = 0; c < 10; c++) { - long start = System.nanoTime(); - long delta = 0; - int i; - for (i = 0; delta == 0 && i < 50000; i++) { - delta = System.nanoTime() - start; - // do nothing. - } - if (delta < minDelta) { - minDelta = delta; + for (int i = 0; i < 5_000_000; i++) { + long delta = System.nanoTime() - System.nanoTime(); + if (delta < 30_000) { + return true; } } - - // better get at least 30 microsecond resolution. - return minDelta > 1 && minDelta < 30000; + if (!GraalDirectives.inCompiledCode()) { + // We don't care about the result for the interpreter, C1 or C2 + return true; + } + return false; } @Test public void run0() throws Throwable { runTest("test"); } - } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/lang/Math_log10.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/lang/Math_log10.java index f94979a7408..c4c11a2ba5d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/lang/Math_log10.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/lang/Math_log10.java @@ -182,7 +182,7 @@ public final class Math_log10 extends JTTTest { // Test for gross inaccuracy by comparing to log; should be // within a few ulps of log(x)/log(10) Random rand = new java.util.Random(0L); - for (int i = 0; i < 10000; i++) { + for (int i = 0; i < 500; i++) { double input = Double.longBitsToDouble(rand.nextLong()); if (!Double.isFinite(input)) { continue; // avoid testing NaN and infinite values diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/lang/UnaryMath.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/lang/UnaryMath.java index 10785fd8f93..95bf1ef1a21 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/lang/UnaryMath.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/lang/UnaryMath.java @@ -31,7 +31,7 @@ import jdk.vm.ci.meta.ResolvedJavaMethod; public abstract class UnaryMath extends JTTTest { - private static final long STEP = Long.MAX_VALUE / 1_000_000; + private static final long STEP = Long.MAX_VALUE / 100_000; /** * Tests a unary {@link Math} method on a wide range of values. diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/threads/SynchronizedLoopExit01.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/threads/SynchronizedLoopExit01.java index 31ab3a6137e..dc3b21d2b08 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/threads/SynchronizedLoopExit01.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/threads/SynchronizedLoopExit01.java @@ -44,6 +44,7 @@ public final class SynchronizedLoopExit01 extends JTTTest { protected Object object = new Object(); protected volatile boolean drained = false; protected volatile boolean someBoolean = true; + protected volatile int someInt = 3; public boolean test() { boolean b = true; @@ -63,4 +64,49 @@ public final class SynchronizedLoopExit01 extends JTTTest { runTest("test"); } + public synchronized boolean test1() { + boolean b = true; + while (!drained) { + synchronized (object) { + boolean c = b = someBoolean; + if (c || drained) { + break; + } + } + } + return b; + } + + @Test + public void run1() throws Throwable { + runTest("test1"); + } + + public synchronized boolean test2() { + boolean b = true; + while (!drained) { + synchronized (object) { + boolean c = b = someBoolean; + if (c || drained) { + break; + } + if (someInt > 0) { + throw new RuntimeException(); + } + } + if (someInt < -10) { + throw new IndexOutOfBoundsException(); + } + } + if (someInt < -5) { + throw new IllegalArgumentException(); + } + return b; + } + + @Test + public void run2() throws Throwable { + runTest("test2"); + } + } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/threads/SynchronizedParserInlineTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/threads/SynchronizedParserInlineTest.java new file mode 100644 index 00000000000..e884afdc8d7 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/threads/SynchronizedParserInlineTest.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +/* + */ + + +package org.graalvm.compiler.jtt.threads; + +import org.graalvm.compiler.jtt.JTTTest; +import org.junit.Test; + +public final class SynchronizedParserInlineTest extends JTTTest { + + private static SynchronizedParserInlineTest object = new SynchronizedParserInlineTest(); + + public static Integer test(boolean b) { + foo(object); + return b ? 42 : 1337; + } + + @BytecodeParserForceInline + public static synchronized void foo(SynchronizedParserInlineTest o) { + o.notifyAll(); + } + + @Test + public void run0() { + runTest("test", false); + } + + public static Integer test1(int b) { + return foo1(b); + } + + @BytecodeParserForceInline + public static synchronized int foo1(int b) { + if (b < 0) { + return 7777; + } else if (b > 100) { + throw new RuntimeException(); + } else { + throw new IllegalArgumentException(); + } + } + + @Test + public void run1() { + runTest("test1", -1); + runTest("test1", 1); + runTest("test1", 101); + } + + public static Integer test2(int b) { + return foo2(b); + } + + @BytecodeParserForceInline + public static int foo2(int b) { + if (b < 0) { + return 7777; + } else if (b > 100) { + throw new RuntimeException(); + } else { + throw new IllegalArgumentException(); + } + } + + @Test + public void run2() { + runTest("test2", -1); + runTest("test2", 1); + runTest("test2", 101); + } + +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64BitManipulationOp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64BitManipulationOp.java index e1d2ca2d097..37d6e3a0279 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64BitManipulationOp.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64BitManipulationOp.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,17 +24,22 @@ package org.graalvm.compiler.lir.aarch64; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; import static jdk.vm.ci.code.ValueUtil.asRegister; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.ILLEGAL; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler; +import org.graalvm.compiler.core.common.LIRKind; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.Opcode; import org.graalvm.compiler.lir.asm.CompilationResultBuilder; +import org.graalvm.compiler.lir.gen.LIRGeneratorTool; +import jdk.vm.ci.aarch64.AArch64Kind; import jdk.vm.ci.code.Register; import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.Value; /** * Bit manipulation ops for ARMv8 ISA. @@ -45,6 +50,7 @@ public class AArch64BitManipulationOp extends AArch64LIRInstruction { BSR, BSWP, CLZ, + POPCNT, } private static final LIRInstructionClass TYPE = LIRInstructionClass.create(AArch64BitManipulationOp.class); @@ -53,11 +59,14 @@ public class AArch64BitManipulationOp extends AArch64LIRInstruction { @Def protected AllocatableValue result; @Use({REG}) protected AllocatableValue input; - public AArch64BitManipulationOp(BitManipulationOpCode opcode, AllocatableValue result, AllocatableValue input) { + @Temp({REG, ILLEGAL}) protected Value temp; + + public AArch64BitManipulationOp(LIRGeneratorTool tool, BitManipulationOpCode opcode, AllocatableValue result, AllocatableValue input) { super(TYPE); this.opcode = opcode; this.result = result; this.input = input; + this.temp = BitManipulationOpCode.POPCNT == opcode ? tool.newVariable(LIRKind.value(AArch64Kind.V64_BYTE)) : Value.ILLEGAL; } @Override @@ -83,6 +92,11 @@ public class AArch64BitManipulationOp extends AArch64LIRInstruction { case BSWP: masm.rev(size, dst, src); break; + case POPCNT: + assert !Value.ILLEGAL.equals(temp) : "Auxiliary register not allocated."; + Register vreg = asRegister(temp); + masm.popcnt(size, dst, src, vreg); + break; default: throw GraalError.shouldNotReachHere(); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ControlFlow.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ControlFlow.java index 44d4f934df8..5c6764f6816 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ControlFlow.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ControlFlow.java @@ -175,7 +175,10 @@ public class AArch64ControlFlow { boolean isFarBranch; if (label.isBound()) { - isFarBranch = NumUtil.isSignedNbit(18, masm.position() - label.position()); + // The label.position() is a byte based index. The TBZ instruction has 14 bits for + // the offset and AArch64 instruction is 4 bytes aligned. So TBZ can encode 16 bits + // signed offset. + isFarBranch = !NumUtil.isSignedNbit(16, masm.position() - label.position()); } else { // Max range of tbz is +-2^13 instructions. We estimate that each LIR instruction // emits 2 AArch64 instructions on average. Thus we test for maximum 2^12 LIR diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64Move.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64Move.java index f3762230b7b..d1dfae76127 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64Move.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64Move.java @@ -497,7 +497,12 @@ public class AArch64Move { masm.fmov(32, dst, scratch); } } else { - masm.fldr(32, dst, (AArch64Address) crb.asFloatConstRef(input)); + try (ScratchRegister scr = masm.getScratchRegister()) { + Register scratch = scr.getRegister(); + crb.asFloatConstRef(input); + masm.addressOf(scratch); + masm.fldr(32, dst, AArch64Address.createBaseRegisterOnlyAddress(scratch)); + } } break; case Double: @@ -510,7 +515,12 @@ public class AArch64Move { masm.fmov(64, dst, scratch); } } else { - masm.fldr(64, dst, (AArch64Address) crb.asDoubleConstRef(input)); + try (ScratchRegister scr = masm.getScratchRegister()) { + Register scratch = scr.getRegister(); + crb.asDoubleConstRef(input); + masm.addressOf(scratch); + masm.fldr(64, dst, AArch64Address.createBaseRegisterOnlyAddress(scratch)); + } } break; case Object: diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ControlFlow.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ControlFlow.java index e4b44e443ec..84d230a918c 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ControlFlow.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ControlFlow.java @@ -299,7 +299,7 @@ public class AMD64ControlFlow { int imm32 = label.position() - jumpTablePos; masm.emitInt(imm32); } else { - label.addPatchAt(masm.position()); + label.addPatchAt(masm.position(), masm); masm.emitByte(0); // pseudo-opcode for jump table entry masm.emitShort(offsetToJumpTableBase); @@ -393,7 +393,7 @@ public class AMD64ControlFlow { masm.emitInt(imm32); } else { int offsetToJumpTableBase = masm.position() - jumpTablePos; - label.addPatchAt(masm.position()); + label.addPatchAt(masm.position(), masm); masm.emitByte(0); // pseudo-opcode for jump table entry masm.emitShort(offsetToJumpTableBase); masm.emitByte(0); // padding to make jump table entry 4 bytes wide diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/alloc/lsra/LinearScanWalker.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/alloc/lsra/LinearScanWalker.java index fc2c0a3b8c7..58bc4b155ac 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/alloc/lsra/LinearScanWalker.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/alloc/lsra/LinearScanWalker.java @@ -878,6 +878,9 @@ class LinearScanWalker extends IntervalWalker { break; } + // Fortify: Suppress Null Dereference false positive + assert reg != null; + boolean needSplit = blockPos[reg.number] <= intervalTo; int splitPos = blockPos[reg.number]; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/asm/CompilationResultBuilder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/asm/CompilationResultBuilder.java index 08e6fe6b060..14f78a0ffe4 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/asm/CompilationResultBuilder.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/asm/CompilationResultBuilder.java @@ -598,8 +598,8 @@ public class CompilationResultBuilder { if (label != null) { labelBindLirPositions.put(label, instructionPosition); } - lirPositions.put(op, instructionPosition); } + lirPositions.put(op, instructionPosition); instructionPosition++; } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/constopt/ConstantLoadOptimization.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/constopt/ConstantLoadOptimization.java index 7d6e62daa59..86eca0c09e9 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/constopt/ConstantLoadOptimization.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/constopt/ConstantLoadOptimization.java @@ -278,7 +278,8 @@ public final class ConstantLoadOptimization extends PreAllocationOptimizationPha } if (cost.getNumMaterializations() > 1 || cost.getBestCost() < tree.getBlock().getRelativeFrequency()) { - try (DebugContext.Scope s = debug.scope("CLOmodify", constTree); Indent i = debug.logAndIndent("Replacing %s = %s", tree.getVariable(), tree.getConstant().toValueString())) { + try (DebugContext.Scope s = debug.scope("CLOmodify", constTree); + Indent i = debug.isLogEnabled() ? debug.logAndIndent("Replacing %s = %s", tree.getVariable(), tree.getConstant().toValueString()) : null) { // mark original load for removal deleteInstruction(tree); constantsOptimized.increment(debug); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop.phases/src/org/graalvm/compiler/loop/phases/LoopTransformations.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop.phases/src/org/graalvm/compiler/loop/phases/LoopTransformations.java index cffec1889a2..c883da4bb62 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop.phases/src/org/graalvm/compiler/loop/phases/LoopTransformations.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop.phases/src/org/graalvm/compiler/loop/phases/LoopTransformations.java @@ -394,9 +394,14 @@ public abstract class LoopTransformations { invariantValue = switchNode.value(); controls = new ArrayList<>(); controls.add(switchNode); - } else if (switchNode.value() == invariantValue && firstSwitch.structureEquals(switchNode)) { - // Only collect switches which test the same values in the same order - controls.add(switchNode); + } else if (switchNode.value() == invariantValue) { + // Fortify: Suppress Null Dereference false positive + assert firstSwitch != null; + + if (firstSwitch.structureEquals(switchNode)) { + // Only collect switches which test the same values in the same order + controls.add(switchNode); + } } } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopEx.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopEx.java index f3a398d247a..8660303c034 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopEx.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopEx.java @@ -164,7 +164,7 @@ public class LoopEx { @Override public String toString() { - return (isCounted() ? "CountedLoop [" + counted() + "] " : "Loop ") + "(depth=" + loop().getDepth() + ") " + loopBegin(); + return (countedLoopChecked && isCounted() ? "CountedLoop [" + counted() + "] " : "Loop ") + "(depth=" + loop().getDepth() + ") " + loopBegin(); } private class InvariantPredicate implements NodePredicate { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/CallTargetNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/CallTargetNode.java index 33522114035..7d974e0121c 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/CallTargetNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/CallTargetNode.java @@ -137,4 +137,8 @@ public abstract class CallTargetNode extends ValueNode implements LIRLowerable { public void setInvokeKind(InvokeKind kind) { this.invokeKind = kind; } + + public Invoke invoke() { + return (Invoke) this.usages().first(); + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/DeoptimizingNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/DeoptimizingNode.java index 50312d35849..380609ca646 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/DeoptimizingNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/DeoptimizingNode.java @@ -28,6 +28,12 @@ import org.graalvm.compiler.nodes.spi.NodeWithState; /** * Interface implemented by nodes which may need {@linkplain FrameState deoptimization information}. + *

+ * Sub-interfaces are used to specify exactly when the deoptimization can take place: + * {@linkplain DeoptBefore before}, {@linkplain DeoptAfter after}, and/or {@linkplain DeoptDuring + * during}.
+ * Note that these sub-interfaces are not mutually exclusive so that nodes that may deoptimize at + * multiple times can be modeled. */ public interface DeoptimizingNode extends NodeWithState { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GraphDecoder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GraphDecoder.java index 636080aaa97..2c791dae0a6 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GraphDecoder.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GraphDecoder.java @@ -878,33 +878,37 @@ public class GraphDecoder { registerNode(outerScope, proxyOrderId, phiInput, true, false); replacement = phiInput; - } else if (!merge.isPhiAtMerge(existing)) { - /* Now we have two different values, so we need to create a phi node. */ - PhiNode phi; - if (proxy instanceof ValueProxyNode) { - phi = graph.addWithoutUnique(new ValuePhiNode(proxy.stamp(NodeView.DEFAULT), merge)); - } else if (proxy instanceof GuardProxyNode) { - phi = graph.addWithoutUnique(new GuardPhiNode(merge)); - } else { - throw GraalError.shouldNotReachHere(); - } - /* Add the inputs from all previous exits. */ - for (int j = 0; j < merge.phiPredecessorCount() - 1; j++) { - phi.addInput(existing); - } - /* Add the input from this exit. */ - phi.addInput(phiInput); - registerNode(outerScope, proxyOrderId, phi, true, false); - replacement = phi; - phiCreated = true; - } else { - /* Phi node has been created before, so just add the new input. */ - PhiNode phi = (PhiNode) existing; - phi.addInput(phiInput); - replacement = phi; - } + // Fortify: Suppress Null Dereference false positive + assert merge != null; + if (!merge.isPhiAtMerge(existing)) { + /* Now we have two different values, so we need to create a phi node. */ + PhiNode phi; + if (proxy instanceof ValueProxyNode) { + phi = graph.addWithoutUnique(new ValuePhiNode(proxy.stamp(NodeView.DEFAULT), merge)); + } else if (proxy instanceof GuardProxyNode) { + phi = graph.addWithoutUnique(new GuardPhiNode(merge)); + } else { + throw GraalError.shouldNotReachHere(); + } + /* Add the inputs from all previous exits. */ + for (int j = 0; j < merge.phiPredecessorCount() - 1; j++) { + phi.addInput(existing); + } + /* Add the input from this exit. */ + phi.addInput(phiInput); + registerNode(outerScope, proxyOrderId, phi, true, false); + replacement = phi; + phiCreated = true; + + } else { + /* Phi node has been created before, so just add the new input. */ + PhiNode phi = (PhiNode) existing; + phi.addInput(phiInput); + replacement = phi; + } + } proxy.replaceAtUsagesAndDelete(replacement); } @@ -1814,6 +1818,7 @@ class LoopDetector implements Runnable { } } assert loopVariableIndex != -1; + assert explosionHeadValue != null; ValuePhiNode loopVariablePhi; SortedMap dispatchTable = new TreeMap<>(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/IfNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/IfNode.java index 682219b415f..1e2c2b7cb8a 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/IfNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/IfNode.java @@ -768,6 +768,9 @@ public final class IfNode extends ControlSplitNode implements Simplifiable, LIRL removeThroughFalseBranch(tool, merge); return true; } else if (distinct == 1) { + // Fortify: Suppress Null Dereference false positive + assert singlePhi != null; + ValueNode trueValue = singlePhi.valueAt(trueEnd); ValueNode falseValue = singlePhi.valueAt(falseEnd); ValueNode conditional = canonicalizeConditionalCascade(tool, trueValue, falseValue); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/Invoke.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/Invoke.java index 1a9be662944..9e01fceabd8 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/Invoke.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/Invoke.java @@ -118,4 +118,6 @@ public interface Invoke extends StateSplit, Lowerable, MemoryCheckpoint.Single, default InvokeKind getInvokeKind() { return callTarget().invokeKind(); } + + void replaceBci(int newBci); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/InvokeNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/InvokeNode.java index 5d14709258e..7815a792e98 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/InvokeNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/InvokeNode.java @@ -55,6 +55,7 @@ import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; import org.graalvm.compiler.nodes.spi.UncheckedInterfaceProvider; import jdk.internal.vm.compiler.word.LocationIdentity; +import jdk.vm.ci.code.BytecodeFrame; import jdk.vm.ci.meta.JavaKind; /** @@ -76,7 +77,7 @@ public final class InvokeNode extends AbstractMemoryCheckpoint implements Invoke @OptionalInput ValueNode classInit; @Input(Extension) CallTargetNode callTarget; @OptionalInput(State) FrameState stateDuring; - protected final int bci; + protected int bci; protected boolean polymorphic; protected boolean useForInlining; protected final LocationIdentity identity; @@ -102,15 +103,19 @@ public final class InvokeNode extends AbstractMemoryCheckpoint implements Invoke this.identity = identity; } - public InvokeNode replaceWithNewBci(int newBci) { - InvokeNode newInvoke = graph().add(new InvokeNode(callTarget, newBci, stamp, identity)); - newInvoke.setUseForInlining(useForInlining); - newInvoke.setPolymorphic(polymorphic); - newInvoke.setStateAfter(stateAfter); - newInvoke.setStateDuring(stateDuring); - newInvoke.setClassInit(classInit); - graph().replaceFixedWithFixed(this, newInvoke); - return newInvoke; + public InvokeNode(InvokeWithExceptionNode invoke) { + super(TYPE, invoke.stamp); + this.callTarget = invoke.callTarget; + this.bci = invoke.bci; + this.polymorphic = invoke.polymorphic; + this.useForInlining = invoke.useForInlining; + this.identity = invoke.getLocationIdentity(); + } + + @Override + public void replaceBci(int newBci) { + assert BytecodeFrame.isPlaceholderBci(bci) && !BytecodeFrame.isPlaceholderBci(newBci) : "can only replace placeholder with better bci"; + bci = newBci; } @Override diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/InvokeWithExceptionNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/InvokeWithExceptionNode.java index ba68cf25557..926c302d3d2 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/InvokeWithExceptionNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/InvokeWithExceptionNode.java @@ -24,7 +24,16 @@ package org.graalvm.compiler.nodes; +import static org.graalvm.compiler.nodeinfo.InputType.Extension; +import static org.graalvm.compiler.nodeinfo.InputType.Memory; +import static org.graalvm.compiler.nodeinfo.InputType.State; +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_UNKNOWN; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_UNKNOWN; + +import java.util.Map; + import org.graalvm.compiler.core.common.type.Stamp; +import org.graalvm.compiler.debug.DebugCloseable; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeClass; import org.graalvm.compiler.nodeinfo.NodeInfo; @@ -38,13 +47,7 @@ import org.graalvm.compiler.nodes.spi.UncheckedInterfaceProvider; import org.graalvm.compiler.nodes.util.GraphUtil; import jdk.internal.vm.compiler.word.LocationIdentity; -import java.util.Map; - -import static org.graalvm.compiler.nodeinfo.InputType.Extension; -import static org.graalvm.compiler.nodeinfo.InputType.Memory; -import static org.graalvm.compiler.nodeinfo.InputType.State; -import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_UNKNOWN; -import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_UNKNOWN; +import jdk.vm.ci.code.BytecodeFrame; @NodeInfo(nameTemplate = "Invoke!#{p#targetMethod/s}", allowedUsageTypes = {Memory}, cycles = CYCLES_UNKNOWN, size = SIZE_UNKNOWN) public final class InvokeWithExceptionNode extends ControlSplitNode implements Invoke, MemoryCheckpoint.Single, LIRLowerable, UncheckedInterfaceProvider { @@ -58,7 +61,7 @@ public final class InvokeWithExceptionNode extends ControlSplitNode implements I @Input(Extension) CallTargetNode callTarget; @OptionalInput(State) FrameState stateDuring; @OptionalInput(State) FrameState stateAfter; - protected final int bci; + protected int bci; protected boolean polymorphic; protected boolean useForInlining; protected double exceptionProbability; @@ -207,20 +210,25 @@ public final class InvokeWithExceptionNode extends ControlSplitNode implements I GraphUtil.killCFG(edge); } - public void replaceWithNewBci(int newBci) { - AbstractBeginNode nextNode = next(); - AbstractBeginNode exceptionObject = exceptionEdge; - setExceptionEdge(null); - setNext(null); - InvokeWithExceptionNode repl = graph().add(new InvokeWithExceptionNode(callTarget(), exceptionObject, newBci)); - repl.setStateAfter(stateAfter); - this.setStateAfter(null); - this.replaceAtPredecessor(repl); - repl.setNext(nextNode); - boolean removed = this.callTarget().removeUsage(this); - assert removed; - this.replaceAtUsages(repl); - this.markDeleted(); + @SuppressWarnings("try") + public AbstractBeginNode killKillingBegin() { + AbstractBeginNode begin = next(); + if (begin instanceof KillingBeginNode) { + try (DebugCloseable position = begin.withNodeSourcePosition()) { + AbstractBeginNode newBegin = new BeginNode(); + graph().addAfterFixed(begin, graph().add(newBegin)); + begin.replaceAtUsages(newBegin); + graph().removeFixed(begin); + return newBegin; + } + } + return begin; + } + + @Override + public void replaceBci(int newBci) { + assert BytecodeFrame.isPlaceholderBci(bci) && !BytecodeFrame.isPlaceholderBci(newBci) : "can only replace placeholder with better bci"; + bci = newBci; } @Override diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/StructuredGraph.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/StructuredGraph.java index d88b5f4e2db..4e682f229ac 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/StructuredGraph.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/StructuredGraph.java @@ -966,7 +966,7 @@ public final class StructuredGraph extends Graph implements JavaMethodContext { */ public List getMethods() { if (methods != null) { - assert checkFrameStatesAgainstInlinedMethods(); + assert isSubstitution || checkFrameStatesAgainstInlinedMethods(); return Collections.unmodifiableList(methods); } return Collections.emptyList(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerLowerThanNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerLowerThanNode.java index c7e28991f0f..a6ff9c74462 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerLowerThanNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerLowerThanNode.java @@ -289,11 +289,17 @@ public abstract class IntegerLowerThanNode extends CompareNode { } if ((xResidue == 0 && left != null) || (yResidue == 0 && right != null)) { if (left == null) { + // Fortify: Suppress Null Dereference false positive + assert leftCst != null; + left = ConstantNode.forIntegerBits(bits, leftCst.asLong() - min); } else if (xResidue != 0) { left = AddNode.create(left, ConstantNode.forIntegerBits(bits, xResidue), view); } if (right == null) { + // Fortify: Suppress Null Dereference false positive + assert rightCst != null; + right = ConstantNode.forIntegerBits(bits, rightCst.asLong() - min); } else if (yResidue != 0) { right = AddNode.create(right, ConstantNode.forIntegerBits(bits, yResidue), view); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IsNullNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IsNullNode.java index 147e55ef4d6..07c0c85359c 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IsNullNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IsNullNode.java @@ -41,6 +41,7 @@ import org.graalvm.compiler.nodes.spi.LIRLowerable; import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; import org.graalvm.compiler.nodes.spi.Virtualizable; import org.graalvm.compiler.nodes.spi.VirtualizerTool; +import org.graalvm.compiler.nodes.type.NarrowOopStamp; import org.graalvm.compiler.nodes.type.StampTool; import jdk.vm.ci.meta.JavaConstant; @@ -67,6 +68,7 @@ public final class IsNullNode extends UnaryOpLogicNode implements LIRLowerable, public IsNullNode(ValueNode object) { this(object, JavaConstant.NULL_POINTER); + assertNonNarrow(object); } public JavaConstant nullConstant() { @@ -74,9 +76,19 @@ public final class IsNullNode extends UnaryOpLogicNode implements LIRLowerable, } public static LogicNode create(ValueNode forValue) { + assertNonNarrow(forValue); return canonicalized(null, forValue, JavaConstant.NULL_POINTER); } + public static LogicNode create(ValueNode forValue, JavaConstant nullConstant) { + assert nullConstant.isNull() : "Null constant is not null: " + nullConstant; + return canonicalized(null, forValue, nullConstant); + } + + private static void assertNonNarrow(ValueNode object) { + assert !(object.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp) : "Value to compare against null is a NarrowOop" + object; + } + @Override public void generate(NodeLIRBuilderTool gen) { // Nothing to do. diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/PointerEqualsNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/PointerEqualsNode.java index f36e437826d..cfac12f4962 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/PointerEqualsNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/PointerEqualsNode.java @@ -138,14 +138,22 @@ public class PointerEqualsNode extends CompareNode implements BinaryCommutative< } else if (forX.stamp(view).alwaysDistinct(forY.stamp(view))) { return LogicConstantNode.contradiction(); } else if (((AbstractPointerStamp) forX.stamp(view)).alwaysNull()) { - return IsNullNode.create(forY); + return nullSynonym(forY, forX); } else if (((AbstractPointerStamp) forY.stamp(view)).alwaysNull()) { - return IsNullNode.create(forX); + return nullSynonym(forX, forY); } else { return null; } } + private static LogicNode nullSynonym(ValueNode nonNullValue, ValueNode nullValue) { + if (nullValue.isConstant()) { + return IsNullNode.create(nonNullValue, nullValue.asJavaConstant()); + } else { + return IsNullNode.create(nonNullValue); + } + } + @Override public Stamp getSucceedingStampForX(boolean negated, Stamp xStamp, Stamp yStamp) { if (!negated) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/cfg/ControlFlowGraph.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/cfg/ControlFlowGraph.java index 2c4d8c6d63e..d3a58069142 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/cfg/ControlFlowGraph.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/cfg/ControlFlowGraph.java @@ -317,6 +317,9 @@ public final class ControlFlowGraph implements AbstractControlFlowGraph { dominator = ((dominator == null) ? pred : commonDominatorRaw(dominator, pred)); } } + // Fortify: Suppress Null Dereference false positive (every block apart from the first + // is guaranteed to have a predecessor) + assert dominator != null; // Set dominator. block.setDominator(dominator); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/ForeignCallNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/ForeignCallNode.java index f5fbe46534f..cfa31d329cc 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/ForeignCallNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/ForeignCallNode.java @@ -79,6 +79,13 @@ public class ForeignCallNode extends AbstractMemoryCheckpoint implements LIRLowe public static boolean intrinsify(GraphBuilderContext b, ResolvedJavaMethod targetMethod, @InjectedNodeParameter Stamp returnStamp, @InjectedNodeParameter ForeignCallsProvider foreignCalls, ForeignCallDescriptor descriptor, ValueNode... arguments) { + if (!foreignCalls.isAvailable(descriptor)) { + // When using encoded snippets a graph main contain a reference to a foreign call that's + // not actually available in the current configuration. It's assumed that further + // simplification of the graph will eliminate this call completely. + return false; + } + ForeignCallNode node = new ForeignCallNode(foreignCalls, descriptor, arguments); node.setStamp(returnStamp); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/GuardedUnsafeLoadNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/GuardedUnsafeLoadNode.java index 2c4c0ccd8ae..d8723cfac6a 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/GuardedUnsafeLoadNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/GuardedUnsafeLoadNode.java @@ -39,9 +39,9 @@ public class GuardedUnsafeLoadNode extends RawLoadNode implements GuardedNode { @OptionalInput(Guard) protected GuardingNode guard; - public GuardedUnsafeLoadNode(ValueNode object, ValueNode offset, JavaKind accessKind, LocationIdentity locationIdentity, GuardingNode guard) { + public GuardedUnsafeLoadNode(ValueNode object, ValueNode offset, JavaKind accessKind, LocationIdentity locationIdentity, ValueNode guard) { super(TYPE, object, offset, accessKind, locationIdentity); - this.guard = guard; + this.guard = (GuardingNode) guard; } @Override @@ -54,4 +54,7 @@ public class GuardedUnsafeLoadNode extends RawLoadNode implements GuardedNode { updateUsagesInterface(this.guard, guard); this.guard = guard; } + + @NodeIntrinsic + public static native Object guardedLoad(Object object, long offset, @ConstantNodeParameter JavaKind kind, @ConstantNodeParameter LocationIdentity locationIdentity, GuardingNode guard); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GeneratedInvocationPlugin.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GeneratedInvocationPlugin.java index 3c585fc184d..49cea565a6b 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GeneratedInvocationPlugin.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GeneratedInvocationPlugin.java @@ -45,6 +45,8 @@ import jdk.vm.ci.meta.ResolvedJavaType; */ public abstract class GeneratedInvocationPlugin implements InvocationPlugin { + private ResolvedJavaMethod executeMethod; + /** * Gets the class of the annotation for which this plugin was generated. */ @@ -69,23 +71,37 @@ public abstract class GeneratedInvocationPlugin implements InvocationPlugin { return true; } - if (IS_IN_NATIVE_IMAGE || IS_BUILDING_NATIVE_IMAGE) { + if (IS_IN_NATIVE_IMAGE) { // The reflection here is problematic for SVM. return true; } - MetaAccessProvider metaAccess = b.getMetaAccess(); - ResolvedJavaMethod executeMethod = metaAccess.lookupJavaMethod(getExecuteMethod()); - ResolvedJavaType thisClass = metaAccess.lookupJavaType(getClass()); - ResolvedJavaMethod thisExecuteMethod = thisClass.resolveConcreteMethod(executeMethod, thisClass); + if (b.getMethod().equals(foldAnnotatedMethod)) { + return false; + } + + ResolvedJavaMethod thisExecuteMethod = getExecutedMethod(b); if (b.getMethod().equals(thisExecuteMethod)) { // The "execute" method of this plugin is itself being compiled. In (only) this context, // the injected argument of the call to the @Fold annotated method will be non-null. + if (IS_BUILDING_NATIVE_IMAGE) { + return false; + } return true; } throw new AssertionError("must pass null to injected argument of " + foldAnnotatedMethod.format("%H.%n(%p)") + ", not " + arg); } + private ResolvedJavaMethod getExecutedMethod(GraphBuilderContext b) { + if (executeMethod == null) { + MetaAccessProvider metaAccess = b.getMetaAccess(); + ResolvedJavaMethod baseMethod = metaAccess.lookupJavaMethod(getExecuteMethod()); + ResolvedJavaType thisClass = metaAccess.lookupJavaType(getClass()); + executeMethod = thisClass.resolveConcreteMethod(baseMethod, thisClass); + } + return executeMethod; + } + private static Method getExecuteMethod() { try { return GeneratedInvocationPlugin.class.getMethod("execute", GraphBuilderContext.class, ResolvedJavaMethod.class, InvocationPlugin.Receiver.class, ValueNode[].class); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GraphBuilderContext.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GraphBuilderContext.java index 71efbdc143b..a40041ca120 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GraphBuilderContext.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GraphBuilderContext.java @@ -47,6 +47,7 @@ import org.graalvm.compiler.nodes.LogicNode; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.PiNode; import org.graalvm.compiler.nodes.StateSplit; +import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.calc.IsNullNode; import org.graalvm.compiler.nodes.calc.NarrowNode; @@ -164,6 +165,19 @@ public interface GraphBuilderContext extends GraphBuilderTool { */ boolean intrinsify(BytecodeProvider bytecodeProvider, ResolvedJavaMethod targetMethod, ResolvedJavaMethod substitute, InvocationPlugin.Receiver receiver, ValueNode[] argsIncludingReceiver); + /** + * Intrinsifies an invocation of a given method by inlining the graph of a given substitution + * method. + * + * @param targetMethod the method being intrinsified + * @param substituteGraph the intrinsic implementation + * @param receiver the receiver, or null for static methods + * @param argsIncludingReceiver the arguments with which to inline the invocation + * + * @return whether the intrinsification was successful + */ + boolean intrinsify(ResolvedJavaMethod targetMethod, StructuredGraph substituteGraph, InvocationPlugin.Receiver receiver, ValueNode[] argsIncludingReceiver); + /** * Creates a snap shot of the current frame state with the BCI of the instruction after the one * currently being parsed and assigns it to a given {@linkplain StateSplit#hasSideEffect() side diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/InlineInvokePlugin.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/InlineInvokePlugin.java index deb8bbf10a8..c3b5fdccdad 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/InlineInvokePlugin.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/InlineInvokePlugin.java @@ -47,35 +47,43 @@ public interface InlineInvokePlugin extends GraphBuilderPlugin { * Denotes a call site that must not be inlined and should be implemented by a node that * does not speculate on the call not raising an exception. */ - public static final InlineInfo DO_NOT_INLINE_WITH_EXCEPTION = new InlineInfo(null, null, null); + public static final InlineInfo DO_NOT_INLINE_WITH_EXCEPTION = new InlineInfo(); /** * Denotes a call site must not be inlined and can be implemented by a node that speculates * the call will not throw an exception. */ - public static final InlineInfo DO_NOT_INLINE_NO_EXCEPTION = new InlineInfo(null, null, null); + public static final InlineInfo DO_NOT_INLINE_NO_EXCEPTION = new InlineInfo(); /** * Denotes a call site must not be inlined and the execution should be transferred to * interpreter in case of an exception. */ - public static final InlineInfo DO_NOT_INLINE_DEOPTIMIZE_ON_EXCEPTION = new InlineInfo(null, null, null); + public static final InlineInfo DO_NOT_INLINE_DEOPTIMIZE_ON_EXCEPTION = new InlineInfo(); private final ResolvedJavaMethod methodToInline; - private final ResolvedJavaMethod originalMethod; + private final MethodSubstitutionPlugin plugin; private final BytecodeProvider intrinsicBytecodeProvider; public static InlineInfo createStandardInlineInfo(ResolvedJavaMethod methodToInline) { return new InlineInfo(methodToInline, null, null); } - public static InlineInfo createIntrinsicInlineInfo(ResolvedJavaMethod methodToInline, ResolvedJavaMethod originalMethod, BytecodeProvider intrinsicBytecodeProvider) { - return new InlineInfo(methodToInline, originalMethod, intrinsicBytecodeProvider); + public static InlineInfo createIntrinsicInlineInfo(ResolvedJavaMethod methodToInline, BytecodeProvider intrinsicBytecodeProvider) { + return new InlineInfo(methodToInline, null, intrinsicBytecodeProvider); } - private InlineInfo(ResolvedJavaMethod methodToInline, ResolvedJavaMethod originalMethod, BytecodeProvider intrinsicBytecodeProvider) { + public static InlineInfo createMethodSubstitutionInlineInfo(ResolvedJavaMethod methodToInline, MethodSubstitutionPlugin plugin) { + return new InlineInfo(methodToInline, plugin, plugin.getBytecodeProvider()); + } + + private InlineInfo() { + this(null, null, null); + } + + private InlineInfo(ResolvedJavaMethod methodToInline, MethodSubstitutionPlugin plugin, BytecodeProvider intrinsicBytecodeProvider) { this.methodToInline = methodToInline; - this.originalMethod = originalMethod; + this.plugin = plugin; this.intrinsicBytecodeProvider = intrinsicBytecodeProvider; } @@ -90,14 +98,6 @@ public interface InlineInvokePlugin extends GraphBuilderPlugin { return methodToInline != null; } - /** - * Returns the original method if this is an inline of an intrinsic, or {@code null} if the - * call site must not be inlined. - */ - public ResolvedJavaMethod getOriginalMethod() { - return originalMethod; - } - /** * Gets the provider of bytecode to be parsed for {@link #getMethodToInline()} if is is an * intrinsic for the original method (i.e., the {@code method} passed to @@ -107,6 +107,14 @@ public interface InlineInvokePlugin extends GraphBuilderPlugin { public BytecodeProvider getIntrinsicBytecodeProvider() { return intrinsicBytecodeProvider; } + + public boolean isSubstitution() { + return plugin != null; + } + + public MethodSubstitutionPlugin getPlugin() { + return plugin; + } } /** diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/IntrinsicContext.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/IntrinsicContext.java index 513daad5003..c6d5f6b74df 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/IntrinsicContext.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/IntrinsicContext.java @@ -191,6 +191,11 @@ public class IntrinsicContext { void addSideEffect(StateSplit sideEffect); } + @SuppressWarnings("unused") + public boolean isDeferredInvoke(StateSplit stateSplit) { + return false; + } + public FrameState createFrameState(StructuredGraph graph, SideEffectsState sideEffects, StateSplit forStateSplit, NodeSourcePosition sourcePosition) { assert forStateSplit != graph.start(); if (forStateSplit.hasSideEffect()) { @@ -205,12 +210,16 @@ public class IntrinsicContext { lastSideEffect.setStateAfter(invalid); } } - sideEffects.addSideEffect(forStateSplit); FrameState frameState; - if (forStateSplit instanceof ExceptionObjectNode) { - frameState = graph.add(new FrameState(AFTER_EXCEPTION_BCI, (ExceptionObjectNode) forStateSplit)); + if (isDeferredInvoke(forStateSplit)) { + frameState = graph.add(new FrameState(INVALID_FRAMESTATE_BCI)); } else { - frameState = graph.add(new FrameState(AFTER_BCI)); + sideEffects.addSideEffect(forStateSplit); + if (forStateSplit instanceof ExceptionObjectNode) { + frameState = graph.add(new FrameState(AFTER_EXCEPTION_BCI, (ExceptionObjectNode) forStateSplit)); + } else { + frameState = graph.add(new FrameState(AFTER_BCI)); + } } if (graph.trackNodeSourcePosition()) { frameState.setNodeSourcePosition(sourcePosition); @@ -243,6 +252,7 @@ public class IntrinsicContext { @Override public String toString() { - return "Intrinsic{original: " + originalMethod.format("%H.%n(%p)") + ", intrinsic: " + intrinsicMethod.format("%H.%n(%p)") + ", context: " + compilationContext + "}"; + return "Intrinsic{original: " + originalMethod.format("%H.%n(%p)") + ", intrinsic: " + (intrinsicMethod != null ? intrinsicMethod.format("%H.%n(%p)") : "null") + ", context: " + + compilationContext + "}"; } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/InvocationPlugin.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/InvocationPlugin.java index c04a5831594..4ea5f8e0740 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/InvocationPlugin.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/InvocationPlugin.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -197,6 +197,11 @@ public interface InvocationPlugin extends GraphBuilderPlugin { return apply(b, targetMethod, receiver, argsIncludingReceiver[1], argsIncludingReceiver[2], argsIncludingReceiver[3]); } else if (argsIncludingReceiver.length == 5) { return apply(b, targetMethod, receiver, argsIncludingReceiver[1], argsIncludingReceiver[2], argsIncludingReceiver[3], argsIncludingReceiver[4]); + } else if (argsIncludingReceiver.length == 6) { + return apply(b, targetMethod, receiver, argsIncludingReceiver[1], argsIncludingReceiver[2], argsIncludingReceiver[3], argsIncludingReceiver[4], argsIncludingReceiver[5]); + } else if (argsIncludingReceiver.length == 7) { + return apply(b, targetMethod, receiver, argsIncludingReceiver[1], argsIncludingReceiver[2], argsIncludingReceiver[3], argsIncludingReceiver[4], argsIncludingReceiver[5], + argsIncludingReceiver[6]); } else { return defaultHandler(b, targetMethod, receiver, argsIncludingReceiver); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/InvocationPlugins.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/InvocationPlugins.java index 0645cc89646..d3ed14f5fbc 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/InvocationPlugins.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/InvocationPlugins.java @@ -1023,18 +1023,10 @@ public class InvocationPlugins { if (parent != null) { InvocationPlugin plugin = parent.lookupInvocation(method); if (plugin != null) { - if (IS_IN_NATIVE_IMAGE && plugin instanceof MethodSubstitutionPlugin) { - // Disable method substitutions until GR-13607 - return null; - } return plugin; } } InvocationPlugin invocationPlugin = get(method); - if (IS_IN_NATIVE_IMAGE && invocationPlugin instanceof MethodSubstitutionPlugin) { - // Disable method substitutions until GR-13607 - return null; - } return invocationPlugin; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/MethodSubstitutionPlugin.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/MethodSubstitutionPlugin.java index 4fc559987e7..9328c4dffdb 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/MethodSubstitutionPlugin.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/MethodSubstitutionPlugin.java @@ -25,6 +25,8 @@ package org.graalvm.compiler.nodes.graphbuilderconf; import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; +import static org.graalvm.compiler.core.common.GraalOptions.UseEncodedGraphs; +import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.INLINE_AFTER_PARSING; import static org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins.resolveType; import java.lang.reflect.Method; @@ -35,6 +37,7 @@ import java.util.stream.Collectors; import org.graalvm.compiler.bytecode.BytecodeProvider; import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.ValueNode; import jdk.vm.ci.meta.MetaAccessProvider; @@ -115,10 +118,17 @@ public final class MethodSubstitutionPlugin implements InvocationPlugin { return bytecodeProvider; } + /** + * Gets the class in which the substitute method is declared. + */ + public Class getDeclaringClass() { + return declaringClass; + } + /** * Gets the reflection API version of the substitution method. */ - Method getJavaSubstitute() throws GraalError { + public Method getJavaSubstitute() throws GraalError { Method substituteMethod = lookupSubstitute(); int modifiers = substituteMethod.getModifiers(); if (Modifier.isAbstract(modifiers) || Modifier.isNative(modifiers)) { @@ -178,12 +188,18 @@ public final class MethodSubstitutionPlugin implements InvocationPlugin { @Override public boolean execute(GraphBuilderContext b, ResolvedJavaMethod targetMethod, InvocationPlugin.Receiver receiver, ValueNode[] argsIncludingReceiver) { - if (IS_IN_NATIVE_IMAGE) { - // these are currently unimplemented - return false; + if (IS_IN_NATIVE_IMAGE || (UseEncodedGraphs.getValue(b.getOptions()) && !b.parsingIntrinsic())) { + if (!IS_IN_NATIVE_IMAGE && UseEncodedGraphs.getValue(b.getOptions())) { + b.getReplacements().registerMethodSubstitution(this, targetMethod, INLINE_AFTER_PARSING, b.getOptions()); + } + StructuredGraph subst = b.getReplacements().getMethodSubstitution(this, targetMethod, INLINE_AFTER_PARSING, StructuredGraph.AllowAssumptions.ifNonNull(b.getAssumptions()), b.getOptions()); + if (subst == null) { + throw new GraalError("No graphs found for substitution %s", this); + } + return b.intrinsify(targetMethod, subst, receiver, argsIncludingReceiver); } - ResolvedJavaMethod subst = getSubstitute(b.getMetaAccess()); - return b.intrinsify(bytecodeProvider, targetMethod, subst, receiver, argsIncludingReceiver); + ResolvedJavaMethod substitute = getSubstitute(b.getMetaAccess()); + return b.intrinsify(bytecodeProvider, targetMethod, substitute, receiver, argsIncludingReceiver); } @Override diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/MethodCallTargetNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/MethodCallTargetNode.java index 92c9d1581d9..f0fa02f5e92 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/MethodCallTargetNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/MethodCallTargetNode.java @@ -95,10 +95,6 @@ public class MethodCallTargetNode extends CallTargetNode implements IterableNode return targetMethod().getSignature().getReturnKind(); } - public Invoke invoke() { - return (Invoke) this.usages().first(); - } - @Override public boolean verify() { assert getUsageCount() <= 1 : "call target may only be used by a single invoke"; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/MonitorExitNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/MonitorExitNode.java index 0d2cfedf767..ffa4768c4a8 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/MonitorExitNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/MonitorExitNode.java @@ -56,15 +56,15 @@ public final class MonitorExitNode extends AccessMonitorNode implements Virtuali */ @OptionalInput ValueNode escapedReturnValue; - public MonitorExitNode(ValueNode object, MonitorIdNode monitorId, ValueNode escapedReturnValue) { + public MonitorExitNode(ValueNode object, MonitorIdNode monitorId, ValueNode escapedValue) { super(TYPE, object, monitorId); - this.escapedReturnValue = escapedReturnValue; + this.escapedReturnValue = escapedValue; } /** * Return value is cleared when a synchronized method graph is inlined. */ - public void clearEscapedReturnValue() { + public void clearEscapedValue() { updateUsages(escapedReturnValue, null); this.escapedReturnValue = null; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/TypeSwitchNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/TypeSwitchNode.java index 79d4c0d6222..b49a61fa6cd 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/TypeSwitchNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/TypeSwitchNode.java @@ -188,11 +188,10 @@ public final class TypeSwitchNode extends SwitchNode implements LIRLowerable, Si } } + ArrayList oldSuccessors = new ArrayList<>(); for (int i = 0; i < blockSuccessorCount(); i++) { AbstractBeginNode successor = blockSuccessor(i); - if (!newSuccessors.contains(successor)) { - tool.deleteBranch(successor); - } + oldSuccessors.add(successor); setBlockSuccessor(i, null); } @@ -200,6 +199,13 @@ public final class TypeSwitchNode extends SwitchNode implements LIRLowerable, Si TypeSwitchNode newSwitch = graph().add(new TypeSwitchNode(value(), successorsArray, newKeys, newKeyProbabilities, newKeySuccessors, tool.getConstantReflection())); ((FixedWithNextNode) predecessor()).setNext(newSwitch); GraphUtil.killWithUnusedFloatingInputs(this); + + for (int i = 0; i < oldSuccessors.size(); i++) { + AbstractBeginNode successor = oldSuccessors.get(i); + if (!newSuccessors.contains(successor)) { + GraphUtil.killCFG(successor); + } + } } } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/DelegatingReplacements.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/DelegatingReplacements.java index ddaa22563ab..ab913eb80f1 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/DelegatingReplacements.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/DelegatingReplacements.java @@ -25,7 +25,6 @@ package org.graalvm.compiler.nodes.spi; import org.graalvm.compiler.api.replacements.SnippetTemplateCache; -import org.graalvm.compiler.bytecode.Bytecode; import org.graalvm.compiler.bytecode.BytecodeProvider; import org.graalvm.compiler.core.common.CompilationIdentifier; import org.graalvm.compiler.debug.DebugContext; @@ -33,6 +32,8 @@ import org.graalvm.compiler.graph.NodeSourcePosition; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderPlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext; +import org.graalvm.compiler.nodes.graphbuilderconf.MethodSubstitutionPlugin; import org.graalvm.compiler.options.OptionValues; import jdk.vm.ci.meta.ResolvedJavaMethod; @@ -48,8 +49,8 @@ public class DelegatingReplacements implements Replacements { } @Override - public OptionValues getOptions() { - return delegate.getOptions(); + public CoreProviders getProviders() { + return delegate.getProviders(); } @Override @@ -63,28 +64,30 @@ public class DelegatingReplacements implements Replacements { } @Override - public StructuredGraph getSnippet(ResolvedJavaMethod method, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) { - return delegate.getSnippet(method, args, trackNodeSourcePosition, replaceePosition); + public StructuredGraph getSnippet(ResolvedJavaMethod method, ResolvedJavaMethod recursiveEntry, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition, + OptionValues options) { + return delegate.getSnippet(method, recursiveEntry, args, trackNodeSourcePosition, replaceePosition, options); } @Override - public StructuredGraph getSnippet(ResolvedJavaMethod method, ResolvedJavaMethod recursiveEntry, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) { - return delegate.getSnippet(method, recursiveEntry, args, trackNodeSourcePosition, replaceePosition); + public void registerSnippet(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean trackNodeSourcePosition, OptionValues options) { + delegate.registerSnippet(method, original, receiver, trackNodeSourcePosition, options); } @Override - public void registerSnippet(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean trackNodeSourcePosition) { - delegate.registerSnippet(method, original, receiver, trackNodeSourcePosition); + public StructuredGraph getMethodSubstitution(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original, IntrinsicContext.CompilationContext context, + StructuredGraph.AllowAssumptions allowAssumptions, OptionValues options) { + return delegate.getMethodSubstitution(plugin, original, context, allowAssumptions, options); } @Override - public StructuredGraph getSubstitution(ResolvedJavaMethod method, int invokeBci, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) { - return delegate.getSubstitution(method, invokeBci, trackNodeSourcePosition, replaceePosition); + public void registerMethodSubstitution(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original, IntrinsicContext.CompilationContext context, OptionValues options) { + delegate.registerMethodSubstitution(plugin, original, context, options); } @Override - public Bytecode getSubstitutionBytecode(ResolvedJavaMethod method) { - return delegate.getSubstitutionBytecode(method); + public StructuredGraph getSubstitution(ResolvedJavaMethod method, int invokeBci, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition, OptionValues options) { + return delegate.getSubstitution(method, invokeBci, trackNodeSourcePosition, replaceePosition, options); } @Override diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/Replacements.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/Replacements.java index 705afdb438e..28be8c2e616 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/Replacements.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/Replacements.java @@ -26,7 +26,6 @@ package org.graalvm.compiler.nodes.spi; import org.graalvm.compiler.api.replacements.MethodSubstitution; import org.graalvm.compiler.api.replacements.SnippetTemplateCache; -import org.graalvm.compiler.bytecode.Bytecode; import org.graalvm.compiler.bytecode.BytecodeProvider; import org.graalvm.compiler.core.common.CompilationIdentifier; import org.graalvm.compiler.debug.DebugContext; @@ -34,7 +33,9 @@ import org.graalvm.compiler.graph.NodeSourcePosition; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderPlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext; import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.MethodSubstitutionPlugin; import org.graalvm.compiler.options.OptionValues; import jdk.vm.ci.meta.ResolvedJavaMethod; @@ -44,7 +45,7 @@ import jdk.vm.ci.meta.ResolvedJavaMethod; */ public interface Replacements { - OptionValues getOptions(); + CoreProviders getProviders(); /** * Gets the object managing the various graph builder plugins used by this object when parsing @@ -57,17 +58,6 @@ public interface Replacements { */ Class getIntrinsifyingPlugin(ResolvedJavaMethod method); - /** - * Gets the snippet graph derived from a given method. - * - * @param args arguments to the snippet if available, otherwise {@code null} - * @param trackNodeSourcePosition - * @return the snippet graph, if any, that is derived from {@code method} - */ - default StructuredGraph getSnippet(ResolvedJavaMethod method, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) { - return getSnippet(method, null, args, trackNodeSourcePosition, replaceePosition); - } - /** * Gets the snippet graph derived from a given method. * @@ -76,14 +66,36 @@ public interface Replacements { * substitutions}. * @param args arguments to the snippet if available, otherwise {@code null} * @param trackNodeSourcePosition + * @param options * @return the snippet graph, if any, that is derived from {@code method} */ - StructuredGraph getSnippet(ResolvedJavaMethod method, ResolvedJavaMethod recursiveEntry, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition); + StructuredGraph getSnippet(ResolvedJavaMethod method, ResolvedJavaMethod recursiveEntry, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition, + OptionValues options); /** * Registers a method as snippet. */ - void registerSnippet(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean trackNodeSourcePosition); + void registerSnippet(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean trackNodeSourcePosition, OptionValues options); + + /** + * Gets a graph that is a substitution for a given {@link MethodSubstitutionPlugin plugin} in + * the {@link org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext + * context}. + * + * @param plugin the plugin being substituted + * @param original the method being substituted + * @param context the kind of inlining to be performed for the substitution + * @param allowAssumptions + * @param options + * @return the method substitution graph, if any, that is derived from {@code method} + */ + StructuredGraph getMethodSubstitution(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original, IntrinsicContext.CompilationContext context, + StructuredGraph.AllowAssumptions allowAssumptions, OptionValues options); + + /** + * Registers a plugin as a substitution. + */ + void registerMethodSubstitution(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original, IntrinsicContext.CompilationContext context, OptionValues options); /** * Gets a graph that is a substitution for a given method. @@ -91,17 +103,10 @@ public interface Replacements { * @param invokeBci the call site BCI if this request is made for inlining a substitute * otherwise {@code -1} * @param trackNodeSourcePosition + * @param options * @return the graph, if any, that is a substitution for {@code method} */ - StructuredGraph getSubstitution(ResolvedJavaMethod method, int invokeBci, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition); - - /** - * Gets the substitute bytecode for a given method. - * - * @return the bytecode to substitute for {@code method} or {@code null} if there is no - * substitute bytecode for {@code method} - */ - Bytecode getSubstitutionBytecode(ResolvedJavaMethod method); + StructuredGraph getSubstitution(ResolvedJavaMethod method, int invokeBci, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition, OptionValues options); /** * Gets a graph produced from the intrinsic for a given method that can be compiled and @@ -116,7 +121,7 @@ public interface Replacements { /** * Determines if there may be a - * {@linkplain #getSubstitution(ResolvedJavaMethod, int, boolean, NodeSourcePosition) + * {@linkplain #getSubstitution(ResolvedJavaMethod, int, boolean, NodeSourcePosition, OptionValues) * substitution graph} for a given method. * * A call to {@link #getSubstitution} may still return {@code null} for {@code method} and diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/util/GraphUtil.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/util/GraphUtil.java index 0d01e7630fb..348441b0804 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/util/GraphUtil.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/util/GraphUtil.java @@ -243,10 +243,12 @@ public class GraphUtil { EconomicSet unsafeNodes = null; Graph.NodeEventScope nodeEventScope = null; OptionValues options = node.getOptions(); - if (Graph.Options.VerifyGraalGraphEdges.getValue(options)) { + boolean verifyGraalGraphEdges = Graph.Options.VerifyGraalGraphEdges.getValue(options); + boolean verifyKillCFGUnusedNodes = GraphUtil.Options.VerifyKillCFGUnusedNodes.getValue(options); + if (verifyGraalGraphEdges) { unsafeNodes = collectUnsafeNodes(node.graph()); } - if (GraphUtil.Options.VerifyKillCFGUnusedNodes.getValue(options)) { + if (verifyKillCFGUnusedNodes) { EconomicSet collectedUnusedNodes = unusedNodes = EconomicSet.create(Equivalence.IDENTITY); nodeEventScope = node.graph().trackNodeEvents(new Graph.NodeEventListener() { @Override @@ -260,12 +262,12 @@ public class GraphUtil { debug.dump(DebugContext.VERY_DETAILED_LEVEL, node.graph(), "Before killCFG %s", node); killCFGInner(node); debug.dump(DebugContext.VERY_DETAILED_LEVEL, node.graph(), "After killCFG %s", node); - if (Graph.Options.VerifyGraalGraphEdges.getValue(options)) { + if (verifyGraalGraphEdges) { EconomicSet newUnsafeNodes = collectUnsafeNodes(node.graph()); newUnsafeNodes.removeAll(unsafeNodes); assert newUnsafeNodes.isEmpty() : "New unsafe nodes: " + newUnsafeNodes; } - if (GraphUtil.Options.VerifyKillCFGUnusedNodes.getValue(options)) { + if (verifyKillCFGUnusedNodes) { nodeEventScope.close(); Iterator iterator = unusedNodes.iterator(); while (iterator.hasNext()) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/ConditionalEliminationPhase.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/ConditionalEliminationPhase.java index b12e035862b..fff9826b1a7 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/ConditionalEliminationPhase.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/ConditionalEliminationPhase.java @@ -511,6 +511,8 @@ public class ConditionalEliminationPhase extends BasePhase { allow = true; } } else { + // Fortify: Suppress Null Dereference false positive + assert bestPossibleStamp != null; allow = (bestPossibleStamp.asConstant() != null); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/DeoptimizationGroupingPhase.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/DeoptimizationGroupingPhase.java index b48b39e437c..30f6cef359c 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/DeoptimizationGroupingPhase.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/DeoptimizationGroupingPhase.java @@ -24,6 +24,7 @@ package org.graalvm.compiler.phases.common; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -34,7 +35,6 @@ import org.graalvm.compiler.nodes.AbstractDeoptimizeNode; import org.graalvm.compiler.nodes.AbstractMergeNode; import org.graalvm.compiler.nodes.DynamicDeoptimizeNode; import org.graalvm.compiler.nodes.EndNode; -import org.graalvm.compiler.nodes.FixedNode; import org.graalvm.compiler.nodes.FrameState; import org.graalvm.compiler.nodes.LoopBeginNode; import org.graalvm.compiler.nodes.LoopExitNode; @@ -59,54 +59,53 @@ public class DeoptimizationGroupingPhase extends BasePhase { protected void run(StructuredGraph graph, MidTierContext context) { ControlFlowGraph cfg = null; for (FrameState fs : graph.getNodes(FrameState.TYPE)) { - FixedNode target = null; - PhiNode reasonActionPhi = null; - PhiNode speculationPhi = null; - List obsoletes = null; - for (AbstractDeoptimizeNode deopt : fs.usages().filter(AbstractDeoptimizeNode.class)) { - if (target == null) { - target = deopt; - } else { - if (cfg == null) { - cfg = ControlFlowGraph.compute(graph, true, true, false, false); - } - AbstractMergeNode merge; - if (target instanceof AbstractDeoptimizeNode) { - merge = graph.add(new MergeNode()); - EndNode firstEnd = graph.add(new EndNode()); - ValueNode actionAndReason = ((AbstractDeoptimizeNode) target).getActionAndReason(context.getMetaAccess()); - ValueNode speculation = ((AbstractDeoptimizeNode) target).getSpeculation(context.getMetaAccess()); - reasonActionPhi = graph.addWithoutUnique(new ValuePhiNode(StampFactory.forKind(actionAndReason.getStackKind()), merge)); - speculationPhi = graph.addWithoutUnique(new ValuePhiNode(StampFactory.forKind(speculation.getStackKind()), merge)); - merge.addForwardEnd(firstEnd); - reasonActionPhi.addInput(actionAndReason); - speculationPhi.addInput(speculation); - target.replaceAtPredecessor(firstEnd); - - exitLoops((AbstractDeoptimizeNode) target, firstEnd, cfg); - try (DebugCloseable position = target.withNodeSourcePosition()) { - merge.setNext(graph.add(new DynamicDeoptimizeNode(reasonActionPhi, speculationPhi))); - } - obsoletes = new LinkedList<>(); - obsoletes.add((AbstractDeoptimizeNode) target); - target = merge; - } else { - merge = (AbstractMergeNode) target; - } - EndNode newEnd = graph.add(new EndNode()); - merge.addForwardEnd(newEnd); - reasonActionPhi.addInput(deopt.getActionAndReason(context.getMetaAccess())); - speculationPhi.addInput(deopt.getSpeculation(context.getMetaAccess())); - deopt.replaceAtPredecessor(newEnd); - exitLoops(deopt, newEnd, cfg); - obsoletes.add(deopt); - } + Iterator iterator = fs.usages().filter(AbstractDeoptimizeNode.class).iterator(); + if (!iterator.hasNext()) { + // No deopt + continue; } - if (obsoletes != null) { - ((DynamicDeoptimizeNode) ((AbstractMergeNode) target).next()).setStateBefore(fs); - for (AbstractDeoptimizeNode obsolete : obsoletes) { - obsolete.safeDelete(); - } + AbstractDeoptimizeNode first = iterator.next(); + if (!iterator.hasNext()) { + // Only 1 deopt + continue; + } + // There is more than one deopt, create a merge + if (cfg == null) { + cfg = ControlFlowGraph.compute(graph, true, true, false, false); + } + AbstractMergeNode merge = graph.add(new MergeNode()); + EndNode firstEnd = graph.add(new EndNode()); + ValueNode actionAndReason = first.getActionAndReason(context.getMetaAccess()); + ValueNode speculation = first.getSpeculation(context.getMetaAccess()); + PhiNode reasonActionPhi = graph.addWithoutUnique(new ValuePhiNode(StampFactory.forKind(actionAndReason.getStackKind()), merge)); + PhiNode speculationPhi = graph.addWithoutUnique(new ValuePhiNode(StampFactory.forKind(speculation.getStackKind()), merge)); + merge.addForwardEnd(firstEnd); + reasonActionPhi.addInput(actionAndReason); + speculationPhi.addInput(speculation); + first.replaceAtPredecessor(firstEnd); + exitLoops(first, firstEnd, cfg); + DynamicDeoptimizeNode dynamicDeopt; + try (DebugCloseable position = first.withNodeSourcePosition()) { + dynamicDeopt = new DynamicDeoptimizeNode(reasonActionPhi, speculationPhi); + merge.setNext(graph.add(dynamicDeopt)); + } + List obsoletes = new LinkedList<>(); + obsoletes.add(first); + + do { + AbstractDeoptimizeNode deopt = iterator.next(); + EndNode newEnd = graph.add(new EndNode()); + merge.addForwardEnd(newEnd); + reasonActionPhi.addInput(deopt.getActionAndReason(context.getMetaAccess())); + speculationPhi.addInput(deopt.getSpeculation(context.getMetaAccess())); + deopt.replaceAtPredecessor(newEnd); + exitLoops(deopt, newEnd, cfg); + obsoletes.add(deopt); + } while (iterator.hasNext()); + + dynamicDeopt.setStateBefore(fs); + for (AbstractDeoptimizeNode obsolete : obsoletes) { + obsolete.safeDelete(); } } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/FloatingReadPhase.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/FloatingReadPhase.java index 4d31246554b..fceddb54022 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/FloatingReadPhase.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/FloatingReadPhase.java @@ -254,6 +254,8 @@ public class FloatingReadPhase extends Phase { for (MemoryMap state : states) { MemoryNode last = state.getLastLocationAccess(key); if (isPhi) { + // Fortify: Suppress Null Deference false positive (`isPhi == true` implies + // `merged != null`) ((MemoryPhiNode) merged).addInput(ValueNodeUtil.asNode(last)); } else { if (merged == last) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/UseTrappingNullChecksPhase.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/UseTrappingNullChecksPhase.java index d030982e84a..4b428d91af2 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/UseTrappingNullChecksPhase.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/UseTrappingNullChecksPhase.java @@ -137,7 +137,8 @@ public class UseTrappingNullChecksPhase extends BasePhase { int index = 0; for (AbstractEndNode end : merge.cfgPredecessors().snapshot()) { ValueNode thisReason = reasons != null ? reasons.get(index) : reason; - ValueNode thisSpeculation = speculations != null ? speculations.get(index++) : speculation; + ValueNode thisSpeculation = speculations != null ? speculations.get(index) : speculation; + index++; if (!thisReason.isConstant() || !thisSpeculation.isConstant()) { continue; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/InliningUtil.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/InliningUtil.java index eae26ed3cfa..da5fd7f8a25 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/InliningUtil.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/InliningUtil.java @@ -72,7 +72,6 @@ import org.graalvm.compiler.nodes.InliningLog; import org.graalvm.compiler.nodes.Invoke; import org.graalvm.compiler.nodes.InvokeNode; import org.graalvm.compiler.nodes.InvokeWithExceptionNode; -import org.graalvm.compiler.nodes.KillingBeginNode; import org.graalvm.compiler.nodes.LogicNode; import org.graalvm.compiler.nodes.MergeNode; import org.graalvm.compiler.nodes.NodeView; @@ -93,7 +92,6 @@ import org.graalvm.compiler.nodes.java.ExceptionObjectNode; import org.graalvm.compiler.nodes.java.MethodCallTargetNode; import org.graalvm.compiler.nodes.java.MonitorExitNode; import org.graalvm.compiler.nodes.java.MonitorIdNode; -import org.graalvm.compiler.nodes.spi.Replacements; import org.graalvm.compiler.nodes.type.StampTool; import org.graalvm.compiler.nodes.util.GraphUtil; import org.graalvm.compiler.phases.common.inlining.info.InlineInfo; @@ -466,11 +464,7 @@ public class InliningUtil extends ValueMergeUtil { // A partial intrinsic exit must be replaced with a call to // the intrinsified method. Invoke dup = (Invoke) duplicates.get(exit.asNode()); - if (dup instanceof InvokeNode) { - ((InvokeNode) dup).replaceWithNewBci(invoke.bci()); - } else { - ((InvokeWithExceptionNode) dup).replaceWithNewBci(invoke.bci()); - } + dup.replaceBci(invoke.bci()); } if (unwindNode != null) { unwindNode = (UnwindNode) duplicates.get(unwindNode); @@ -544,15 +538,7 @@ public class InliningUtil extends ValueMergeUtil { } // get rid of memory kill - AbstractBeginNode begin = invokeWithException.next(); - if (begin instanceof KillingBeginNode) { - try (DebugCloseable position = begin.withNodeSourcePosition()) { - AbstractBeginNode newBegin = new BeginNode(); - graph.addAfterFixed(begin, graph.add(newBegin)); - begin.replaceAtUsages(newBegin); - graph.removeFixed(begin); - } - } + invokeWithException.killKillingBegin(); } else { if (unwindNode != null && unwindNode.isAlive()) { try (DebugCloseable position = unwindNode.withNodeSourcePosition()) { @@ -836,7 +822,7 @@ public class InliningUtil extends ValueMergeUtil { // Return value does no longer need to be limited by the monitor exit. for (MonitorExitNode n : frameState.usages().filter(MonitorExitNode.class)) { - n.clearEscapedReturnValue(); + n.clearEscapedValue(); } frameState.replaceAndDelete(stateAfterReturn); @@ -1005,14 +991,6 @@ public class InliningUtil extends ValueMergeUtil { } } - public static boolean canIntrinsify(Replacements replacements, ResolvedJavaMethod target, int invokeBci) { - return replacements.hasSubstitution(target, invokeBci); - } - - public static StructuredGraph getIntrinsicGraph(Replacements replacements, ResolvedJavaMethod target, int invokeBci, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) { - return replacements.getSubstitution(target, invokeBci, trackNodeSourcePosition, replaceePosition); - } - /** * This method exclude InstrumentationNode from inlining heuristics. */ diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/info/elem/InlineableGraph.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/info/elem/InlineableGraph.java index c4b840ac341..b099f19221c 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/info/elem/InlineableGraph.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/info/elem/InlineableGraph.java @@ -69,7 +69,7 @@ public class InlineableGraph implements Inlineable { private FixedNodeRelativeFrequencyCache probabilites = new FixedNodeRelativeFrequencyCache(); public InlineableGraph(final ResolvedJavaMethod method, final Invoke invoke, final HighTierContext context, CanonicalizerPhase canonicalizer, boolean trackNodeSourcePosition) { - StructuredGraph original = InliningUtil.getIntrinsicGraph(context.getReplacements(), method, invoke.bci(), trackNodeSourcePosition, null); + StructuredGraph original = context.getReplacements().getSubstitution(method, invoke.bci(), trackNodeSourcePosition, null, invoke.asNode().getOptions()); if (original == null) { original = parseBytecodes(method, context, canonicalizer, invoke.asNode().graph(), trackNodeSourcePosition); } else if (original.isFrozen()) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/AbstractInliningPolicy.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/AbstractInliningPolicy.java index 1f9a0dab0cc..648d2d2a36a 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/AbstractInliningPolicy.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/AbstractInliningPolicy.java @@ -31,7 +31,6 @@ import java.util.Map; import org.graalvm.compiler.nodes.Invoke; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.spi.Replacements; -import org.graalvm.compiler.phases.common.inlining.InliningUtil; import org.graalvm.compiler.phases.common.inlining.info.InlineInfo; import org.graalvm.compiler.phases.common.inlining.info.elem.Inlineable; @@ -69,7 +68,7 @@ public abstract class AbstractInliningPolicy implements InliningPolicy { private static boolean onlyIntrinsics(Replacements replacements, InlineInfo info) { for (int i = 0; i < info.numberOfMethods(); i++) { - if (!InliningUtil.canIntrinsify(replacements, info.methodAt(i), info.invoke().bci())) { + if (!replacements.hasSubstitution(info.methodAt(i), info.invoke().bci())) { return false; } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/GreedyInliningPolicy.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/GreedyInliningPolicy.java index 6ba39ae80a4..b06509a7a11 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/GreedyInliningPolicy.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/GreedyInliningPolicy.java @@ -64,10 +64,10 @@ public class GreedyInliningPolicy extends AbstractInliningPolicy { } @Override - public Decision isWorthInlining(Replacements replacements, MethodInvocation invocation, int inliningDepth, boolean fullyProcessed) { - final boolean isTracing = TraceInlining.getValue(replacements.getOptions()); + public Decision isWorthInlining(Replacements replacements, MethodInvocation invocation, InlineInfo calleeInfo, int inliningDepth, boolean fullyProcessed) { + OptionValues options = calleeInfo.graph().getOptions(); + final boolean isTracing = TraceInlining.getValue(options); final InlineInfo info = invocation.callee(); - OptionValues options = info.graph().getOptions(); final double probability = invocation.probability(); final double relevance = invocation.relevance(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/InlineEverythingPolicy.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/InlineEverythingPolicy.java index a8c7a567fbd..12441df58a9 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/InlineEverythingPolicy.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/InlineEverythingPolicy.java @@ -31,6 +31,7 @@ import org.graalvm.compiler.core.common.PermanentBailoutException; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.spi.Replacements; import org.graalvm.compiler.phases.common.inlining.InliningUtil; +import org.graalvm.compiler.phases.common.inlining.info.InlineInfo; import org.graalvm.compiler.phases.common.inlining.walker.MethodInvocation; public class InlineEverythingPolicy implements InliningPolicy { @@ -44,7 +45,7 @@ public class InlineEverythingPolicy implements InliningPolicy { } @Override - public Decision isWorthInlining(Replacements replacements, MethodInvocation invocation, int inliningDepth, boolean fullyProcessed) { - return Decision.YES.withReason(GraalOptions.TraceInlining.getValue(replacements.getOptions()), "inline everything"); + public Decision isWorthInlining(Replacements replacements, MethodInvocation invocation, InlineInfo calleeInfo, int inliningDepth, boolean fullyProcessed) { + return Decision.YES.withReason(GraalOptions.TraceInlining.getValue(calleeInfo.graph().getOptions()), "inline everything"); } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/InlineMethodSubstitutionsPolicy.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/InlineMethodSubstitutionsPolicy.java index 5ed79772a6a..c633b2be8f3 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/InlineMethodSubstitutionsPolicy.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/InlineMethodSubstitutionsPolicy.java @@ -28,6 +28,7 @@ import org.graalvm.compiler.core.common.GraalOptions; import org.graalvm.compiler.nodes.CallTargetNode; import org.graalvm.compiler.nodes.java.MethodCallTargetNode; import org.graalvm.compiler.nodes.spi.Replacements; +import org.graalvm.compiler.phases.common.inlining.info.InlineInfo; import org.graalvm.compiler.phases.common.inlining.walker.MethodInvocation; import jdk.vm.ci.meta.ResolvedJavaMethod; @@ -38,11 +39,11 @@ import jdk.vm.ci.meta.ResolvedJavaMethod; public final class InlineMethodSubstitutionsPolicy extends InlineEverythingPolicy { @Override - public Decision isWorthInlining(Replacements replacements, MethodInvocation invocation, int inliningDepth, boolean fullyProcessed) { - final boolean isTracing = GraalOptions.TraceInlining.getValue(replacements.getOptions()); + public Decision isWorthInlining(Replacements replacements, MethodInvocation invocation, InlineInfo calleeInfo, int inliningDepth, boolean fullyProcessed) { + final boolean isTracing = GraalOptions.TraceInlining.getValue(calleeInfo.graph().getOptions()); CallTargetNode callTarget = invocation.callee().invoke().callTarget(); if (callTarget instanceof MethodCallTargetNode) { - ResolvedJavaMethod calleeMethod = ((MethodCallTargetNode) callTarget).targetMethod(); + ResolvedJavaMethod calleeMethod = callTarget.targetMethod(); if (replacements.hasSubstitution(calleeMethod, invocation.callee().invoke().bci())) { return Decision.YES.withReason(isTracing, "has a method subtitution"); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/InliningPolicy.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/InliningPolicy.java index 8b6698b6450..b8853f02a81 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/InliningPolicy.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/policy/InliningPolicy.java @@ -26,6 +26,7 @@ package org.graalvm.compiler.phases.common.inlining.policy; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.spi.Replacements; +import org.graalvm.compiler.phases.common.inlining.info.InlineInfo; import org.graalvm.compiler.phases.common.inlining.walker.MethodInvocation; public interface InliningPolicy { @@ -60,5 +61,5 @@ public interface InliningPolicy { boolean continueInlining(StructuredGraph graph); - Decision isWorthInlining(Replacements replacements, MethodInvocation invocation, int inliningDepth, boolean fullyProcessed); + Decision isWorthInlining(Replacements replacements, MethodInvocation invocation, InlineInfo calleeInfo, int inliningDepth, boolean fullyProcessed); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/walker/InliningData.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/walker/InliningData.java index 2489d8d5dc0..47350a5f54b 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/walker/InliningData.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/walker/InliningData.java @@ -45,6 +45,7 @@ import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.Graph; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.nodes.CallTargetNode; +import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind; import org.graalvm.compiler.nodes.Invoke; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.ParameterNode; @@ -144,7 +145,7 @@ public class InliningData { OptionValues options = rootGraph.getOptions(); if (method == null) { return "the method is not resolved"; - } else if (method.isNative() && (!Intrinsify.getValue(options) || !InliningUtil.canIntrinsify(context.getReplacements(), method, invokeBci))) { + } else if (method.isNative() && (!Intrinsify.getValue(options) || !context.getReplacements().hasSubstitution(method, invokeBci))) { return "it is a non-intrinsic native method"; } else if (method.isAbstract()) { return "it is an abstract method"; @@ -191,11 +192,12 @@ public class InliningData { MethodCallTargetNode callTarget = (MethodCallTargetNode) invoke.callTarget(); ResolvedJavaMethod targetMethod = callTarget.targetMethod(); - if (callTarget.invokeKind() == CallTargetNode.InvokeKind.Special || targetMethod.canBeStaticallyBound()) { + InvokeKind invokeKind = callTarget.invokeKind(); + if (invokeKind == CallTargetNode.InvokeKind.Special || invokeKind == CallTargetNode.InvokeKind.Static || targetMethod.canBeStaticallyBound()) { return getExactInlineInfo(invoke, targetMethod); } - assert callTarget.invokeKind().isIndirect(); + assert invokeKind.isIndirect(); ResolvedJavaType holder = targetMethod.getDeclaringClass(); if (!(callTarget.receiver().stamp(NodeView.DEFAULT) instanceof ObjectStamp)) { @@ -457,7 +459,7 @@ public class InliningData { assert callerCallsiteHolder.containsInvoke(calleeInfo.invoke()); counterInliningConsidered.increment(debug); - InliningPolicy.Decision decision = inliningPolicy.isWorthInlining(context.getReplacements(), calleeInvocation, inliningDepth, true); + InliningPolicy.Decision decision = inliningPolicy.isWorthInlining(context.getReplacements(), calleeInvocation, calleeInfo, inliningDepth, true); if (decision.shouldInline()) { doInline(callerCallsiteHolder, calleeInvocation, decision.getReason()); return true; @@ -718,7 +720,8 @@ public class InliningData { final MethodInvocation currentInvocation = currentInvocation(); - final boolean backtrack = (!currentInvocation.isRoot() && !inliningPolicy.isWorthInlining(context.getReplacements(), currentInvocation, inliningDepth(), false).shouldInline()); + final boolean backtrack = (!currentInvocation.isRoot() && + !inliningPolicy.isWorthInlining(context.getReplacements(), currentInvocation, currentInvocation.callee(), inliningDepth(), false).shouldInline()); if (backtrack) { int remainingGraphs = currentInvocation.totalGraphs() - currentInvocation.processedGraphs(); assert remainingGraphs > 0; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.printer/src/org/graalvm/compiler/printer/BinaryGraphPrinter.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.printer/src/org/graalvm/compiler/printer/BinaryGraphPrinter.java index 3c017ed723d..c1ab9556666 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.printer/src/org/graalvm/compiler/printer/BinaryGraphPrinter.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.printer/src/org/graalvm/compiler/printer/BinaryGraphPrinter.java @@ -69,6 +69,7 @@ import org.graalvm.compiler.nodes.util.JavaConstantFormattable; import org.graalvm.compiler.phases.schedule.SchedulePhase; import org.graalvm.graphio.GraphBlocks; import org.graalvm.graphio.GraphElements; +import org.graalvm.graphio.GraphLocations; import org.graalvm.graphio.GraphOutput; import org.graalvm.graphio.GraphStructure; import org.graalvm.graphio.GraphTypes; @@ -77,7 +78,6 @@ import jdk.vm.ci.meta.JavaType; import jdk.vm.ci.meta.ResolvedJavaField; import jdk.vm.ci.meta.ResolvedJavaMethod; import jdk.vm.ci.meta.Signature; -import org.graalvm.graphio.GraphLocations; public class BinaryGraphPrinter implements GraphStructure, Edges>, @@ -528,7 +528,7 @@ public class BinaryGraphPrinter implements public URI getURI() { String path = e.getFileName(); try { - return path == null ? null : new URI(null, null, path, null); + return new URI(null, null, path == null ? "(Unknown Source)" : path, null); } catch (URISyntaxException ex) { throw new IllegalArgumentException(ex); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64IntegerSubstitutions.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64BitCountNode.java similarity index 50% rename from src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64IntegerSubstitutions.java rename to src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64BitCountNode.java index f3669f2630a..4993fc328ac 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64IntegerSubstitutions.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64BitCountNode.java @@ -1,5 +1,6 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, Arm Limited. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,28 +25,20 @@ package org.graalvm.compiler.replacements.aarch64; -import org.graalvm.compiler.api.replacements.ClassSubstitution; -import org.graalvm.compiler.api.replacements.MethodSubstitution; +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_4; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_4; -/** - * AArch64 ISA offers a count leading zeros instruction which can be used to implement - * numberOfLeadingZeros more efficiently than using BitScanReverse. - */ -@ClassSubstitution(Integer.class) -public class AArch64IntegerSubstitutions { +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.replacements.nodes.BitCountNode; - @MethodSubstitution - public static int bitCount(int value) { - // Based on Warren, Hacker's Delight, slightly adapted to profit from Aarch64 add + shift - // instruction. - // Assuming the peephole optimizer optimizes all x - y >>> z into a single instruction - // this takes 10 instructions. - int x = value; - x = x - ((x & 0xaaaaaaaa) >>> 1); - x = (x & 0x33333333) + ((x & 0xcccccccc) >>> 2); - x = (x + (x >>> 4)) & 0x0f0f0f0f; - x = x + (x >>> 8); - x = x + (x >>> 16); - return x & 0x3f; +@NodeInfo(cycles = CYCLES_4, size = SIZE_4) +public final class AArch64BitCountNode extends BitCountNode { + + public static final NodeClass TYPE = NodeClass.create(AArch64BitCountNode.class); + + public AArch64BitCountNode(ValueNode value) { + super(TYPE, value); } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64GraphBuilderPlugins.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64GraphBuilderPlugins.java index 6af275d6790..482f57aeff9 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64GraphBuilderPlugins.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64GraphBuilderPlugins.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -64,8 +64,8 @@ public class AArch64GraphBuilderPlugins { invocationPlugins.defer(new Runnable() { @Override public void run() { - registerIntegerLongPlugins(invocationPlugins, AArch64IntegerSubstitutions.class, JavaKind.Int, bytecodeProvider); - registerIntegerLongPlugins(invocationPlugins, AArch64LongSubstitutions.class, JavaKind.Long, bytecodeProvider); + registerIntegerLongPlugins(invocationPlugins, JavaKind.Int, bytecodeProvider); + registerIntegerLongPlugins(invocationPlugins, JavaKind.Long, bytecodeProvider); if (registerMathPlugins) { registerMathPlugins(invocationPlugins); } @@ -80,7 +80,7 @@ public class AArch64GraphBuilderPlugins { }); } - private static void registerIntegerLongPlugins(InvocationPlugins plugins, Class substituteDeclaringClass, JavaKind kind, BytecodeProvider bytecodeProvider) { + private static void registerIntegerLongPlugins(InvocationPlugins plugins, JavaKind kind, BytecodeProvider bytecodeProvider) { Class declaringClass = kind.toBoxedJavaClass(); Class type = kind.toJavaClass(); Registration r = new Registration(plugins, declaringClass, bytecodeProvider); @@ -108,7 +108,13 @@ public class AArch64GraphBuilderPlugins { return true; } }); - r.registerMethodSubstitution(substituteDeclaringClass, "bitCount", type); + r.register1("bitCount", type, new InvocationPlugin() { + @Override + public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) { + b.push(JavaKind.Int, b.append(new AArch64BitCountNode(value).canonical(null))); + return true; + } + }); } private static void registerMathPlugins(InvocationPlugins plugins) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64LongSubstitutions.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64LongSubstitutions.java deleted file mode 100644 index 5338df18fec..00000000000 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64LongSubstitutions.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - - -package org.graalvm.compiler.replacements.aarch64; - -import org.graalvm.compiler.api.replacements.ClassSubstitution; -import org.graalvm.compiler.api.replacements.MethodSubstitution; - -/** - * Aarch64 ISA offers a count leading zeros instruction which can be used to implement - * numberOfLeadingZeros more efficiently than using BitScanReverse. - */ -@ClassSubstitution(Long.class) -public class AArch64LongSubstitutions { - - @MethodSubstitution - public static int bitCount(long value) { - // Based on Warren, Hacker's Delight, slightly adapted to profit from Aarch64 add + shift - // instruction. - // Assuming the peephole optimizer optimizes all x - y >>> z into a single instruction - // this takes 11 instructions. - long x = value; - x = x - ((x & 0xaaaaaaaaaaaaaaaaL) >>> 1); - x = (x & 0x3333333333333333L) + ((x & 0xccccccccccccccccL) >>> 2); - x = (x + (x >>> 4)) & 0x0f0f0f0f0f0f0f0fL; - x = x + (x >>> 8); - x = x + (x >>> 16); - x = x + (x >>> 32); - return (int) x & 0x7f; - } - -} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.processor/src/org/graalvm/compiler/replacements/processor/GeneratedFoldPlugin.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.processor/src/org/graalvm/compiler/replacements/processor/GeneratedFoldPlugin.java index 70035f4cb9e..82283e31c9d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.processor/src/org/graalvm/compiler/replacements/processor/GeneratedFoldPlugin.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.processor/src/org/graalvm/compiler/replacements/processor/GeneratedFoldPlugin.java @@ -84,7 +84,9 @@ public class GeneratedFoldPlugin extends GeneratedPlugin { if (processor.getAnnotation(param, processor.getType(INJECTED_PARAMETER_CLASS_NAME)) == null) { constantArgument(processor, out, deps, argCount, param.asType(), argCount); } else { - out.printf(" assert checkInjectedArgument(b, args[%d], targetMethod);\n", argCount); + out.printf(" if (!checkInjectedArgument(b, args[%d], targetMethod)) {\n", argCount); + out.printf(" return false;\n"); + out.printf(" }\n", argCount); out.printf(" %s arg%d = %s;\n", param.asType(), argCount, deps.use(processor, (DeclaredType) param.asType())); } argCount++; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/MethodSubstitutionTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/MethodSubstitutionTest.java index 7e32dbcadbe..ad784806863 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/MethodSubstitutionTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/MethodSubstitutionTest.java @@ -116,7 +116,7 @@ public abstract class MethodSubstitutionTest extends GraalCompilerTest { StructuredGraph graph = testGraph(testMethodName); // Check to see if the resulting graph contains the expected node - StructuredGraph replacement = getReplacements().getSubstitution(realMethod, -1, false, null); + StructuredGraph replacement = getReplacements().getSubstitution(realMethod, -1, false, null, graph.getOptions()); if (replacement == null && !optional) { assertInGraph(graph, intrinsicClass); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/ReplacementsParseTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/ReplacementsParseTest.java index 9d9d8d920a6..1c857153f07 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/ReplacementsParseTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/ReplacementsParseTest.java @@ -430,7 +430,7 @@ public class ReplacementsParseTest extends ReplacementsTest { String compiledReturnValue = IN_COMPILED_HANDLER_MARKER; forceCompileOverride = true; inlineInvokeDecision = InlineInvokePlugin.InlineInfo.DO_NOT_INLINE_WITH_EXCEPTION; - inlineInvokeMethodName = "stringizeId"; + inlineInvokeMethodName = "stringize"; try { testWithDifferentReturnValues(options, standardReturnValue, compiledReturnValue, "callStringize", THROW_EXCEPTION_MARKER); } finally { @@ -681,6 +681,11 @@ public class ReplacementsParseTest extends ReplacementsTest { public ForeignCallLinkage lookupForeignCall(ForeignCallDescriptor descriptor) { return null; } + + @Override + public boolean isAvailable(ForeignCallDescriptor descriptor) { + return true; + } }; } if (type == SnippetReflectionProvider.class) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/RootMethodSubstitutionTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/RootMethodSubstitutionTest.java new file mode 100644 index 00000000000..b5eac892e87 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/RootMethodSubstitutionTest.java @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.replacements.test; + +import java.util.ArrayList; +import java.util.List; + +import jdk.internal.vm.compiler.collections.MapCursor; +import org.graalvm.compiler.api.test.Graal; +import org.graalvm.compiler.core.common.CompilationIdentifier; +import org.graalvm.compiler.core.common.GraalOptions; +import org.graalvm.compiler.core.target.Backend; +import org.graalvm.compiler.core.test.GraalCompilerTest; +import org.graalvm.compiler.debug.DebugContext; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins; +import org.graalvm.compiler.nodes.graphbuilderconf.MethodSubstitutionPlugin; +import org.graalvm.compiler.options.OptionValues; +import org.graalvm.compiler.phases.util.Providers; +import org.graalvm.compiler.runtime.RuntimeProvider; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import jdk.vm.ci.meta.MetaAccessProvider; +import jdk.vm.ci.meta.ResolvedJavaMethod; +import jdk.vm.ci.meta.ResolvedJavaType; + +/** + * Exercise + * {@link org.graalvm.compiler.nodes.spi.Replacements#getIntrinsicGraph(ResolvedJavaMethod, CompilationIdentifier, DebugContext)} + * with regular method substitutions and encoded graphs. + */ +@RunWith(Parameterized.class) +public class RootMethodSubstitutionTest extends GraalCompilerTest { + + public RootMethodSubstitutionTest(ResolvedJavaMethod method) { + this.method = method; + } + + @Parameterized.Parameters(name = "{0}") + public static List data() { + ArrayList ret = new ArrayList<>(); + + Backend backend = Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend(); + Providers providers = backend.getProviders(); + + MapCursor> cursor = providers.getReplacements().getGraphBuilderPlugins().getInvocationPlugins().getBindings(true).getEntries(); + MetaAccessProvider metaAccess = providers.getMetaAccess(); + while (cursor.advance()) { + String className = cursor.getKey(); + ResolvedJavaType type = null; + try { + String typeName = className.substring(1, className.length() - 1).replace('/', '.'); + ClassLoader cl = ClassLoader.getSystemClassLoader(); + Class clazz = Class.forName(typeName, true, cl); + type = metaAccess.lookupJavaType(clazz); + } catch (ClassNotFoundException e) { + continue; + } + + for (InvocationPlugins.Binding binding : cursor.getValue()) { + if (binding.plugin instanceof MethodSubstitutionPlugin) { + ResolvedJavaMethod original = null; + for (ResolvedJavaMethod declared : type.getDeclaredMethods()) { + if (declared.getName().equals(binding.name)) { + if (declared.isStatic() == binding.isStatic) { + if (declared.getSignature().toMethodDescriptor().startsWith(binding.argumentsDescriptor)) { + original = declared; + break; + } + } + } + } + if (!original.isNative()) { + ret.add(new Object[]{original}); + } + } + } + } + return ret; + } + + private final ResolvedJavaMethod method; + + private StructuredGraph getIntrinsicGraph(boolean useEncodedGraphs) { + OptionValues options = new OptionValues(getDebugContext().getOptions(), GraalOptions.UseEncodedGraphs, useEncodedGraphs); + DebugContext debugContext = DebugContext.create(options, getDebugContext().getDescription(), getDebugHandlersFactories()); + return getReplacements().getIntrinsicGraph(method, CompilationIdentifier.INVALID_COMPILATION_ID, debugContext); + } + + StructuredGraph expectedGraph; + StructuredGraph actualGraph; + + @Override + protected boolean checkHighTierGraph(StructuredGraph graph) { + // Capture the graphs after high tier + if (expectedGraph == null) { + expectedGraph = (StructuredGraph) graph.copy(graph.getDebug()); + } else { + assert actualGraph == null; + actualGraph = (StructuredGraph) graph.copy(graph.getDebug()); + } + return super.checkHighTierGraph(graph); + } + + @Test + public void test() { + StructuredGraph regularGraph = getIntrinsicGraph(false); + assertTrue(regularGraph != null, "must produce a graph"); + getCode(method, regularGraph); + + StructuredGraph encodedGraph = getIntrinsicGraph(true); + assertTrue(encodedGraph != null, "must produce a graph"); + getCode(method, encodedGraph); + + // Compare the high tier graphs since the final graph might have scheduler + // differences because of different usage ordering. + assertEquals(expectedGraph, actualGraph, true, false); + } + +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/SnippetsTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/SnippetsTest.java index 587baba61ac..29d5795543b 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/SnippetsTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/SnippetsTest.java @@ -39,7 +39,7 @@ public abstract class SnippetsTest extends ReplacementsTest { protected SnippetsTest() { ReplacementsImpl d = (ReplacementsImpl) getReplacements(); bytecodeProvider = getSystemClassLoaderBytecodeProvider(); - installer = new ReplacementsImpl(getInitialOptions(), null, d.getProviders(), d.snippetReflection, bytecodeProvider, d.target); + installer = new ReplacementsImpl(null, d.getProviders(), d.snippetReflection, bytecodeProvider, d.target); installer.setGraphBuilderPlugins(d.getGraphBuilderPlugins()); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StandardMethodSubstitutionsTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StandardMethodSubstitutionsTest.java index 2eb823bb08d..d0cd1604c2b 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StandardMethodSubstitutionsTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StandardMethodSubstitutionsTest.java @@ -138,7 +138,7 @@ public class StandardMethodSubstitutionsTest extends MethodSubstitutionTest { StructuredGraph graph = testGraph(testMethodName); // Check to see if the resulting graph contains the expected node - StructuredGraph replacement = getReplacements().getSubstitution(realJavaMethod, -1, false, null); + StructuredGraph replacement = getReplacements().getSubstitution(realJavaMethod, -1, false, null, graph.getOptions()); if (replacement == null && !optional) { assertInGraph(graph, intrinsicClass); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringCompareToTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringCompareToTest.java index 383cc0e4a96..dfa685689ed 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringCompareToTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringCompareToTest.java @@ -83,7 +83,7 @@ public class StringCompareToTest extends StringSubstitutionTestBase { StructuredGraph graph = testGraph(testMethod.getName()); // Check to see if the resulting graph contains the expected node - StructuredGraph replacement = getReplacements().getSubstitution(realMethod, -1, false, null); + StructuredGraph replacement = getReplacements().getSubstitution(realMethod, -1, false, null, graph.getOptions()); if (replacement == null) { assertInGraph(graph, expectedNode); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringCompressInflateTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringCompressInflateTest.java index 4cd9a2fe49c..e8bbd23b37f 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringCompressInflateTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringCompressInflateTest.java @@ -312,7 +312,7 @@ public final class StringCompressInflateTest extends MethodSubstitutionTest { } StructuredGraph replacementGraph() { - return getReplacements().getSubstitution(javamethod, -1, false, null); + return getReplacements().getSubstitution(javamethod, -1, false, null, getInitialOptions()); } StructuredGraph testMethodGraph() { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringIndexOfCharTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringIndexOfCharTest.java index 61e9c59e30c..1a07abcd414 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringIndexOfCharTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringIndexOfCharTest.java @@ -53,8 +53,8 @@ public class StringIndexOfCharTest extends GraalCompilerTest { "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; String[] targets = new String[]{"foobar", "foo", "bar", "\u03bbfoobar", mediumString, mediumUTF16String, longString, longUTF16String}; int[] targetChars = new int[]{'f', 'o', 'r', 'x', Character.MIN_SUPPLEMENTARY_CODE_POINT}; - int[] targetOffsets = new int[18]; - for (int i = 0; i < 18; i++) { + int[] targetOffsets = new int[12]; + for (int i = 0; i < targetOffsets.length; i++) { targetOffsets[i] = i - 1; } for (String source : targets) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringSubstitutionTestBase.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringSubstitutionTestBase.java index 6346b7076e4..2e49c6c9f71 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringSubstitutionTestBase.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringSubstitutionTestBase.java @@ -28,8 +28,8 @@ import org.graalvm.compiler.nodes.StructuredGraph; import org.junit.Assert; import org.junit.Assume; -import jdk.vm.ci.amd64.AMD64; import jdk.vm.ci.aarch64.AArch64; +import jdk.vm.ci.amd64.AMD64; import jdk.vm.ci.code.InstalledCode; import jdk.vm.ci.meta.ResolvedJavaMethod; @@ -71,7 +71,7 @@ public class StringSubstitutionTestBase extends MethodSubstitutionTest { StructuredGraph graph = testGraph(testMethod.getName()); // Check to see if the resulting graph contains the expected node - StructuredGraph replacement = getReplacements().getSubstitution(realMethod, -1, false, null); + StructuredGraph replacement = getReplacements().getSubstitution(realMethod, -1, false, null, graph.getOptions()); if (replacement == null) { assertInGraph(graph, expectedNode); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringSubstitutionsTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringSubstitutionsTest.java index f489820b3ff..7da80a6eef2 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringSubstitutionsTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringSubstitutionsTest.java @@ -43,7 +43,7 @@ public class StringSubstitutionsTest extends MethodSubstitutionTest { StructuredGraph graph = testGraph(testMethodName); // Check to see if the resulting graph contains the expected node - StructuredGraph replacement = getReplacements().getSubstitution(realMethod, -1, false, null); + StructuredGraph replacement = getReplacements().getSubstitution(realMethod, -1, false, null, graph.getOptions()); if (replacement == null && !optional) { assertInGraph(graph, intrinsicClass); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/SubstitutionNodeSourcePositionTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/SubstitutionNodeSourcePositionTest.java index dff3612ba5b..43bd3c0db19 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/SubstitutionNodeSourcePositionTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/SubstitutionNodeSourcePositionTest.java @@ -26,6 +26,7 @@ package org.graalvm.compiler.replacements.test; import static org.graalvm.compiler.core.GraalCompiler.compileGraph; import static org.graalvm.compiler.core.common.GraalOptions.TrackNodeSourcePosition; +import static org.graalvm.compiler.core.common.GraalOptions.UseEncodedGraphs; import java.util.List; @@ -44,6 +45,7 @@ import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.OptimisticOptimizations; import org.graalvm.compiler.replacements.classfile.ClassfileBytecodeProvider; import org.junit.Assert; +import org.junit.Assume; import org.junit.Test; import jdk.vm.ci.meta.ResolvedJavaMethod; @@ -100,6 +102,7 @@ public class SubstitutionNodeSourcePositionTest extends ReplacementsTest { // at org.graalvm.compiler.replacements.test.SubstitutionNodeSourcePositionTest$TestMethod.test(int) [bci: -1] // at org.graalvm.compiler.replacements.test.SubstitutionNodeSourcePositionTest.methodSubstitution(SubstitutionNodeSourcePositionTest.java:89) [bci: 2] // @formatter:on + Assume.assumeFalse(UseEncodedGraphs.getValue(getInitialOptions())); checkMappings("methodSubstitution", true, TestMethod.class, "test"); } @@ -119,6 +122,7 @@ public class SubstitutionNodeSourcePositionTest extends ReplacementsTest { // // The precise snippet bytecodes don't matter, just ensure that some actually appear after // lowering. + Assume.assumeFalse(UseEncodedGraphs.getValue(getInitialOptions())); checkMappings("snippetLowering", true, SubstitutionNodeSourcePositionTest.class, "snippetLowering"); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/classfile/ClassfileBytecodeProviderTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/classfile/ClassfileBytecodeProviderTest.java index ce95c485f85..4a397fe4b04 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/classfile/ClassfileBytecodeProviderTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/classfile/ClassfileBytecodeProviderTest.java @@ -140,6 +140,11 @@ public class ClassfileBytecodeProviderTest extends GraalCompilerTest { return false; } + /** + * Keep test time down by only sampling a limited number of class files per jar. + */ + private static final int CLASSES_PER_JAR = 250; + @Test public void test() { RuntimeProvider rt = Graal.getRequiredCapability(RuntimeProvider.class); @@ -156,28 +161,33 @@ public class ClassfileBytecodeProviderTest extends GraalCompilerTest { if (shouldProcess(path)) { try { final ZipFile zipFile = new ZipFile(new File(path)); + int index = 0; + int step = zipFile.size() > CLASSES_PER_JAR ? zipFile.size() / CLASSES_PER_JAR : 1; for (final Enumeration entry = zipFile.entries(); entry.hasMoreElements();) { final ZipEntry zipEntry = entry.nextElement(); - String name = zipEntry.getName(); - if (name.endsWith(".class") && !name.equals("module-info.class") && !name.startsWith("META-INF/versions/")) { - String className = name.substring(0, name.length() - ".class".length()).replace('/', '.'); - if (isInNativeImage(className)) { - /* - * Native image requires non-graalsdk classes to be present in the - * classpath. - */ - continue; - } - if (isGSON(className)) { - /* uses old class format */ - continue; - } - try { - checkClass(metaAccess, getSnippetReflection(), className); - } catch (ClassNotFoundException e) { - throw new AssertionError(e); + if ((index % step) == 0) { + String name = zipEntry.getName(); + if (name.endsWith(".class") && !name.equals("module-info.class") && !name.startsWith("META-INF/versions/")) { + String className = name.substring(0, name.length() - ".class".length()).replace('/', '.'); + if (isInNativeImage(className)) { + /* + * Native image requires non-graalsdk classes to be present in + * the classpath. + */ + continue; + } + if (isGSON(className)) { + /* uses old class format */ + continue; + } + try { + checkClass(metaAccess, getSnippetReflection(), className); + } catch (ClassNotFoundException e) { + throw new AssertionError(e); + } } } + index++; } } catch (IOException ex) { Assert.fail(ex.toString()); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/CachingPEGraphDecoder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/CachingPEGraphDecoder.java index 187c6ff3a6b..8a0e8d34b74 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/CachingPEGraphDecoder.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/CachingPEGraphDecoder.java @@ -24,6 +24,8 @@ package org.graalvm.compiler.replacements; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; +import static org.graalvm.compiler.core.common.GraalOptions.UseEncodedGraphs; import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.INLINE_AFTER_PARSING; import jdk.internal.vm.compiler.collections.EconomicMap; @@ -41,6 +43,7 @@ import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin; import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext; import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins; import org.graalvm.compiler.nodes.graphbuilderconf.LoopExplosionPlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.MethodSubstitutionPlugin; import org.graalvm.compiler.nodes.graphbuilderconf.NodePlugin; import org.graalvm.compiler.nodes.graphbuilderconf.ParameterPlugin; import org.graalvm.compiler.phases.OptimisticOptimizations; @@ -82,45 +85,65 @@ public class CachingPEGraphDecoder extends PEGraphDecoder { } @SuppressWarnings("try") - private EncodedGraph createGraph(ResolvedJavaMethod method, ResolvedJavaMethod originalMethod, BytecodeProvider intrinsicBytecodeProvider, boolean isSubstitution) { - // @formatter:off - StructuredGraph graphToEncode = new StructuredGraph.Builder(options, debug, allowAssumptions). - useProfilingInfo(false). - trackNodeSourcePosition(graphBuilderConfig.trackNodeSourcePosition()). - method(method). - setIsSubstitution(isSubstitution). - cancellable(graph.getCancellable()). - build(); + private EncodedGraph createGraph(ResolvedJavaMethod method, MethodSubstitutionPlugin plugin, BytecodeProvider intrinsicBytecodeProvider, boolean isSubstitution) { + StructuredGraph graphToEncode; + if (isSubstitution && (UseEncodedGraphs.getValue(options) || IS_IN_NATIVE_IMAGE)) { + // These must go through Replacements to find the graph to use. + graphToEncode = providers.getReplacements().getMethodSubstitution(plugin, method, INLINE_AFTER_PARSING, allowAssumptions, + options); + } else { + graphToEncode = buildGraph(method, plugin, intrinsicBytecodeProvider, isSubstitution); + } + + /* + * ConvertDeoptimizeToGuardPhase reduces the number of merges in the graph, so that fewer + * frame states will be created. This significantly reduces the number of nodes in the + * initial graph. + */ + try (DebugContext.Scope scope = debug.scope("createGraph", graphToEncode)) { + PhaseContext context = new PhaseContext(providers); + new ConvertDeoptimizeToGuardPhase().apply(graphToEncode, context); + } catch (Throwable t) { + throw debug.handle(t); + } + + EncodedGraph encodedGraph = GraphEncoder.encodeSingleGraph(graphToEncode, architecture); + graphCache.put(method, encodedGraph); + return encodedGraph; + } + + @SuppressWarnings("try") + private StructuredGraph buildGraph(ResolvedJavaMethod method, MethodSubstitutionPlugin plugin, BytecodeProvider intrinsicBytecodeProvider, boolean isSubstitution) { + StructuredGraph graphToEncode;// @formatter:off + graphToEncode = new StructuredGraph.Builder(options, debug, allowAssumptions). + useProfilingInfo(false). + trackNodeSourcePosition(graphBuilderConfig.trackNodeSourcePosition()). + method(plugin != null ? plugin.getSubstitute(providers.getMetaAccess()) : method). + setIsSubstitution(isSubstitution). + cancellable(graph.getCancellable()). + build(); // @formatter:on try (DebugContext.Scope scope = debug.scope("createGraph", graphToEncode)) { - IntrinsicContext initialIntrinsicContext = intrinsicBytecodeProvider != null ? new IntrinsicContext(originalMethod, method, intrinsicBytecodeProvider, INLINE_AFTER_PARSING) : null; + IntrinsicContext initialIntrinsicContext = intrinsicBytecodeProvider != null + ? new IntrinsicContext(method, plugin.getSubstitute(providers.getMetaAccess()), intrinsicBytecodeProvider, INLINE_AFTER_PARSING) : null; GraphBuilderPhase.Instance graphBuilderPhaseInstance = createGraphBuilderPhaseInstance(initialIntrinsicContext); graphBuilderPhaseInstance.apply(graphToEncode); PhaseContext context = new PhaseContext(providers); new CanonicalizerPhase().apply(graphToEncode, context); - /* - * ConvertDeoptimizeToGuardPhase reduces the number of merges in the graph, so that - * fewer frame states will be created. This significantly reduces the number of nodes in - * the initial graph. - */ - new ConvertDeoptimizeToGuardPhase().apply(graphToEncode, context); - - EncodedGraph encodedGraph = GraphEncoder.encodeSingleGraph(graphToEncode, architecture); - graphCache.put(method, encodedGraph); - return encodedGraph; } catch (Throwable ex) { throw debug.handle(ex); } + return graphToEncode; } @Override - protected EncodedGraph lookupEncodedGraph(ResolvedJavaMethod method, ResolvedJavaMethod originalMethod, BytecodeProvider intrinsicBytecodeProvider, boolean isSubstitution, + protected EncodedGraph lookupEncodedGraph(ResolvedJavaMethod method, MethodSubstitutionPlugin plugin, BytecodeProvider intrinsicBytecodeProvider, boolean isSubstitution, boolean trackNodeSourcePosition) { EncodedGraph result = graphCache.get(method); if (result == null && method.hasBytecodes()) { - result = createGraph(method, originalMethod, intrinsicBytecodeProvider, isSubstitution); + result = createGraph(method, plugin, intrinsicBytecodeProvider, isSubstitution); } return result; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/GraphKit.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/GraphKit.java index b397eff2c65..7d1125d70fe 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/GraphKit.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/GraphKit.java @@ -366,7 +366,7 @@ public class GraphKit implements GraphBuilderTool { StructuredGraph calleeGraph; if (IS_IN_NATIVE_IMAGE) { - calleeGraph = providers.getReplacements().getSnippet(method, null, false, null); + calleeGraph = providers.getReplacements().getSnippet(method, null, null, false, null, invoke.getOptions()); } else { calleeGraph = new StructuredGraph.Builder(invoke.getOptions(), invoke.getDebug()).method(method).trackNodeSourcePosition(invoke.graph().trackNodeSourcePosition()).setIsSubstitution( true).build(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/IntrinsicGraphBuilder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/IntrinsicGraphBuilder.java index 348ad6e10b4..ba576eaf5f6 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/IntrinsicGraphBuilder.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/IntrinsicGraphBuilder.java @@ -280,6 +280,11 @@ public class IntrinsicGraphBuilder implements GraphBuilderContext, Receiver { throw GraalError.shouldNotReachHere(); } + @Override + public boolean intrinsify(ResolvedJavaMethod targetMethod, StructuredGraph substituteGraph, Receiver receiver, ValueNode[] argsIncludingReceiver) { + throw GraalError.shouldNotReachHere(); + } + @Override public String toString() { return String.format("%s:intrinsic", method.format("%H.%n(%p)")); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/PEGraphDecoder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/PEGraphDecoder.java index 9c8428d04f1..00b744e777f 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/PEGraphDecoder.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/PEGraphDecoder.java @@ -95,6 +95,7 @@ import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins; import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins.InvocationPluginReceiver; import org.graalvm.compiler.nodes.graphbuilderconf.LoopExplosionPlugin; import org.graalvm.compiler.nodes.graphbuilderconf.LoopExplosionPlugin.LoopExplosionKind; +import org.graalvm.compiler.nodes.graphbuilderconf.MethodSubstitutionPlugin; import org.graalvm.compiler.nodes.graphbuilderconf.NodePlugin; import org.graalvm.compiler.nodes.graphbuilderconf.ParameterPlugin; import org.graalvm.compiler.nodes.java.LoadFieldNode; @@ -346,7 +347,7 @@ public abstract class PEGraphDecoder extends SimplifyingGraphDecoder { @Override public IntrinsicContext getIntrinsic() { - return null; + return PEGraphDecoder.this.getIntrinsic(); } @Override @@ -374,6 +375,11 @@ public abstract class PEGraphDecoder extends SimplifyingGraphDecoder { return false; } + @Override + public boolean intrinsify(ResolvedJavaMethod targetMethod, StructuredGraph substituteGraph, InvocationPlugin.Receiver receiver, ValueNode[] argsIncludingReceiver) { + return false; + } + @Override public void setStateAfter(StateSplit stateSplit) { throw unimplemented(); @@ -422,6 +428,10 @@ public abstract class PEGraphDecoder extends SimplifyingGraphDecoder { } } + protected IntrinsicContext getIntrinsic() { + return null; + } + protected class PEAppendGraphBuilderContext extends PENonAppendGraphBuilderContext { protected FixedWithNextNode lastInstr; protected ValueNode pushedNode; @@ -523,6 +533,11 @@ public abstract class PEGraphDecoder extends SimplifyingGraphDecoder { appendInvoke(methodScope.caller, methodScope.callerLoopScope, methodScope.invokeData, callTarget); updateLastInstruction(invoke.asNode()); } + + @Override + public GraphBuilderContext getNonIntrinsicAncestor() { + return null; + } } @NodeInfo(cycles = CYCLES_IGNORED, size = SIZE_IGNORED) @@ -829,10 +844,10 @@ public abstract class PEGraphDecoder extends SimplifyingGraphDecoder { for (InlineInvokePlugin plugin : inlineInvokePlugins) { InlineInfo inlineInfo = plugin.shouldInlineInvoke(graphBuilderContext, targetMethod, arguments); if (inlineInfo != null) { - if (inlineInfo.getMethodToInline() == null) { - return null; - } else { + if (inlineInfo.allowsInlining()) { return doInline(methodScope, loopScope, invokeData, inlineInfo, arguments); + } else { + return null; } } } @@ -844,9 +859,7 @@ public abstract class PEGraphDecoder extends SimplifyingGraphDecoder { return null; } ResolvedJavaMethod inlineMethod = inlineInfo.getMethodToInline(); - ResolvedJavaMethod originalMethod = inlineInfo.getOriginalMethod(); - boolean isSubstitution = originalMethod != null && !originalMethod.equals(inlineMethod); - EncodedGraph graphToInline = lookupEncodedGraph(inlineMethod, originalMethod, inlineInfo.getIntrinsicBytecodeProvider(), isSubstitution, graph.trackNodeSourcePosition()); + EncodedGraph graphToInline = lookupEncodedGraph(inlineMethod, inlineInfo.getPlugin(), inlineInfo.getIntrinsicBytecodeProvider(), inlineInfo.isSubstitution(), graph.trackNodeSourcePosition()); if (graphToInline == null) { return null; } @@ -1115,7 +1128,7 @@ public abstract class PEGraphDecoder extends SimplifyingGraphDecoder { } } - protected abstract EncodedGraph lookupEncodedGraph(ResolvedJavaMethod method, ResolvedJavaMethod originalMethod, BytecodeProvider intrinsicBytecodeProvider, boolean isSubstitution, + protected abstract EncodedGraph lookupEncodedGraph(ResolvedJavaMethod method, MethodSubstitutionPlugin plugin, BytecodeProvider intrinsicBytecodeProvider, boolean isSubstitution, boolean trackNodeSourcePosition); @Override diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/ReplacementsImpl.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/ReplacementsImpl.java index 9479914c58b..82fedc2308a 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/ReplacementsImpl.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/ReplacementsImpl.java @@ -27,9 +27,11 @@ package org.graalvm.compiler.replacements; import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; import static org.graalvm.compiler.core.common.GraalOptions.UseSnippetGraphCache; import static org.graalvm.compiler.debug.DebugContext.DEFAULT_LOG_STREAM; +import static org.graalvm.compiler.debug.DebugOptions.DebugStubsAndSnippets; import static org.graalvm.compiler.java.BytecodeParserOptions.InlineDuringParsing; import static org.graalvm.compiler.java.BytecodeParserOptions.InlineIntrinsicsDuringParsing; import static org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin.InlineInfo.createIntrinsicInlineInfo; +import static org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin.InlineInfo.createMethodSubstitutionInlineInfo; import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.INLINE_AFTER_PARSING; import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.ROOT_COMPILATION; import static org.graalvm.compiler.phases.common.DeadCodeEliminationPhase.Optionality.Required; @@ -80,9 +82,6 @@ import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin; import org.graalvm.compiler.nodes.graphbuilderconf.MethodSubstitutionPlugin; import org.graalvm.compiler.nodes.java.MethodCallTargetNode; import org.graalvm.compiler.nodes.spi.Replacements; -import org.graalvm.compiler.options.Option; -import org.graalvm.compiler.options.OptionKey; -import org.graalvm.compiler.options.OptionType; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.OptimisticOptimizations; import org.graalvm.compiler.phases.common.CanonicalizerPhase; @@ -99,15 +98,7 @@ import jdk.vm.ci.meta.ResolvedJavaType; public class ReplacementsImpl implements Replacements, InlineInvokePlugin { - public static class Options { - // @formatter:off - @Option(help = "This is a testing option to exercise the SymbolicSnippetEncoder", type = OptionType.Expert) - public static final OptionKey UseEncodedSnippets = new OptionKey<>(false); - // @formatter:on - } - - protected final OptionValues options; - + @Override public Providers getProviders() { return providers; } @@ -122,11 +113,6 @@ public class ReplacementsImpl implements Replacements, InlineInvokePlugin { private GraphBuilderConfiguration.Plugins graphBuilderPlugins; private final DebugHandlersFactory debugHandlersFactory; - @Override - public OptionValues getOptions() { - return options; - } - /** * The preprocessed replacement graphs. */ @@ -170,11 +156,11 @@ public class ReplacementsImpl implements Replacements, InlineInvokePlugin { */ @Override public InlineInfo shouldInlineInvoke(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode[] args) { - Bytecode subst = getSubstitutionBytecode(method); - if (subst != null) { + MethodSubstitutionPlugin msPlugin = getMethodSubstitution(method); + if (msPlugin != null) { if (b.parsingIntrinsic() || InlineDuringParsing.getValue(b.getOptions()) || InlineIntrinsicsDuringParsing.getValue(b.getOptions())) { // Forced inlining of intrinsics - return createIntrinsicInlineInfo(subst.getMethod(), method, subst.getOrigin()); + return createMethodSubstitutionInlineInfo(method, msPlugin); } return null; } @@ -182,7 +168,7 @@ public class ReplacementsImpl implements Replacements, InlineInvokePlugin { assert b.getDepth() < MAX_GRAPH_INLINING_DEPTH : "inlining limit exceeded"; // Force inlining when parsing replacements - return createIntrinsicInlineInfo(method, null, defaultBytecodeProvider); + return createIntrinsicInlineInfo(method, defaultBytecodeProvider); } else { assert IS_BUILDING_NATIVE_IMAGE || method.getAnnotation(NodeIntrinsic.class) == null : String.format("@%s method %s must only be called from within a replacement%n%s", NodeIntrinsic.class.getSimpleName(), @@ -215,9 +201,8 @@ public class ReplacementsImpl implements Replacements, InlineInvokePlugin { // it is stable across VM executions (in support of replay compilation). private final EconomicMap snippetTemplateCache; - public ReplacementsImpl(OptionValues options, DebugHandlersFactory debugHandlersFactory, Providers providers, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider, + public ReplacementsImpl(DebugHandlersFactory debugHandlersFactory, Providers providers, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider, TargetDescription target) { - this.options = options; this.providers = providers.copyWith(this); this.snippetReflection = snippetReflection; this.target = target; @@ -232,24 +217,28 @@ public class ReplacementsImpl implements Replacements, InlineInvokePlugin { private static final AtomicInteger nextDebugContextId = new AtomicInteger(); - public DebugContext openDebugContext(String idPrefix, ResolvedJavaMethod method) { - DebugContext outer = DebugContext.forCurrentThread(); - Description description = new Description(method, idPrefix + nextDebugContextId.incrementAndGet()); - List factories = debugHandlersFactory == null ? Collections.emptyList() : Collections.singletonList(debugHandlersFactory); - return DebugContext.create(options, description, outer.getGlobalMetrics(), DEFAULT_LOG_STREAM, factories); + public DebugContext openDebugContext(String idPrefix, ResolvedJavaMethod method, OptionValues options) { + if (DebugStubsAndSnippets.getValue(options)) { + DebugContext outer = DebugContext.forCurrentThread(); + Description description = new Description(method, idPrefix + nextDebugContextId.incrementAndGet()); + List factories = debugHandlersFactory == null ? Collections.emptyList() : Collections.singletonList(debugHandlersFactory); + return DebugContext.create(options, description, outer.getGlobalMetrics(), DEFAULT_LOG_STREAM, factories); + } + return DebugContext.disabled(options); } @Override @SuppressWarnings("try") - public StructuredGraph getSnippet(ResolvedJavaMethod method, ResolvedJavaMethod recursiveEntry, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) { + public StructuredGraph getSnippet(ResolvedJavaMethod method, ResolvedJavaMethod recursiveEntry, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition, + OptionValues options) { assert method.getAnnotation(Snippet.class) != null : "Snippet must be annotated with @" + Snippet.class.getSimpleName(); assert method.hasBytecodes() : "Snippet must not be abstract or native"; StructuredGraph graph = UseSnippetGraphCache.getValue(options) ? graphs.get(method) : null; if (graph == null || (trackNodeSourcePosition && !graph.trackNodeSourcePosition())) { - try (DebugContext debug = openDebugContext("Snippet_", method); + try (DebugContext debug = openDebugContext("Snippet_", method, options); DebugCloseable a = SnippetPreparationTime.start(debug)) { - StructuredGraph newGraph = makeGraph(debug, defaultBytecodeProvider, method, args, recursiveEntry, trackNodeSourcePosition, replaceePosition); + StructuredGraph newGraph = makeGraph(debug, defaultBytecodeProvider, method, args, recursiveEntry, trackNodeSourcePosition, replaceePosition, INLINE_AFTER_PARSING); DebugContext.counter("SnippetNodeCount[%#s]", method).add(newGraph.getDebug(), newGraph.getNodeCount()); if (!UseSnippetGraphCache.getValue(options) || args != null) { return newGraph; @@ -268,10 +257,22 @@ public class ReplacementsImpl implements Replacements, InlineInvokePlugin { } @Override - public void registerSnippet(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean trackNodeSourcePosition) { + public void registerSnippet(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean trackNodeSourcePosition, OptionValues options) { // No initialization needed as snippet graphs are created on demand in getSnippet } + @Override + public StructuredGraph getMethodSubstitution(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original, IntrinsicContext.CompilationContext context, + StructuredGraph.AllowAssumptions allowAssumptions, OptionValues options) { + // Method substitutions are parsed by the BytecodeParser. + return null; + } + + @Override + public void registerMethodSubstitution(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original, IntrinsicContext.CompilationContext context, OptionValues options) { + // No initialization needed as method substitutions are parsed by the BytecodeParser. + } + @Override public boolean hasSubstitution(ResolvedJavaMethod method, int invokeBci) { InvocationPlugin plugin = graphBuilderPlugins.getInvocationPlugins().lookupInvocation(method); @@ -283,19 +284,17 @@ public class ReplacementsImpl implements Replacements, InlineInvokePlugin { return defaultBytecodeProvider; } - @Override - public Bytecode getSubstitutionBytecode(ResolvedJavaMethod method) { + protected MethodSubstitutionPlugin getMethodSubstitution(ResolvedJavaMethod method) { InvocationPlugin plugin = graphBuilderPlugins.getInvocationPlugins().lookupInvocation(method); if (plugin instanceof MethodSubstitutionPlugin) { MethodSubstitutionPlugin msPlugin = (MethodSubstitutionPlugin) plugin; - ResolvedJavaMethod substitute = msPlugin.getSubstitute(providers.getMetaAccess()); - return msPlugin.getBytecodeProvider().getBytecode(substitute); + return msPlugin; } return null; } @Override - public StructuredGraph getSubstitution(ResolvedJavaMethod method, int invokeBci, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) { + public StructuredGraph getSubstitution(ResolvedJavaMethod method, int invokeBci, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition, OptionValues options) { StructuredGraph result; InvocationPlugin plugin = graphBuilderPlugins.getInvocationPlugins().lookupInvocation(method); if (plugin != null && (!plugin.inlineOnly() || invokeBci >= 0)) { @@ -305,8 +304,8 @@ public class ReplacementsImpl implements Replacements, InlineInvokePlugin { ResolvedJavaMethod substitute = msPlugin.getSubstitute(metaAccess); StructuredGraph graph = UseSnippetGraphCache.getValue(options) ? graphs.get(substitute) : null; if (graph == null || graph.trackNodeSourcePosition() != trackNodeSourcePosition) { - try (DebugContext debug = openDebugContext("Substitution_", method)) { - graph = makeGraph(debug, msPlugin.getBytecodeProvider(), substitute, null, method, trackNodeSourcePosition, replaceePosition); + try (DebugContext debug = openDebugContext("Substitution_", method, options)) { + graph = makeGraph(debug, msPlugin.getBytecodeProvider(), substitute, null, method, trackNodeSourcePosition, replaceePosition, INLINE_AFTER_PARSING); if (!UseSnippetGraphCache.getValue(options)) { return graph; } @@ -319,7 +318,7 @@ public class ReplacementsImpl implements Replacements, InlineInvokePlugin { result = graph; } else { Bytecode code = new ResolvedJavaMethodBytecode(method); - try (DebugContext debug = openDebugContext("Substitution_", method)) { + try (DebugContext debug = openDebugContext("Substitution_", method, options)) { result = new IntrinsicGraphBuilder(options, debug, providers, code, invokeBci).buildGraph(plugin); } } @@ -332,13 +331,13 @@ public class ReplacementsImpl implements Replacements, InlineInvokePlugin { @SuppressWarnings("try") @Override public StructuredGraph getIntrinsicGraph(ResolvedJavaMethod method, CompilationIdentifier compilationId, DebugContext debug) { - Bytecode subst = getSubstitutionBytecode(method); - if (subst != null) { - ResolvedJavaMethod substMethod = subst.getMethod(); + MethodSubstitutionPlugin msPlugin = getMethodSubstitution(method); + if (msPlugin != null) { + ResolvedJavaMethod substMethod = msPlugin.getSubstitute(providers.getMetaAccess()); assert !substMethod.equals(method); - BytecodeProvider bytecodeProvider = subst.getOrigin(); + BytecodeProvider bytecodeProvider = msPlugin.getBytecodeProvider(); // @formatter:off - StructuredGraph graph = new StructuredGraph.Builder(options, debug, StructuredGraph.AllowAssumptions.YES). + StructuredGraph graph = new StructuredGraph.Builder(debug.getOptions(), debug, StructuredGraph.AllowAssumptions.YES). method(substMethod). compilationId(compilationId). recordInlinedMethods(bytecodeProvider.shouldRecordMethodDependencies()). @@ -367,11 +366,32 @@ public class ReplacementsImpl implements Replacements, InlineInvokePlugin { * @param args * @param original the original method if {@code method} is a {@linkplain MethodSubstitution * substitution} otherwise null - * @param trackNodeSourcePosition + * @param trackNodeSourcePosition record source information + * @param context + * {@link org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext + * compilation context} for the graph */ public StructuredGraph makeGraph(DebugContext debug, BytecodeProvider bytecodeProvider, ResolvedJavaMethod method, Object[] args, ResolvedJavaMethod original, boolean trackNodeSourcePosition, - NodeSourcePosition replaceePosition) { - return createGraphMaker(method, original).makeGraph(debug, bytecodeProvider, args, trackNodeSourcePosition, replaceePosition); + NodeSourcePosition replaceePosition, IntrinsicContext.CompilationContext context) { + return createGraphMaker(method, original).makeGraph(debug, bytecodeProvider, args, trackNodeSourcePosition, replaceePosition, context); + } + + /** + * Creates a preprocessed graph for a snippet or method substitution with a context of . + * {@link org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext#INLINE_AFTER_PARSING} + * . + * + * + * @param bytecodeProvider how to access the bytecode of {@code method} + * @param method the snippet or method substitution for which a graph will be created + * @param args + * @param original the original method if {@code method} is a {@linkplain MethodSubstitution + * substitution} otherwise null + * @param trackNodeSourcePosition record source information + */ + public final StructuredGraph makeGraph(DebugContext debug, BytecodeProvider bytecodeProvider, ResolvedJavaMethod method, Object[] args, ResolvedJavaMethod original, + boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) { + return makeGraph(debug, bytecodeProvider, method, args, original, trackNodeSourcePosition, replaceePosition, INLINE_AFTER_PARSING); } /** @@ -408,10 +428,11 @@ public class ReplacementsImpl implements Replacements, InlineInvokePlugin { } @SuppressWarnings("try") - public StructuredGraph makeGraph(DebugContext debug, BytecodeProvider bytecodeProvider, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) { + public StructuredGraph makeGraph(DebugContext debug, BytecodeProvider bytecodeProvider, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition, + IntrinsicContext.CompilationContext context) { try (DebugContext.Scope s = debug.scope("BuildSnippetGraph", method)) { assert method.hasBytecodes() : method; - StructuredGraph graph = buildInitialGraph(debug, bytecodeProvider, method, args, trackNodeSourcePosition, replaceePosition); + StructuredGraph graph = buildInitialGraph(debug, bytecodeProvider, method, args, trackNodeSourcePosition, replaceePosition, context); finalizeGraph(graph); @@ -427,7 +448,7 @@ public class ReplacementsImpl implements Replacements, InlineInvokePlugin { * Does final processing of a snippet graph. */ protected void finalizeGraph(StructuredGraph graph) { - if (!GraalOptions.SnippetCounters.getValue(replacements.options) || graph.getNodes().filter(SnippetCounterNode.class).isEmpty()) { + if (!GraalOptions.SnippetCounters.getValue(graph.getOptions()) || graph.getNodes().filter(SnippetCounterNode.class).isEmpty()) { int sideEffectCount = 0; assert (sideEffectCount = graph.getNodes().filter(e -> hasSideEffect(e)).count()) >= 0; new ConvertDeoptimizeToGuardPhase().apply(graph, null); @@ -471,16 +492,39 @@ public class ReplacementsImpl implements Replacements, InlineInvokePlugin { return false; } + static class EncodedIntrinsicContext extends IntrinsicContext { + EncodedIntrinsicContext(ResolvedJavaMethod method, ResolvedJavaMethod intrinsic, BytecodeProvider bytecodeProvider, CompilationContext compilationContext, + boolean allowPartialIntrinsicArgumentMismatch) { + super(method, intrinsic, bytecodeProvider, compilationContext, allowPartialIntrinsicArgumentMismatch); + } + + @Override + public boolean isDeferredInvoke(StateSplit stateSplit) { + if (stateSplit instanceof Invoke) { + Invoke invoke = (Invoke) stateSplit; + ResolvedJavaMethod method = invoke.callTarget().targetMethod(); + if (method.getAnnotation(Fold.class) != null) { + return true; + } + Node.NodeIntrinsic annotation = method.getAnnotation(Node.NodeIntrinsic.class); + if (annotation != null && !annotation.hasSideEffect()) { + return true; + } + } + return false; + } + } + /** * Builds the initial graph for a replacement. */ @SuppressWarnings("try") protected StructuredGraph buildInitialGraph(DebugContext debug, BytecodeProvider bytecodeProvider, final ResolvedJavaMethod methodToParse, Object[] args, boolean trackNodeSourcePosition, - NodeSourcePosition replaceePosition) { + NodeSourcePosition replaceePosition, IntrinsicContext.CompilationContext context) { // @formatter:off // Replacements cannot have optimistic assumptions since they have // to be valid for the entire run of the VM. - final StructuredGraph graph = new StructuredGraph.Builder(replacements.options, debug). + final StructuredGraph graph = new StructuredGraph.Builder(debug.getOptions(), debug). method(methodToParse). trackNodeSourcePosition(trackNodeSourcePosition). callerContext(replaceePosition). @@ -506,11 +550,11 @@ public class ReplacementsImpl implements Replacements, InlineInvokePlugin { MethodSubstitution methodAnnotation = method.getAnnotation(MethodSubstitution.class); if (methodAnnotation == null && snippetAnnotation == null) { // Post-parse inlined intrinsic - initialIntrinsicContext = new IntrinsicContext(substitutedMethod, method, bytecodeProvider, INLINE_AFTER_PARSING); + initialIntrinsicContext = new EncodedIntrinsicContext(substitutedMethod, method, bytecodeProvider, context, false); } else { // Snippet ResolvedJavaMethod original = substitutedMethod != null ? substitutedMethod : method; - initialIntrinsicContext = new IntrinsicContext(original, method, bytecodeProvider, INLINE_AFTER_PARSING, + initialIntrinsicContext = new EncodedIntrinsicContext(original, method, bytecodeProvider, context, snippetAnnotation != null ? snippetAnnotation.allowPartialIntrinsicArgumentMismatch() : true); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetTemplate.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetTemplate.java index 236fccd9427..a9f34891ad4 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetTemplate.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetTemplate.java @@ -675,7 +675,7 @@ public class SnippetTemplate { ResolvedJavaMethod javaMethod = findMethod(providers.getMetaAccess(), declaringClass, methodName); assert javaMethod != null : "did not find @" + Snippet.class.getSimpleName() + " method in " + declaringClass + " named " + methodName; assert javaMethod.getAnnotation(Snippet.class) != null : javaMethod + " must be annotated with @" + Snippet.class.getSimpleName(); - providers.getReplacements().registerSnippet(javaMethod, original, receiver, GraalOptions.TrackNodeSourcePosition.getValue(options)); + providers.getReplacements().registerSnippet(javaMethod, original, receiver, GraalOptions.TrackNodeSourcePosition.getValue(options), options); LocationIdentity[] privateLocations = GraalOptions.SnippetCounters.getValue(options) ? SnippetCounterNode.addSnippetCounters(initialPrivateLocations) : initialPrivateLocations; if (GraalOptions.EagerSnippets.getValue(options)) { return new EagerSnippetInfo(javaMethod, original, privateLocations, receiver); @@ -691,7 +691,7 @@ public class SnippetTemplate { Description description = new Description(args.cacheKey.method, "SnippetTemplate_" + nextSnippetTemplateId.incrementAndGet()); return DebugContext.create(options, description, outer.getGlobalMetrics(), DEFAULT_LOG_STREAM, factories); } - return DebugContext.DISABLED; + return DebugContext.disabled(options); } /** @@ -764,7 +764,8 @@ public class SnippetTemplate { Object[] constantArgs = getConstantArgs(args); boolean shouldTrackNodeSourcePosition1 = trackNodeSourcePosition || (providers.getCodeCache() != null && providers.getCodeCache().shouldDebugNonSafepoints()); - StructuredGraph snippetGraph = providers.getReplacements().getSnippet(args.info.method, args.info.original, constantArgs, shouldTrackNodeSourcePosition1, replacee.getNodeSourcePosition()); + StructuredGraph snippetGraph = providers.getReplacements().getSnippet(args.info.method, args.info.original, constantArgs, shouldTrackNodeSourcePosition1, replacee.getNodeSourcePosition(), + options); ResolvedJavaMethod method = snippetGraph.method(); Signature signature = method.getSignature(); @@ -965,7 +966,9 @@ public class SnippetTemplate { } retNode.setMemoryMap(null); } - memoryMap.safeDelete(); + if (memoryMap != null) { + memoryMap.safeDelete(); + } } if (needsAnchor) { snippetCopy.addAfterFixed(snippetCopy.start(), anchor); @@ -1484,55 +1487,6 @@ public class SnippetTemplate { rewireFrameStates(replacee, duplicates); - if (replacee instanceof DeoptimizingNode) { - DeoptimizingNode replaceeDeopt = (DeoptimizingNode) replacee; - - FrameState stateBefore = null; - FrameState stateDuring = null; - FrameState stateAfter = null; - if (replaceeDeopt.canDeoptimize()) { - if (replaceeDeopt instanceof DeoptimizingNode.DeoptBefore) { - stateBefore = ((DeoptimizingNode.DeoptBefore) replaceeDeopt).stateBefore(); - } - if (replaceeDeopt instanceof DeoptimizingNode.DeoptDuring) { - stateDuring = ((DeoptimizingNode.DeoptDuring) replaceeDeopt).stateDuring(); - } - if (replaceeDeopt instanceof DeoptimizingNode.DeoptAfter) { - stateAfter = ((DeoptimizingNode.DeoptAfter) replaceeDeopt).stateAfter(); - } - } - - for (DeoptimizingNode deoptNode : deoptNodes) { - DeoptimizingNode deoptDup = (DeoptimizingNode) duplicates.get(deoptNode.asNode()); - if (deoptDup.canDeoptimize()) { - if (deoptDup instanceof DeoptimizingNode.DeoptBefore) { - ((DeoptimizingNode.DeoptBefore) deoptDup).setStateBefore(stateBefore); - } - if (deoptDup instanceof DeoptimizingNode.DeoptDuring) { - DeoptimizingNode.DeoptDuring deoptDupDuring = (DeoptimizingNode.DeoptDuring) deoptDup; - if (stateDuring != null) { - deoptDupDuring.setStateDuring(stateDuring); - } else if (stateAfter != null) { - deoptDupDuring.computeStateDuring(stateAfter); - } else if (stateBefore != null) { - assert !deoptDupDuring.hasSideEffect() : "can't use stateBefore as stateDuring for state split " + deoptDupDuring; - deoptDupDuring.setStateDuring(stateBefore); - } - } - if (deoptDup instanceof DeoptimizingNode.DeoptAfter) { - DeoptimizingNode.DeoptAfter deoptDupAfter = (DeoptimizingNode.DeoptAfter) deoptDup; - if (stateAfter != null) { - deoptDupAfter.setStateAfter(stateAfter); - } else { - assert !deoptDupAfter.hasSideEffect() : "can't use stateBefore as stateAfter for state split " + deoptDupAfter; - deoptDupAfter.setStateAfter(stateBefore); - } - - } - } - } - } - updateStamps(replacee, duplicates); rewireMemoryGraph(replacee, duplicates); @@ -1656,7 +1610,8 @@ public class SnippetTemplate { FixedNode firstCFGNodeDuplicate = (FixedNode) duplicates.get(firstCFGNode); replaceeGraph.addAfterFixed(lastFixedNode, firstCFGNodeDuplicate); - rewireFrameStates(replacee, duplicates); + // floating nodes are not state-splits not need to re-wire frame states + assert !(replacee instanceof StateSplit); updateStamps(replacee, duplicates); rewireMemoryGraph(replacee, duplicates); @@ -1710,7 +1665,8 @@ public class SnippetTemplate { } UnmodifiableEconomicMap duplicates = inlineSnippet(replacee, debug, replaceeGraph, replacements); - rewireFrameStates(replacee, duplicates); + // floating nodes are not state-splits not need to re-wire frame states + assert !(replacee instanceof StateSplit); updateStamps(replacee, duplicates); rewireMemoryGraph(replacee, duplicates); @@ -1725,12 +1681,64 @@ public class SnippetTemplate { } protected void rewireFrameStates(ValueNode replacee, UnmodifiableEconomicMap duplicates) { - if (replacee instanceof StateSplit) { + if (replacee.graph().getGuardsStage().areFrameStatesAtSideEffects() && replacee instanceof StateSplit) { for (StateSplit sideEffectNode : sideEffectNodes) { assert ((StateSplit) replacee).hasSideEffect(); Node sideEffectDup = duplicates.get(sideEffectNode.asNode()); ((StateSplit) sideEffectDup).setStateAfter(((StateSplit) replacee).stateAfter()); } + } else if (replacee.graph().getGuardsStage().areFrameStatesAtDeopts() && replacee instanceof DeoptimizingNode) { + DeoptimizingNode replaceeDeopt = (DeoptimizingNode) replacee; + + FrameState stateBefore = null; + FrameState stateDuring = null; + FrameState stateAfter = null; + if (replaceeDeopt.canDeoptimize()) { + if (replaceeDeopt instanceof DeoptimizingNode.DeoptBefore) { + stateBefore = ((DeoptimizingNode.DeoptBefore) replaceeDeopt).stateBefore(); + } + if (replaceeDeopt instanceof DeoptimizingNode.DeoptDuring) { + stateDuring = ((DeoptimizingNode.DeoptDuring) replaceeDeopt).stateDuring(); + } + if (replaceeDeopt instanceof DeoptimizingNode.DeoptAfter) { + stateAfter = ((DeoptimizingNode.DeoptAfter) replaceeDeopt).stateAfter(); + } + } + + for (DeoptimizingNode deoptNode : deoptNodes) { + DeoptimizingNode deoptDup = (DeoptimizingNode) duplicates.get(deoptNode.asNode()); + if (deoptDup.canDeoptimize()) { + if (deoptDup instanceof DeoptimizingNode.DeoptBefore) { + ((DeoptimizingNode.DeoptBefore) deoptDup).setStateBefore(stateBefore); + } + if (deoptDup instanceof DeoptimizingNode.DeoptDuring) { + // compute a state "during" for a DeoptDuring inside the snippet depending + // on what kind of states we had on the node we are replacing. + // If the original node had a state "during" already, we just use that, + // otherwise we need to find a strategy to compute a state during based on + // some other state (before or after). + DeoptimizingNode.DeoptDuring deoptDupDuring = (DeoptimizingNode.DeoptDuring) deoptDup; + if (stateDuring != null) { + deoptDupDuring.setStateDuring(stateDuring); + } else if (stateAfter != null) { + deoptDupDuring.computeStateDuring(stateAfter); + } else if (stateBefore != null) { + assert !deoptDupDuring.hasSideEffect() : "can't use stateBefore as stateDuring for state split " + deoptDupDuring; + deoptDupDuring.setStateDuring(stateBefore); + } + } + if (deoptDup instanceof DeoptimizingNode.DeoptAfter) { + DeoptimizingNode.DeoptAfter deoptDupAfter = (DeoptimizingNode.DeoptAfter) deoptDup; + if (stateAfter != null) { + deoptDupAfter.setStateAfter(stateAfter); + } else { + assert !deoptDupAfter.hasSideEffect() : "can't use stateBefore as stateAfter for state split " + deoptDupAfter; + deoptDupAfter.setStateAfter(stateBefore); + } + + } + } + } } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyCallNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyCallNode.java index 015987dad4e..e3acc823db9 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyCallNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyCallNode.java @@ -188,11 +188,11 @@ public final class ArrayCopyCallNode extends AbstractMemoryCheckpoint implements return locationIdentity; } - @NodeIntrinsic + @NodeIntrinsic(hasSideEffect = true) private static native void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter JavaKind elementKind, @ConstantNodeParameter boolean aligned, @ConstantNodeParameter boolean disjoint, @ConstantNodeParameter boolean uninitialized, @ConstantNodeParameter int heapWordSize); - @NodeIntrinsic + @NodeIntrinsic(hasSideEffect = true) private static native void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter JavaKind elementKind, @ConstantNodeParameter LocationIdentity locationIdentity, @ConstantNodeParameter boolean aligned, @ConstantNodeParameter boolean disjoint, @ConstantNodeParameter boolean uninitialized, @ConstantNodeParameter int heapWordSize); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopySnippets.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopySnippets.java index c9f2eb70680..e61b7438774 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopySnippets.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopySnippets.java @@ -49,9 +49,10 @@ import org.graalvm.compiler.nodes.InvokeWithExceptionNode; import org.graalvm.compiler.nodes.NamedLocationIdentity; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.PiNode; +import org.graalvm.compiler.nodes.SnippetAnchorNode; import org.graalvm.compiler.nodes.StructuredGraph; -import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.extended.RawLoadNode; +import org.graalvm.compiler.nodes.extended.GuardedUnsafeLoadNode; +import org.graalvm.compiler.nodes.extended.GuardingNode; import org.graalvm.compiler.nodes.extended.RawStoreNode; import org.graalvm.compiler.nodes.java.ArrayLengthNode; import org.graalvm.compiler.nodes.spi.LoweringTool; @@ -69,7 +70,6 @@ import org.graalvm.compiler.replacements.SnippetTemplate.Arguments; import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo; import org.graalvm.compiler.replacements.Snippets; import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode; -import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode; import org.graalvm.compiler.word.Word; import jdk.internal.vm.compiler.word.LocationIdentity; import jdk.internal.vm.compiler.word.Pointer; @@ -108,19 +108,10 @@ public abstract class ArrayCopySnippets implements Snippets { protected abstract int heapWordSize(); - @Snippet - public void arraycopyZeroLengthSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, - @ConstantParameter Counters counters) { - Object nonNullSrc = GraalDirectives.guardingNonNull(src); - Object nonNullDest = GraalDirectives.guardingNonNull(dest); - this.checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); - checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); - counters.zeroLengthStaticCounter.inc(); - } - + @SuppressWarnings("unused") @Snippet public void arraycopyExactSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, - @ConstantParameter JavaKind elementKind, @ConstantParameter LocationIdentity locationIdentity, + @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind, @ConstantParameter LocationIdentity locationIdentity, @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter, @ConstantParameter Counters counters) { Object nonNullSrc = GraalDirectives.guardingNonNull(src); Object nonNullDest = GraalDirectives.guardingNonNull(dest); @@ -130,31 +121,37 @@ public abstract class ArrayCopySnippets implements Snippets { elementKindCounter.inc(); elementKindCopiedCounter.add(length); + + ArrayCopyWithDelayedLoweringNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind); + } + + @SuppressWarnings("unused") + @Snippet + public void arraycopyExactStubCallSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, + @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind, @ConstantParameter LocationIdentity locationIdentity, + @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter, @ConstantParameter Counters counters) { + Object nonNullSrc = GraalDirectives.guardingNonNull(src); + Object nonNullDest = GraalDirectives.guardingNonNull(dest); + checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); + incrementLengthCounter(length, counters); + + elementKindCounter.inc(); + elementKindCopiedCounter.add(length); + ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind, locationIdentity, heapWordSize()); } @Snippet - public void arraycopyUnrolledSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, - @ConstantParameter JavaKind elementKind, @ConstantParameter LocationIdentity locationIdentity, @ConstantParameter int unrolledLength, @ConstantParameter Counters counters) { + public void arraycopyCheckcastSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, @ConstantParameter Counters counters, + @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) { Object nonNullSrc = GraalDirectives.guardingNonNull(src); Object nonNullDest = GraalDirectives.guardingNonNull(dest); checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); incrementLengthCounter(length, counters); - unrolledArraycopyWork(nonNullSrc, srcPos, nonNullDest, destPos, unrolledLength, elementKind, locationIdentity); - } - - @Snippet - public void arraycopyCheckcastSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, - @ConstantParameter Counters counters, @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) { - Object nonNullSrc = GraalDirectives.guardingNonNull(src); - Object nonNullDest = GraalDirectives.guardingNonNull(dest); - checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); - checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); - incrementLengthCounter(length, counters); - - ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind); + ArrayCopyWithDelayedLoweringNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind); } @Snippet @@ -166,7 +163,7 @@ public abstract class ArrayCopySnippets implements Snippets { checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); incrementLengthCounter(length, counters); - ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind); + ArrayCopyWithDelayedLoweringNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind); } @Snippet @@ -179,31 +176,35 @@ public abstract class ArrayCopySnippets implements Snippets { System.arraycopy(src, srcPos, dest, destPos, length); } - private static void unrolledArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, JavaKind elementKind, LocationIdentity arrayLocation) { + @SuppressWarnings("unused") + @Snippet(allowPartialIntrinsicArgumentMismatch = true) + public void exactArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, @ConstantParameter LocationIdentity arrayLocation, + @ConstantParameter Counters counters) { int scale = ReplacementsUtil.arrayIndexScale(INJECTED_META_ACCESS, elementKind); int arrayBaseOffset = ReplacementsUtil.getArrayBaseOffset(INJECTED_META_ACCESS, elementKind); - long sourceOffset = arrayBaseOffset + (long) srcPos * scale; long destOffset = arrayBaseOffset + (long) destPos * scale; - long position = 0; - long delta = scale; - if (probability(NOT_FREQUENT_PROBABILITY, nonNullSrc == nonNullDest && srcPos < destPos)) { - // bad aliased case so we need to copy the array from back to front - position = (long) (length - 1) * scale; - delta = -delta; - } - // the length was already checked before - we can emit unconditional instructions - ExplodeLoopNode.explodeLoop(); - for (int iteration = 0; iteration < length; iteration++) { - Object value = RawLoadNode.load(nonNullSrc, sourceOffset + position, elementKind, arrayLocation); - RawStoreNode.storeObject(nonNullDest, destOffset + position, value, elementKind, arrayLocation, false); - position += delta; + GuardingNode anchor = SnippetAnchorNode.anchor(); + if (probability(NOT_FREQUENT_PROBABILITY, src == dest && srcPos < destPos)) { + // bad aliased case so we need to copy the array from back to front + for (int position = length - 1; position >= 0; position--) { + Object value = GuardedUnsafeLoadNode.guardedLoad(src, sourceOffset + ((long) position) * scale, elementKind, arrayLocation, anchor); + RawStoreNode.storeObject(dest, destOffset + ((long) position) * scale, value, elementKind, arrayLocation, true); + } + } else { + for (int position = 0; position < length; position++) { + Object value = GuardedUnsafeLoadNode.guardedLoad(src, sourceOffset + ((long) position) * scale, elementKind, arrayLocation, anchor); + RawStoreNode.storeObject(dest, destOffset + ((long) position) * scale, value, elementKind, arrayLocation, true); + } } } + @SuppressWarnings("unused") @Snippet(allowPartialIntrinsicArgumentMismatch = true) - public void checkcastArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) { + public void checkcastArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, + @ConstantParameter LocationIdentity arrayLocation, + @ConstantParameter Counters counters) { if (probability(FREQUENT_PROBABILITY, length > 0)) { Object nonNullSrc = PiNode.asNonNullObject(src); Object nonNullDest = PiNode.asNonNullObject(dest); @@ -234,8 +235,11 @@ public abstract class ArrayCopySnippets implements Snippets { } } + @SuppressWarnings("unused") @Snippet(allowPartialIntrinsicArgumentMismatch = true) - public void genericArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) { + public void genericArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, + @ConstantParameter LocationIdentity arrayLocation, + @ConstantParameter Counters counters) { // The length > 0 check should not be placed here because generic array copy stub should // enforce type check. This is fine performance-wise because this snippet is rarely used. counters.genericArraycopyDifferentTypeCounter.inc(); @@ -352,16 +356,17 @@ public abstract class ArrayCopySnippets implements Snippets { public static class Templates extends SnippetTemplate.AbstractTemplates { private final SnippetInfo arraycopyGenericSnippet; - private final SnippetInfo arraycopyUnrolledSnippet; private final SnippetInfo arraycopyExactSnippet; - private final SnippetInfo arraycopyZeroLengthSnippet; + private final SnippetInfo arraycopyExactStubCallSnippet; private final SnippetInfo arraycopyCheckcastSnippet; private final SnippetInfo arraycopyNativeSnippet; private final SnippetInfo checkcastArraycopyWithSlowPathWork; private final SnippetInfo genericArraycopyWithSlowPathWork; + private final SnippetInfo exactArraycopyWithSlowPathWork; private ResolvedJavaMethod originalArraycopy; private final Counters counters; + private boolean expandArraycopyLoop; public Templates(ArrayCopySnippets receiver, OptionValues options, Iterable factories, Factory factory, Providers providers, SnippetReflectionProvider snippetReflection, TargetDescription target) { @@ -369,13 +374,13 @@ public abstract class ArrayCopySnippets implements Snippets { this.counters = new Counters(factory); arraycopyGenericSnippet = snippet(receiver, "arraycopyGenericSnippet"); - arraycopyUnrolledSnippet = snippet(receiver, "arraycopyUnrolledSnippet"); arraycopyExactSnippet = snippet(receiver, "arraycopyExactSnippet"); - arraycopyZeroLengthSnippet = snippet(receiver, "arraycopyZeroLengthSnippet"); + arraycopyExactStubCallSnippet = snippet(receiver, "arraycopyExactStubCallSnippet"); arraycopyCheckcastSnippet = snippet(receiver, "arraycopyCheckcastSnippet"); arraycopyNativeSnippet = snippet(null, "arraycopyNativeSnippet"); checkcastArraycopyWithSlowPathWork = snippet(receiver, "checkcastArraycopyWithSlowPathWork"); genericArraycopyWithSlowPathWork = snippet(receiver, "genericArraycopyWithSlowPathWork"); + exactArraycopyWithSlowPathWork = snippet(receiver, "exactArraycopyWithSlowPathWork"); } protected SnippetInfo snippet(ArrayCopySnippets receiver, String methodName) { @@ -401,7 +406,7 @@ public abstract class ArrayCopySnippets implements Snippets { if (arraycopy.isExact()) { // there is a sufficient type match - we don't need any additional type checks - snippetInfo = arraycopyExactSnippet; + snippetInfo = arraycopyExactStubCallSnippet; arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK; } else if (srcComponentType == null && destComponentType == null) { // we don't know anything about the types - use the generic copying @@ -426,7 +431,7 @@ public abstract class ArrayCopySnippets implements Snippets { if (nonNullComponentType.isPrimitive()) { // one involved object is a primitive array - it is sufficient to directly // compare the hub. - snippetInfo = arraycopyExactSnippet; + snippetInfo = arraycopyExactStubCallSnippet; arrayTypeCheck = ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK; elementKind = nonNullComponentType.getJavaKind(); } else { @@ -438,17 +443,8 @@ public abstract class ArrayCopySnippets implements Snippets { } } - // a few special cases that are easier to handle when all other variables already have a - // value - if (snippetInfo != arraycopyNativeSnippet && snippetInfo != arraycopyGenericSnippet && arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) { - // Copying 0 element between object arrays with conflicting types will not throw an - // exception - once we pass the preliminary element type checks that we are not - // mixing arrays of different basic types, ArrayStoreException is only thrown when - // an *astore would have thrown it. Therefore, copying null between object arrays - // with conflicting types will also succeed (we do not optimize for such case here). - snippetInfo = arraycopyZeroLengthSnippet; - } else if (snippetInfo == arraycopyExactSnippet && shouldUnroll(arraycopy.getLength())) { - snippetInfo = arraycopyUnrolledSnippet; + if (this.expandArraycopyLoop && snippetInfo == arraycopyExactStubCallSnippet) { + snippetInfo = arraycopyExactSnippet; } // create the snippet @@ -463,13 +459,9 @@ public abstract class ArrayCopySnippets implements Snippets { args.addConst("arrayTypeCheck", arrayTypeCheck); } Object locationIdentity = arraycopy.killsAnyLocation() ? LocationIdentity.any() : NamedLocationIdentity.getArrayLocation(elementKind); - if (snippetInfo == arraycopyUnrolledSnippet) { - args.addConst("elementKind", elementKind != null ? elementKind : JavaKind.Illegal); - args.addConst("locationIdentity", locationIdentity); - args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt()); - } - if (snippetInfo == arraycopyExactSnippet) { + if (snippetInfo == arraycopyExactStubCallSnippet || snippetInfo == arraycopyExactSnippet) { assert elementKind != null; + args.addConst("workSnippet", exactArraycopyWithSlowPathWork); args.addConst("elementKind", elementKind); args.addConst("locationIdentity", locationIdentity); args.addConst("elementKindCounter", counters.arraycopyCallCounters.get(elementKind)); @@ -488,11 +480,19 @@ public abstract class ArrayCopySnippets implements Snippets { instantiate(args, arraycopy); } - public void lower(ArrayCopyWithSlowPathNode arraycopy, LoweringTool tool) { + public void lower(ArrayCopyWithDelayedLoweringNode arraycopy, LoweringTool tool) { StructuredGraph graph = arraycopy.graph(); - if (!graph.getGuardsStage().areFrameStatesAtDeopts()) { - // if an arraycopy contains a slow path, we can't lower it right away - return; + + if (arraycopy.getSnippet() == exactArraycopyWithSlowPathWork && this.expandArraycopyLoop) { + if (!graph.getGuardsStage().areDeoptsFixed()) { + // Don't lower until floating guards are fixed. + return; + } + } else { + if (!graph.getGuardsStage().areFrameStatesAtDeopts()) { + // Don't lower until frame states are assigned to deoptimization points. + return; + } } SnippetInfo snippetInfo = arraycopy.getSnippet(); @@ -502,6 +502,12 @@ public abstract class ArrayCopySnippets implements Snippets { args.add("dest", arraycopy.getDestination()); args.add("destPos", arraycopy.getDestinationPosition()); args.add("length", arraycopy.getLength()); + + JavaKind elementKind = arraycopy.getElementKind(); + args.addConst("elementKind", (elementKind == null) ? JavaKind.Illegal : elementKind); + + Object locationIdentity = (elementKind == null) ? LocationIdentity.any() : NamedLocationIdentity.getArrayLocation(arraycopy.getElementKind()); + args.addConst("arrayLocation", locationIdentity); args.addConst("counters", counters); instantiate(args, arraycopy); } @@ -526,10 +532,6 @@ public abstract class ArrayCopySnippets implements Snippets { return srcType.getComponentType().getJavaKind(); } - private static boolean shouldUnroll(ValueNode length) { - return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0; - } - /** * Instantiate the snippet template and fix up the FrameState of any Invokes of * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode. @@ -551,19 +553,19 @@ public abstract class ArrayCopySnippets implements Snippets { throw new GraalError("unexpected invoke %s in snippet", call.targetMethod()); } // Here we need to fix the bci of the invoke - InvokeNode newInvoke = invoke.replaceWithNewBci(arraycopy.getBci()); - newInvoke.setStateDuring(null); - newInvoke.setStateAfter(null); + invoke.replaceBci(arraycopy.getBci()); + invoke.setStateDuring(null); + invoke.setStateAfter(null); if (arraycopy.stateDuring() != null) { - newInvoke.setStateDuring(arraycopy.stateDuring()); + invoke.setStateDuring(arraycopy.stateDuring()); } else { assert arraycopy.stateAfter() != null : arraycopy; - newInvoke.setStateAfter(arraycopy.stateAfter()); + invoke.setStateAfter(arraycopy.stateAfter()); } } else if (originalNode instanceof InvokeWithExceptionNode) { throw new GraalError("unexpected invoke with exception %s in snippet", originalNode); - } else if (originalNode instanceof ArrayCopyWithSlowPathNode) { - ArrayCopyWithSlowPathNode slowPath = (ArrayCopyWithSlowPathNode) replacements.get(originalNode); + } else if (originalNode instanceof ArrayCopyWithDelayedLoweringNode) { + ArrayCopyWithDelayedLoweringNode slowPath = (ArrayCopyWithDelayedLoweringNode) replacements.get(originalNode); assert arraycopy.stateAfter() != null : arraycopy; assert slowPath.stateAfter() == arraycopy.stateAfter(); slowPath.setBci(arraycopy.getBci()); @@ -582,5 +584,9 @@ public abstract class ArrayCopySnippets implements Snippets { } return originalArraycopy; } + + public void setExpandArraycopyLoop(boolean b) { + this.expandArraycopyLoop = b; + } } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyWithSlowPathNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyWithDelayedLoweringNode.java similarity index 82% rename from src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyWithSlowPathNode.java rename to src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyWithDelayedLoweringNode.java index 481e452f46c..87ec9bee297 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyWithSlowPathNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyWithDelayedLoweringNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -36,13 +36,13 @@ import jdk.vm.ci.code.BytecodeFrame; import jdk.vm.ci.meta.JavaKind; @NodeInfo(allowedUsageTypes = InputType.Memory) -public final class ArrayCopyWithSlowPathNode extends BasicArrayCopyNode { +public final class ArrayCopyWithDelayedLoweringNode extends BasicArrayCopyNode { - public static final NodeClass TYPE = NodeClass.create(ArrayCopyWithSlowPathNode.class); + public static final NodeClass TYPE = NodeClass.create(ArrayCopyWithDelayedLoweringNode.class); private final SnippetTemplate.SnippetInfo snippet; - public ArrayCopyWithSlowPathNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, SnippetTemplate.SnippetInfo snippet, JavaKind elementKind) { + public ArrayCopyWithDelayedLoweringNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, SnippetTemplate.SnippetInfo snippet, JavaKind elementKind) { super(TYPE, src, srcPos, dest, destPos, length, elementKind, BytecodeFrame.INVALID_FRAMESTATE_BCI); assert StampTool.isPointerNonNull(src) && StampTool.isPointerNonNull(dest) : "must have been null checked"; this.snippet = snippet; @@ -60,3 +60,4 @@ public final class ArrayCopyWithSlowPathNode extends BasicArrayCopyNode { this.bci = bci; } } + diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BitCountNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BitCountNode.java index 1f76d629df5..bc5d7e049a9 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BitCountNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BitCountNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -46,12 +46,16 @@ import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.JavaKind; @NodeInfo(cycles = CYCLES_2, size = SIZE_1) -public final class BitCountNode extends UnaryNode implements ArithmeticLIRLowerable { +public class BitCountNode extends UnaryNode implements ArithmeticLIRLowerable { public static final NodeClass TYPE = NodeClass.create(BitCountNode.class); public BitCountNode(ValueNode value) { - super(TYPE, computeStamp(value.stamp(NodeView.DEFAULT), value), value); + this(TYPE, value); + } + + public BitCountNode(NodeClass c, ValueNode value) { + super(c, computeStamp(value.stamp(NodeView.DEFAULT), value), value); assert value.getStackKind() == JavaKind.Int || value.getStackKind() == JavaKind.Long; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/GraalServices.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/GraalServices.java index 2748019372c..4f8a6b7b106 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/GraalServices.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/GraalServices.java @@ -41,7 +41,7 @@ import jdk.vm.ci.services.JVMCIPermission; import jdk.vm.ci.services.Services; /** - * Interface to functionality that abstracts over which JDK version Graal is running on. + * JDK 9+ version of {@link GraalServices}. */ public final class GraalServices { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.virtual/src/org/graalvm/compiler/virtual/phases/ea/VirtualizerToolImpl.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.virtual/src/org/graalvm/compiler/virtual/phases/ea/VirtualizerToolImpl.java index f313a935348..0379a3b9cca 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.virtual/src/org/graalvm/compiler/virtual/phases/ea/VirtualizerToolImpl.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.virtual/src/org/graalvm/compiler/virtual/phases/ea/VirtualizerToolImpl.java @@ -126,18 +126,14 @@ class VirtualizerToolImpl implements VirtualizerTool, CanonicalizerTool { public boolean setVirtualEntry(VirtualObjectNode virtual, int index, ValueNode value, JavaKind theAccessKind, long offset) { ObjectState obj = state.getObjectState(virtual); assert obj.isVirtual() : "not virtual: " + obj; - ValueNode newValue; JavaKind entryKind = virtual.entryKind(index); JavaKind accessKind = theAccessKind != null ? theAccessKind : entryKind; - if (value == null) { - newValue = null; - } else { - newValue = closure.getAliasAndResolve(state, value); - } + ValueNode newValue = closure.getAliasAndResolve(state, value); getDebug().log(DebugContext.DETAILED_LEVEL, "Setting entry %d in virtual object %s %s results in %s", index, virtual.getObjectId(), virtual, state.getObjectState(virtual.getObjectId())); ValueNode oldValue = getEntry(virtual, index); boolean canVirtualize = entryKind == accessKind || (entryKind == accessKind.getStackKind() && virtual instanceof VirtualInstanceNode); if (!canVirtualize) { + assert entryKind != JavaKind.Long || newValue != null; if (entryKind == JavaKind.Long && oldValue.getStackKind() == newValue.getStackKind() && oldValue.getStackKind().isPrimitive()) { /* * Special case: If the entryKind is long, allow arbitrary kinds as long as a value diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.graphio/src/org/graalvm/graphio/GraphProtocol.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.graphio/src/org/graalvm/graphio/GraphProtocol.java index 56eb519e4ed..ea117c6372a 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.graphio/src/org/graalvm/graphio/GraphProtocol.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.graphio/src/org/graalvm/graphio/GraphProtocol.java @@ -634,7 +634,7 @@ abstract class GraphProtocol