8247922: Update Graal

Reviewed-by: kvn

make/CompileJavaModules.gmk

@@ -477,6 +477,7 @@ jdk.internal.vm.compiler_EXCLUDES += \
     org.graalvm.compiler.options.test \
     org.graalvm.compiler.phases.common.test \
     org.graalvm.compiler.processor \
+    org.graalvm.compiler.replacements.jdk10.test \
     org.graalvm.compiler.replacements.jdk12.test \
     org.graalvm.compiler.replacements.jdk9.test \
     org.graalvm.compiler.replacements.processor \

make/test/JtregGraalUnit.gmk

@@ -105,6 +105,7 @@ ifeq ($(INCLUDE_GRAAL), true)
               $(SRC_DIR)/org.graalvm.compiler.nodes.test/src \
               $(SRC_DIR)/org.graalvm.compiler.options.test/src \
               $(SRC_DIR)/org.graalvm.compiler.phases.common.test/src \
+              $(SRC_DIR)/org.graalvm.compiler.replacements.jdk10.test/src \
               $(SRC_DIR)/org.graalvm.compiler.replacements.jdk12.test/src \
               $(SRC_DIR)/org.graalvm.compiler.replacements.jdk9.test/src \
               $(SRC_DIR)/org.graalvm.compiler.replacements.test/src \

src/jdk.internal.vm.compiler/share/classes/module-info.java

@@ -38,7 +38,7 @@ module jdk.internal.vm.compiler {
     uses org.graalvm.compiler.hotspot.HotSpotBackendFactory;
     uses org.graalvm.compiler.hotspot.HotSpotCodeCacheListener;
     uses org.graalvm.compiler.hotspot.HotSpotGraalManagementRegistration;
-    uses org.graalvm.compiler.nodes.graphbuilderconf.NodeIntrinsicPluginFactory;
+    uses org.graalvm.compiler.nodes.graphbuilderconf.GeneratedPluginFactory;
     uses org.graalvm.compiler.phases.common.jmx.HotSpotMBeanOperationProvider;
     uses org.graalvm.compiler.serviceprovider.JMXService;
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64.test/src/org/graalvm/compiler/asm/aarch64/test/TestProtectedAssembler.java

@@ -208,7 +208,7 @@ class TestProtectedAssembler extends AArch64Assembler {
     }
 
     @Override
-    protected void sbfm(int size, Register dst, Register src, int r, int s) {
+    public void sbfm(int size, Register dst, Register src, int r, int s) {
         super.sbfm(size, dst, src, r, s);
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64Assembler.java

@@ -1852,7 +1852,7 @@ public abstract class AArch64Assembler extends Assembler {
      * @param r must be in the range 0 to size - 1
      * @param s must be in the range 0 to size - 1
      */
-    protected void sbfm(int size, Register dst, Register src, int r, int s) {
+    public void sbfm(int size, Register dst, Register src, int r, int s) {
         bitfieldInstruction(SBFM, dst, src, r, s, generalFromSize(size));
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64BitFieldTest.java

@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2019, Arm Limited and affiliates. All rights reserved.
+ * Copyright (c) 2019, 2020, Arm Limited and affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -36,12 +36,16 @@ public class AArch64BitFieldTest extends AArch64MatchRuleTest {
     private static final Predicate<LIRInstruction> predicate = op -> (op instanceof AArch64BitFieldOp);
 
     private void testAndCheckLIR(String method, String negativeMethod, Object input) {
-        test(method, input);
+        testAndCheckLIR(method, input);
-        checkLIR(method, predicate, 1);
         test(negativeMethod, input);
         checkLIR(negativeMethod, predicate, 0);
     }
 
+    private void testAndCheckLIR(String method, Object input) {
+        test(method, input);
+        checkLIR(method, predicate, 1);
+    }
+
     /**
      * unsigned bit field extract int.
      */
@@ -133,4 +137,174 @@ public class AArch64BitFieldTest extends AArch64MatchRuleTest {
     public void testInsertLong() {
         testAndCheckLIR("insertLong", "invalidInsertLong", 0xdeadbeefdeadbeefL);
     }
+
+    // Tests for unsigned bitfield move, with integration of zero extend (I2L) operation.
+    //
+    // UBFIZ with I2L.
+    public static long unsignedInsertExtend(int input) {
+        return ((long) (input & 0xffff)) << 8;
+    }
+
+    @Test
+    public void testUnsignedInsertExtend() {
+        testAndCheckLIR("unsignedInsertExtend", 0x234);
+    }
+
+    // I2L with UBFIZ.
+    public static long unsignedExtendInsert(int input) {
+        return (input & 0xfff) << 5;
+    }
+
+    @Test
+    public void testUnsignedExtendInsert() {
+        testAndCheckLIR("unsignedExtendInsert", 0x4334);
+    }
+
+    // I2L with UBFX.
+    public static long unsignedExtendExtract(int input) {
+        return (input >>> 6) & 0xffffff;
+    }
+
+    @Test
+    public void testUnsignedExtendExtract() {
+        testAndCheckLIR("unsignedExtendExtract", 0x325ab);
+    }
+
+    // Signed bitfield insert with extend, generated by (LeftShift (SignExtend value) a) match
+    // rule.
+    // SBFIZ with B2L.
+    public long signedB2LInsert(long input) {
+        byte b = (byte) input;
+        return ((long) b) << 2;
+    }
+
+    @Test
+    public void testSignedB2LInsert() {
+        testAndCheckLIR("signedB2LInsert", 0xab3213efL);
+    }
+
+    // SBFIZ with S2L.
+    public long signedS2LInsert(long input) {
+        short s = (short) input;
+        return ((long) s) << -5;
+    }
+
+    @Test
+    public void testSignedS2LInsert() {
+        testAndCheckLIR("signedS2LInsert", 0x328032bL);
+    }
+
+    // SBFIZ with I2L.
+    public static long signedI2LInsert(int input) {
+        return ((long) input) << 1;
+    }
+
+    @Test
+    public void testSignedI2LInsert() {
+        testAndCheckLIR("signedI2LInsert", 31);
+    }
+
+    // SBFIZ with B2I.
+    public int signedB2IInsert(int input) {
+        byte b = (byte) input;
+        return b << 31;
+    }
+
+    @Test
+    public void testSignedB2IInsert() {
+        testAndCheckLIR("signedB2IInsert", 0x23);
+    }
+
+    // SBFIZ with S2I.
+    public int signedS2IInsert(int input) {
+        short s = (short) input;
+        return s << 2;
+    }
+
+    @Test
+    public void testSignedS2IInsert() {
+        testAndCheckLIR("signedS2IInsert", 0x92);
+    }
+
+    // Tests for bitfield move generated by ([Unsigned]RightShift (LeftShift value a) b) match
+    // rules.
+    // SBFX for int.
+    public static int signedExtractInt(int input) {
+        return (input << 8) >> 15;
+    }
+
+    @Test
+    public void testSignedExtractInt() {
+        testAndCheckLIR("signedExtractInt", 0x123);
+    }
+
+    // SBFX for long.
+    public static long signedExtractLong(long input) {
+        return (input << 8) >> 15;
+    }
+
+    @Test
+    public void testSignedExtractLong() {
+        testAndCheckLIR("signedExtractLong", 0x125L);
+    }
+
+    // SBFIZ for int.
+    public static int signedInsertInt(int input) {
+        return (input << 15) >> 8;
+    }
+
+    @Test
+    public void testSignedInsertInt() {
+        testAndCheckLIR("signedInsertInt", 0x1253);
+    }
+
+    // SBFIZ for long.
+    public static long signedInsertLong(long input) {
+        return (input << 15) >> 8;
+    }
+
+    @Test
+    public void testSignedInsertLong() {
+        testAndCheckLIR("signedInsertLong", 0xabcddbc325L);
+    }
+
+    // UBFX for int.
+    public static int unsignedExtractInt(int input) {
+        return (input << 8) >>> 31;
+    }
+
+    @Test
+    public void testUnsignedExtractInt() {
+        testAndCheckLIR("unsignedExtractInt", 0x125);
+    }
+
+    // UBFX for long.
+    public static long unsignedExtractLong(long input) {
+        return (input << 8) >>> 12;
+    }
+
+    @Test
+    public void testUnsignedExtractLong() {
+        testAndCheckLIR("unsignedExtractLong", 0x32222e125L);
+    }
+
+    // UBFIZ for int.
+    public static int unsignedInsertInt(int input) {
+        return (input << 15) >>> 8;
+    }
+
+    @Test
+    public void testUnsignedInsertInt() {
+        testAndCheckLIR("unsignedInsertInt", 125);
+    }
+
+    // UBFIZ for long.
+    public static long unsignedInsertLong(long input) {
+        return (input << 63) >>> 1;
+    }
+
+    @Test
+    public void testUnsignedInsertLong() {
+        testAndCheckLIR("unsignedInsertLong", 0x2339fb125L);
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64ArithmeticLIRGenerator.java

@@ -371,7 +371,7 @@ public class AArch64ArithmeticLIRGenerator extends ArithmeticLIRGenerator implem
         }
     }
 
-    private void emitBinaryConst(Variable result, AArch64ArithmeticOp op, AllocatableValue a, JavaConstant b) {
+    public void emitBinaryConst(Variable result, AArch64ArithmeticOp op, AllocatableValue a, JavaConstant b) {
         AllocatableValue x = moveSp(a);
         getLIRGen().append(new AArch64ArithmeticOp.BinaryConstOp(op, result, x, b));
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64NodeMatchRules.java

@@ -42,6 +42,7 @@ import org.graalvm.compiler.lir.LabelRef;
 import org.graalvm.compiler.lir.Variable;
 import org.graalvm.compiler.lir.aarch64.AArch64ArithmeticOp;
 import org.graalvm.compiler.lir.aarch64.AArch64BitFieldOp;
+import org.graalvm.compiler.lir.aarch64.AArch64BitFieldOp.BitFieldOpCode;
 import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow;
 import org.graalvm.compiler.lir.gen.LIRGeneratorTool;
 import org.graalvm.compiler.nodes.ConstantNode;
@@ -73,6 +74,7 @@ import org.graalvm.compiler.nodes.memory.MemoryAccess;
 import jdk.vm.ci.aarch64.AArch64Kind;
 import jdk.vm.ci.code.CodeUtil;
 import jdk.vm.ci.meta.AllocatableValue;
+import jdk.vm.ci.meta.JavaConstant;
 import jdk.vm.ci.meta.JavaKind;
 import jdk.vm.ci.meta.Value;
 
@@ -96,8 +98,8 @@ public class AArch64NodeMatchRules extends NodeMatchRules {
         binaryOpMap.put(UnsignedRightShiftNode.class, AArch64ArithmeticOp.LSHR);
 
         bitFieldOpMap = EconomicMap.create(Equivalence.IDENTITY, 2);
-        bitFieldOpMap.put(UnsignedRightShiftNode.class, AArch64BitFieldOp.BitFieldOpCode.UBFX);
+        bitFieldOpMap.put(UnsignedRightShiftNode.class, BitFieldOpCode.UBFX);
-        bitFieldOpMap.put(LeftShiftNode.class, AArch64BitFieldOp.BitFieldOpCode.UBFIZ);
+        bitFieldOpMap.put(LeftShiftNode.class, BitFieldOpCode.UBFIZ);
 
         logicalNotOpMap = EconomicMap.create(Equivalence.IDENTITY, 3);
         logicalNotOpMap.put(AndNode.class, AArch64ArithmeticOp.BIC);
@@ -161,19 +163,6 @@ public class AArch64NodeMatchRules extends NodeMatchRules {
         return getLIRGeneratorTool().moveSp(value);
     }
 
-    private ComplexMatchResult emitBitField(AArch64BitFieldOp.BitFieldOpCode op, ValueNode value, int lsb, int width) {
-        assert op != null;
-        assert value.getStackKind().isNumericInteger();
-
-        return builder -> {
-            Value a = operand(value);
-            Variable result = gen.newVariable(LIRKind.combine(a));
-            AllocatableValue src = moveSp(gen.asAllocatable(a));
-            gen.append(new AArch64BitFieldOp(op, result, src, lsb, width));
-            return result;
-        };
-    }
-
     private ComplexMatchResult emitBinaryShift(AArch64ArithmeticOp op, ValueNode value, BinaryNode shift) {
         AArch64MacroAssembler.ShiftType shiftType = shiftTypeMap.get(shift.getClass());
         assert shiftType != null;
@@ -205,6 +194,51 @@ public class AArch64NodeMatchRules extends NodeMatchRules {
         };
     }
 
+    private ComplexMatchResult emitBitField(JavaKind kind, AArch64BitFieldOp.BitFieldOpCode op, ValueNode value, int lsb, int width) {
+        assert op != null;
+        assert value.getStackKind().isNumericInteger();
+
+        return builder -> {
+            Value a = operand(value);
+            LIRKind resultKind = LIRKind.fromJavaKind(gen.target().arch, kind);
+            Variable result = gen.newVariable(resultKind);
+            AllocatableValue src = moveSp(gen.asAllocatable(a));
+            gen.append(new AArch64BitFieldOp(op, result, src, lsb, width));
+            return result;
+        };
+    }
+
+    private ComplexMatchResult emitUnsignedBitField(JavaKind kind, BinaryNode shift, ValueNode value, ConstantNode scale, ConstantNode mask) {
+        assert kind.isNumericInteger();
+        BitFieldOpCode op = bitFieldOpMap.get(shift.getClass());
+        assert op != null;
+        JavaKind srcKind = shift.getStackKind();
+        // The Java(R) Language Specification CHAPTER 15.19 Shift Operators says:
+        // "If the promoted type of the left-hand operand is int(long), then only the five(six)
+        // lowest-order bits of the right-hand operand are used as the shift distance."
+        int distance = scale.asJavaConstant().asInt();
+        int lsb = distance & (srcKind == JavaKind.Int ? 0x1F : 0x3F);
+
+        long maskValue = mask.asJavaConstant().asLong();
+        // Constraint 1: Mask plus one should be a power-of-2 integer.
+        if (!CodeUtil.isPowerOf2(maskValue + 1)) {
+            return null;
+        }
+        int width = CodeUtil.log2(maskValue + 1);
+        int srcBits = srcKind.getBitCount();
+        // Constraint 2: Bit field width is less than 31(63) for int(long) as any bit field move
+        // operations can be done by a single shift instruction if the width is 31(63).
+        if (width >= srcBits - 1) {
+            return null;
+        }
+        // Constraint 3: Sum of bit field width and the shift distance is less or equal to 32(64)
+        // for int(long) as the specification of AArch64 bit field instructions.
+        if (width + distance > srcBits) {
+            return null;
+        }
+        return emitBitField(kind, op, value, lsb, width);
+    }
+
     private static boolean isNarrowingLongToInt(NarrowNode narrow) {
         return narrow.getInputBits() == Long.SIZE && narrow.getResultBits() == Integer.SIZE;
     }
@@ -325,38 +359,68 @@ public class AArch64NodeMatchRules extends NodeMatchRules {
         return null;
     }
 
-    @MatchRule("(And (UnsignedRightShift=shift a Constant=b) Constant=c)")
+    @MatchRule("(And (UnsignedRightShift=shift value Constant=a) Constant=b)")
-    @MatchRule("(LeftShift=shift (And a Constant=c) Constant=b)")
+    @MatchRule("(LeftShift=shift (And value Constant=b) Constant=a)")
-    public ComplexMatchResult unsignedBitField(BinaryNode shift, ValueNode a, ConstantNode b, ConstantNode c) {
+    public ComplexMatchResult unsignedBitField(BinaryNode shift, ValueNode value, ConstantNode a, ConstantNode b) {
-        JavaKind srcKind = a.getStackKind();
+        JavaKind kind = shift.getStackKind();
+        return emitUnsignedBitField(kind, shift, value, a, b);
+    }
+
+    @MatchRule("(LeftShift=shift (ZeroExtend=extend (And value Constant=b)) Constant=a)")
+    @MatchRule("(ZeroExtend=extend (And (UnsignedRightShift=shift value Constant=a) Constant=b))")
+    @MatchRule("(ZeroExtend=extend (LeftShift=shift (And value Constant=b) Constant=a))")
+    public ComplexMatchResult unsignedExtBitField(ZeroExtendNode extend, BinaryNode shift, ValueNode value, ConstantNode a, ConstantNode b) {
+        JavaKind kind = extend.getStackKind();
+        return emitUnsignedBitField(kind, shift, value, a, b);
+    }
+
+    @MatchRule("(LeftShift=shift (SignExtend value) Constant)")
+    public ComplexMatchResult signedBitField(LeftShiftNode shift) {
+        JavaKind kind = shift.getStackKind();
+        assert kind.isNumericInteger();
+
+        SignExtendNode extend = (SignExtendNode) shift.getX();
+        int srcBits = extend.getInputBits();
+        int resultBits = extend.getResultBits();
+        assert kind.getBitCount() == resultBits;
+
+        int lsb = shift.getY().asJavaConstant().asInt() & (resultBits - 1);
+        // Get the min value of the srcBits and (resultBits - lsb) as the bitfield width.
+        int width = Math.min(srcBits, resultBits - lsb);
+        assert width >= 1 && width <= resultBits - lsb;
+
+        ValueNode value = extend.getValue();
+        return emitBitField(kind, BitFieldOpCode.SBFIZ, value, lsb, width);
+    }
+
+    @MatchRule("(RightShift=rshift (LeftShift=lshift value Constant) Constant)")
+    @MatchRule("(UnsignedRightShift=rshift (LeftShift=lshift value Constant) Constant)")
+    public ComplexMatchResult bitFieldMove(BinaryNode rshift, LeftShiftNode lshift) {
+        JavaKind srcKind = rshift.getStackKind();
         assert srcKind.isNumericInteger();
-        AArch64BitFieldOp.BitFieldOpCode op = bitFieldOpMap.get(shift.getClass());
-        assert op != null;
-        int distance = b.asJavaConstant().asInt();
-        long mask = c.asJavaConstant().asLong();
-
-        // The Java(R) Language Specification CHAPTER 15.19 Shift Operators says:
-        // "If the promoted type of the left-hand operand is int(long), then only the five(six)
-        // lowest-order bits of the right-hand operand are used as the shift distance."
-        distance = distance & (srcKind == JavaKind.Int ? 0x1f : 0x3f);
-
-        // Constraint 1: Mask plus one should be a power-of-2 integer.
-        if (!CodeUtil.isPowerOf2(mask + 1)) {
-            return null;
-        }
-        int width = CodeUtil.log2(mask + 1);
         int srcBits = srcKind.getBitCount();
-        // Constraint 2: Bit field width is less than 31(63) for int(long) as any bit field move
+        int lshiftNum = lshift.getY().asJavaConstant().asInt() & (srcBits - 1);
-        // operations can be done by a single shift instruction if the width is 31(63).
+        int rshiftNum = rshift.getY().asJavaConstant().asInt() & (srcBits - 1);
-        if (width >= srcBits - 1) {
+        int lsb = Math.abs(lshiftNum - rshiftNum);
+        assert lsb >= 0 && lsb <= (srcBits - 1);
+
+        // Get the width of the bitField. It should be in the range 1 to 32(64)-<lsb>.
+        int width = srcBits - Math.max(lshiftNum, rshiftNum);
+        if (width > (srcBits - lsb) || width < 1) {
             return null;
         }
-        // Constraint 3: Sum of bit field width and the shift distance is less or equal to 32(64)
+
-        // for int(long) as the specification of AArch64 bit field instructions.
+        // Use bitfield insert (SBFIZ/UBFIZ) if left shift number is larger than right shift number,
-        if (width + distance > srcBits) {
+        // otherwise use bitfield extract (SBFX/UBFX).
-            return null;
+        boolean bitFieldInsert = lshiftNum > rshiftNum;
+        BitFieldOpCode op;
+        if (rshift instanceof RightShiftNode) {
+            op = bitFieldInsert ? BitFieldOpCode.SBFIZ : BitFieldOpCode.SBFX;
+        } else {
+            assert rshift instanceof UnsignedRightShiftNode;
+            op = bitFieldInsert ? BitFieldOpCode.UBFIZ : BitFieldOpCode.UBFX;
         }
-        return emitBitField(op, a, distance, width);
+        return emitBitField(srcKind, op, lshift.getX(), lsb, width);
     }
 
     @MatchRule("(Or=op (LeftShift=x src Constant=shiftAmt1) (UnsignedRightShift src Constant=shiftAmt2))")
@@ -375,9 +439,11 @@ public class AArch64NodeMatchRules extends NodeMatchRules {
         }
         if ((0 == shift1 + shift2) || (src.getStackKind().getBitCount() == shift1 + shift2)) {
             return builder -> {
-                Value a = operand(src);
+                AllocatableValue a = gen.asAllocatable(operand(src));
-                Value b = x instanceof LeftShiftNode ? operand(shiftAmt2) : operand(shiftAmt1);
+                JavaConstant b = x instanceof LeftShiftNode ? shiftAmt2.asJavaConstant() : shiftAmt1.asJavaConstant();
-                return getArithmeticLIRGenerator().emitBinary(LIRKind.combine(a, b), AArch64ArithmeticOp.ROR, false, a, b);
+                Variable result = gen.newVariable(LIRKind.combine(a));
+                getArithmeticLIRGenerator().emitBinaryConst(result, AArch64ArithmeticOp.ROR, a, b);
+                return result;
             };
         }
         return null;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64ArithmeticLIRGenerator.java

@@ -692,21 +692,26 @@ public class AMD64ArithmeticLIRGenerator extends ArithmeticLIRGenerator implemen
     }
 
     private Variable emitShift(AMD64Shift op, OperandSize size, Value a, Value b) {
+        if (isJavaConstant(b)) {
+            return emitShiftConst(op, size, a, asJavaConstant(b));
+        }
         Variable result = getLIRGen().newVariable(LIRKind.combine(a, b).changeType(a.getPlatformKind()));
         AllocatableValue input = asAllocatable(a);
-        if (isJavaConstant(b)) {
+        getLIRGen().emitMove(RCX_I, b);
-            JavaConstant c = asJavaConstant(b);
+        getLIRGen().append(new AMD64ShiftOp(op.mcOp, size, result, input, RCX_I));
-            if (c.asLong() == 1) {
+        return result;
+    }
+
+    public Variable emitShiftConst(AMD64Shift op, OperandSize size, Value a, JavaConstant b) {
+        Variable result = getLIRGen().newVariable(LIRKind.combine(a).changeType(a.getPlatformKind()));
+        AllocatableValue input = asAllocatable(a);
+        if (b.asLong() == 1) {
             getLIRGen().append(new AMD64Unary.MOp(op.m1Op, size, result, input));
         } else {
             /*
              * c needs to be masked here, because shifts with immediate expect a byte.
              */
-                getLIRGen().append(new AMD64Binary.ConstOp(op.miOp, size, result, input, (byte) c.asLong()));
+            getLIRGen().append(new AMD64Binary.ConstOp(op.miOp, size, result, input, (byte) b.asLong()));
-            }
-        } else {
-            getLIRGen().emitMove(RCX_I, b);
-            getLIRGen().append(new AMD64ShiftOp(op.mcOp, size, result, input, RCX_I));
         }
         return result;
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64NodeMatchRules.java

@@ -33,6 +33,7 @@ import static org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64BinaryArithmeti
 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64RMOp.MOVSX;
 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64RMOp.MOVSXB;
 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64RMOp.MOVSXD;
+import static org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64Shift.ROL;
 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexRVMOp.VADDSD;
 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexRVMOp.VADDSS;
 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.VexRVMOp.VMULSD;
@@ -449,8 +450,15 @@ public class AMD64NodeMatchRules extends NodeMatchRules {
 
     @MatchRule("(Or (LeftShift=lshift value Constant) (UnsignedRightShift=rshift value Constant))")
     public ComplexMatchResult rotateLeftConstant(LeftShiftNode lshift, UnsignedRightShiftNode rshift) {
-        if ((lshift.getShiftAmountMask() & (lshift.getY().asJavaConstant().asInt() + rshift.getY().asJavaConstant().asInt())) == 0) {
+        JavaConstant lshiftConst = lshift.getY().asJavaConstant();
-            return builder -> getArithmeticLIRGenerator().emitRol(operand(lshift.getX()), operand(lshift.getY()));
+        JavaConstant rshiftConst = rshift.getY().asJavaConstant();
+        if ((lshift.getShiftAmountMask() & (lshiftConst.asInt() + rshiftConst.asInt())) == 0) {
+            return builder -> {
+                Value a = operand(lshift.getX());
+                OperandSize size = OperandSize.get(a.getPlatformKind());
+                assert size == OperandSize.DWORD || size == OperandSize.QWORD;
+                return getArithmeticLIRGenerator().emitShiftConst(ROL, size, a, lshiftConst);
+            };
         }
         return null;
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.match.processor/src/org/graalvm/compiler/core/match/processor/MatchProcessor.java

@@ -343,11 +343,16 @@ public class MatchProcessor extends AbstractProcessor {
         final boolean commutative;
 
         /**
-         * Can multiple users of this node subsume it. Constants can be swallowed into a match even
+         * Can multiple users of this node subsume it.
-         * if there are multiple users.
          */
         final boolean shareable;
 
+        /**
+         * Can this node be swallowed into a match. Constants can be consumed by a match even if it
+         * has multiple users.
+         */
+        final boolean consumable;
+
         /**
          * Can this node be subsumed into a match even if there are side effecting nodes between
          * this node and the match.
@@ -356,7 +361,8 @@ public class MatchProcessor extends AbstractProcessor {
 
         final Set<Element> originatingElements = new HashSet<>();
 
-        TypeDescriptor(TypeMirror mirror, String shortName, String nodeClass, String nodePackage, List<String> inputs, boolean commutative, boolean shareable, boolean ignoresSideEffects) {
+        TypeDescriptor(TypeMirror mirror, String shortName, String nodeClass, String nodePackage, List<String> inputs,
+                        boolean commutative, boolean shareable, boolean consumable, boolean ignoresSideEffects) {
             this.mirror = mirror;
             this.shortName = shortName;
             this.nodeClass = nodeClass;
@@ -364,6 +370,7 @@ public class MatchProcessor extends AbstractProcessor {
             this.inputs = inputs;
             this.commutative = commutative;
             this.shareable = shareable;
+            this.consumable = consumable;
             this.ignoresSideEffects = ignoresSideEffects;
             assert !commutative || inputs.size() == 2;
         }
@@ -376,9 +383,9 @@ public class MatchProcessor extends AbstractProcessor {
 
     private TypeDescriptor valueType;
 
-    private void declareType(TypeMirror mirror, String shortName, String nodeClass, String nodePackage, List<String> inputs, boolean commutative, boolean shareable, boolean ignoresSideEffects,
+    private void declareType(TypeMirror mirror, String shortName, String nodeClass, String nodePackage, List<String> inputs,
-                    Element element) {
+                    boolean commutative, boolean shareable, boolean consumable, boolean ignoresSideEffects, Element element) {
-        TypeDescriptor descriptor = new TypeDescriptor(mirror, shortName, nodeClass, nodePackage, inputs, commutative, shareable, ignoresSideEffects);
+        TypeDescriptor descriptor = new TypeDescriptor(mirror, shortName, nodeClass, nodePackage, inputs, commutative, shareable, consumable, ignoresSideEffects);
         descriptor.originatingElements.add(element);
         knownTypes.put(shortName, descriptor);
     }
@@ -461,7 +468,7 @@ public class MatchProcessor extends AbstractProcessor {
 
         private String formatPrefix() {
             if (nodeType == valueType) {
-                return String.format("new MatchPattern(%s, false, false", name != null ? ("\"" + name + "\"") : "null");
+                return String.format("new MatchPattern(%s, false, false, false", name != null ? ("\"" + name + "\"") : "null");
             } else {
                 return String.format("new MatchPattern(%s.class, %s", nodeType.nodeClass, name != null ? ("\"" + name + "\"") : "null");
             }
@@ -470,13 +477,13 @@ public class MatchProcessor extends AbstractProcessor {
         private String formatSuffix() {
             if (nodeType != null) {
                 if (inputs.length != nodeType.inputs.size()) {
-                    return ", true, " + nodeType.ignoresSideEffects + ")";
+                    return ", true, " + nodeType.consumable + ", " + nodeType.ignoresSideEffects + ")";
                 } else {
                     if (nodeType.inputs.size() > 0) {
-                        return ", " + nodeType.nodeClass + "_positions, " + !nodeType.shareable + ", " + nodeType.ignoresSideEffects + ")";
+                        return ", " + nodeType.nodeClass + "_positions, " + !nodeType.shareable + ", " + nodeType.consumable + ", " + nodeType.ignoresSideEffects + ")";
                     }
                     if (nodeType.shareable) {
-                        return ", false, " + nodeType.ignoresSideEffects + ")";
+                        return ", false, " + nodeType.consumable + ", " + nodeType.ignoresSideEffects + ")";
                     }
                 }
             }
@@ -732,7 +739,7 @@ public class MatchProcessor extends AbstractProcessor {
             // Define a TypeDescriptor for the generic node but don't enter it into the nodeTypes
             // table since it shouldn't be mentioned in match rules.
             TypeMirror valueTypeMirror = getTypeElement(VALUE_NODE_CLASS_NAME).asType();
-            valueType = new TypeDescriptor(valueTypeMirror, "Value", "ValueNode", "org.graalvm.compiler.nodes", Collections.emptyList(), false, false, false);
+            valueType = new TypeDescriptor(valueTypeMirror, "Value", "ValueNode", "org.graalvm.compiler.nodes", Collections.emptyList(), false, false, false, false);
 
             Map<TypeElement, MatchRuleDescriptor> map = new HashMap<>();
 
@@ -842,8 +849,9 @@ public class MatchProcessor extends AbstractProcessor {
 
         boolean commutative = getAnnotationValue(matchable, "commutative", Boolean.class);
         boolean shareable = getAnnotationValue(matchable, "shareable", Boolean.class);
+        boolean consumable = getAnnotationValue(matchable, "consumable", Boolean.class);
         boolean ignoresSideEffects = getAnnotationValue(matchable, "ignoresSideEffects", Boolean.class);
-        declareType(nodeClassMirror, shortName, nodeClass, nodePackage, inputs, commutative, shareable, ignoresSideEffects, element);
+        declareType(nodeClassMirror, shortName, nodeClass, nodePackage, inputs, commutative, shareable, consumable, ignoresSideEffects, element);
     }
 
     private void processMatchRules(Map<TypeElement, MatchRuleDescriptor> map, Element element, List<AnnotationMirror> matchRules) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/CheckGraalInvariants.java

@@ -538,7 +538,7 @@ public class CheckGraalInvariants extends GraalCompilerTest {
     }
 
     private static boolean isInNativeImage(String className) {
-        return className.startsWith("org.graalvm.nativeimage");
+        return className.startsWith("jdk.internal.vm.compiler.nativeimage");
     }
 
     private static boolean isGSON(String className) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/DeepUnrollingTest.java

@@ -4,9 +4,7 @@
  *
  * This code is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.  Oracle designates this
+ * published by the Free Software Foundation.
- * particular file as subject to the "Classpath" exception as provided
- * by Oracle in the LICENSE file that accompanied this code.
  *
  * This code is distributed in the hope that it will be useful, but WITHOUT
  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
@@ -22,6 +20,8 @@
  * or visit www.oracle.com if you need additional information or have any
  * questions.
  */
+
+
 package org.graalvm.compiler.core.test;
 
 import java.io.IOException;
@@ -59,6 +59,8 @@ public class DeepUnrollingTest extends SubprocessTest {
         return v;
     }
 
+    private static final int ACCEPTABLE_FACTOR = 50;
+
     public void loopTest() {
         // warmup
         time("reference");
@@ -66,7 +68,9 @@ public class DeepUnrollingTest extends SubprocessTest {
         long reference = time("reference");
         long loops = time("loops");
         // observed ratio is ~20-30x. Pathological case before fix was ~300x
-        assertTrue("Compilation of the loop nest is too slow", loops < reference * 45);
+        if (loops > reference * ACCEPTABLE_FACTOR) {
+            fail("Compilation of the loop nest is too slow. loops: %dms > %d * reference: %dms", loops, ACCEPTABLE_FACTOR, reference);
+        }
     }
 
     public long time(String methodName) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/UnschedulableGraphTest.java

@@ -105,7 +105,7 @@ public class UnschedulableGraphTest extends GraalCompilerTest {
         try (AutoCloseable c = new TTY.Filter();
                         DebugContext debug = getDebugContext(method);
                         DebugCloseable s = debug.disableIntercept()) {
-            test("snippet01", 0, 1, 2);
+            test(debug.getOptions(), "snippet01", 0, 1, 2);
             Assert.fail("Compilation should not reach this point, must throw an exception before");
         } catch (Throwable t) {
             if (t.getMessage().contains("liveIn set of first block must be empty")) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/GraalCompiler.java

@@ -188,17 +188,28 @@ public class GraalCompiler {
                 }
             }
             if (crashLabel != null) {
+                String crashMessage = "Forced crash after compiling " + crashLabel;
+                notifyCrash(crashMessage);
                 if (permanentBailout) {
-                    throw new PermanentBailoutException("Forced crash after compiling " + crashLabel);
+                    throw new PermanentBailoutException(crashMessage);
                 }
                 if (bailout) {
-                    throw new RetryableBailoutException("Forced crash after compiling " + crashLabel);
+                    throw new RetryableBailoutException(crashMessage);
                 }
-                throw new RuntimeException("Forced crash after compiling " + crashLabel);
+                throw new RuntimeException(crashMessage);
             }
         }
     }
 
+    /**
+     * Substituted by {@code com.oracle.svm.graal.hotspot.libgraal.
+     * Target_org_graalvm_compiler_core_GraalCompiler} to optionally test routing fatal error
+     * handling from libgraal to HotSpot.
+     */
+    @SuppressWarnings("unused")
+    private static void notifyCrash(String crashMessage) {
+    }
+
     /**
      * Builds the graph, optimizes it.
      */

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/NodeLIRBuilder.java

@@ -30,6 +30,7 @@ import static jdk.vm.ci.code.ValueUtil.isRegister;
 import static org.graalvm.compiler.core.common.GraalOptions.MatchExpressions;
 import static org.graalvm.compiler.core.common.SpeculativeExecutionAttacksMitigations.AllTargets;
 import static org.graalvm.compiler.core.common.SpeculativeExecutionAttacksMitigations.Options.MitigateSpeculativeExecutionAttacks;
+import static org.graalvm.compiler.core.match.ComplexMatchValue.INTERIOR_MATCH;
 import static org.graalvm.compiler.debug.DebugOptions.LogVerbose;
 import static org.graalvm.compiler.lir.LIR.verifyBlock;
 
@@ -137,6 +138,7 @@ public abstract class NodeLIRBuilder implements NodeLIRBuilderTool, LIRGeneratio
 
     private final NodeMatchRules nodeMatchRules;
     private EconomicMap<Class<? extends Node>, List<MatchStatement>> matchRules;
+    private EconomicMap<Node, Integer> sharedMatchCounts;
 
     public NodeLIRBuilder(StructuredGraph graph, LIRGeneratorTool gen, NodeMatchRules nodeMatchRules) {
         this.gen = (LIRGenerator) gen;
@@ -146,6 +148,7 @@ public abstract class NodeLIRBuilder implements NodeLIRBuilderTool, LIRGeneratio
         OptionValues options = graph.getOptions();
         if (MatchExpressions.getValue(options)) {
             matchRules = MatchRuleRegistry.lookup(nodeMatchRules.getClass(), options, graph.getDebug());
+            sharedMatchCounts = EconomicMap.create();
         }
         traceLIRGeneratorLevel = TTY.isSuppressed() ? 0 : Options.TraceLIRGeneratorLevel.getValue(options);
 
@@ -219,13 +222,30 @@ public abstract class NodeLIRBuilder implements NodeLIRBuilderTool, LIRGeneratio
      * ValueNodes.
      */
     public void setMatchResult(Node x, Value operand) {
-        assert operand.equals(ComplexMatchValue.INTERIOR_MATCH) || operand instanceof ComplexMatchValue;
+        assert operand.equals(INTERIOR_MATCH) || operand instanceof ComplexMatchValue;
         assert operand instanceof ComplexMatchValue || MatchPattern.isSingleValueUser(x) : "interior matches must be single user";
         assert nodeOperands != null && nodeOperands.get(x) == null : "operand cannot be set twice";
         assert !(x instanceof VirtualObjectNode);
         nodeOperands.set(x, operand);
     }
 
+    /**
+     * Track how many users have consumed a sharedable match and disable emission of the value if
+     * all users have consumed it.
+     */
+    public void incrementSharedMatchCount(Node node) {
+        assert nodeOperands != null && nodeOperands.get(node) == null : "operand cannot be set twice";
+        Integer matchValue = sharedMatchCounts.get(node);
+        if (matchValue == null) {
+            matchValue = 0;
+        }
+        matchValue = matchValue + 1;
+        sharedMatchCounts.put(node, matchValue);
+        if (node.getUsageCount() == matchValue) {
+            nodeOperands.set(node, INTERIOR_MATCH);
+        }
+    }
+
     public LabelRef getLIRBlock(FixedNode b) {
         assert gen.getResult().getLIR().getControlFlowGraph() instanceof ControlFlowGraph;
         Block result = ((ControlFlowGraph) gen.getResult().getLIR().getControlFlowGraph()).blockFor(b);
@@ -384,7 +404,7 @@ public abstract class NodeLIRBuilder implements NodeLIRBuilderTool, LIRGeneratio
                                 throw new GraalGraphError(e).addContext(valueNode);
                             }
                         }
-                    } else if (ComplexMatchValue.INTERIOR_MATCH.equals(operand)) {
+                    } else if (INTERIOR_MATCH.equals(operand)) {
                         // Doesn't need to be evaluated
                         debug.log("interior match for %s", valueNode);
                     } else if (operand instanceof ComplexMatchValue) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/match/MatchContext.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -68,10 +68,12 @@ public class MatchContext {
     static final class ConsumedNode {
         final Node node;
         final boolean ignoresSideEffects;
+        final boolean singleUser;
 
-        ConsumedNode(Node node, boolean ignoresSideEffects) {
+        ConsumedNode(Node node, boolean ignoresSideEffects, boolean singleUser) {
             this.node = node;
             this.ignoresSideEffects = ignoresSideEffects;
+            this.singleUser = singleUser;
         }
     }
 
@@ -85,11 +87,11 @@ public class MatchContext {
             this.nodes = null;
         }
 
-        public void add(Node node, boolean ignoresSideEffects) {
+        public void add(Node node, boolean ignoresSideEffects, boolean singlerUser) {
             if (nodes == null) {
                 nodes = new ArrayList<>(2);
             }
-            nodes.add(new ConsumedNode(node, ignoresSideEffects));
+            nodes.add(new ConsumedNode(node, ignoresSideEffects, singlerUser));
         }
 
         public boolean contains(Node node) {
@@ -373,10 +375,7 @@ public class MatchContext {
             if (cn.node == root || cn.node == emitNode) {
                 continue;
             }
-            // All the interior nodes should be skipped during the normal doRoot calls in
+            setResult(cn);
-            // NodeLIRBuilder so mark them as interior matches. The root of the match will get a
-            // closure which will be evaluated to produce the final LIR.
-            builder.setMatchResult(cn.node, ComplexMatchValue.INTERIOR_MATCH);
         }
         builder.setMatchResult(emitNode, value);
         if (root != emitNode) {
@@ -387,23 +386,34 @@ public class MatchContext {
         }
     }
 
+    private void setResult(ConsumedNode consumedNode) {
+        Node node = consumedNode.node;
+        if (consumedNode.singleUser) {
+            // All the interior nodes should be skipped during the normal doRoot calls in
+            // NodeLIRBuilder so mark them as interior matches. The root of the match will get a
+            // closure which will be evaluated to produce the final LIR.
+            builder.setMatchResult(node, ComplexMatchValue.INTERIOR_MATCH);
+            return;
+        }
+        builder.incrementSharedMatchCount(node);
+    }
+
     /**
      * Mark a node as consumed by the match. Consumed nodes will never be evaluated.
      *
      * @return Result.OK if the node can be safely consumed.
      */
-    public Result consume(Node node, boolean ignoresSideEffects, boolean atRoot) {
+    public Result consume(Node node, boolean ignoresSideEffects, boolean atRoot, boolean singleUser) {
         if (atRoot) {
-            consumed.add(node, ignoresSideEffects);
+            consumed.add(node, ignoresSideEffects, singleUser);
             return Result.OK;
         }
-        assert MatchPattern.isSingleValueUser(node) : "should have already been checked";
 
         if (builder.hasOperand(node)) {
             return Result.alreadyUsed(node, rule.getPattern());
         }
 
-        consumed.add(node, ignoresSideEffects);
+        consumed.add(node, ignoresSideEffects, singleUser);
         return Result.OK;
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/match/MatchPattern.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -158,6 +158,8 @@ public class MatchPattern {
      */
     private final boolean singleUser;
 
+    private final boolean consumable;
+
     /**
      * Can this node be subsumed into a match even if there are side effecting nodes between this
      * node and the match.
@@ -166,41 +168,47 @@ public class MatchPattern {
 
     private static final MatchPattern[] EMPTY_PATTERNS = new MatchPattern[0];
 
-    public MatchPattern(String name, boolean singleUser, boolean ignoresSideEffects) {
+    public MatchPattern(String name, boolean singleUser, boolean consumable, boolean ignoresSideEffects) {
-        this(null, name, singleUser, ignoresSideEffects);
+        this(null, name, singleUser, consumable, ignoresSideEffects);
     }
 
-    public MatchPattern(Class<? extends Node> nodeClass, String name, boolean singleUser, boolean ignoresSideEffects) {
+    public MatchPattern(Class<? extends Node> nodeClass, String name, boolean singleUser, boolean consumable, boolean ignoresSideEffects) {
         this.nodeClass = nodeClass;
         this.name = name;
         this.singleUser = singleUser;
+        this.consumable = consumable;
         this.ignoresSideEffects = ignoresSideEffects;
         this.patterns = EMPTY_PATTERNS;
         this.inputs = null;
         assert !ignoresSideEffects || FloatingNode.class.isAssignableFrom(nodeClass);
     }
 
-    private MatchPattern(Class<? extends Node> nodeClass, String name, boolean singleUser, boolean ignoresSideEffects, MatchPattern[] patterns, Position[] inputs) {
+    private MatchPattern(Class<? extends Node> nodeClass, String name, boolean singleUser, boolean consumable,
+                    boolean ignoresSideEffects, MatchPattern[] patterns, Position[] inputs) {
         assert inputs == null || inputs.length == patterns.length;
         this.nodeClass = nodeClass;
         this.name = name;
         this.singleUser = singleUser;
+        this.consumable = consumable;
         this.ignoresSideEffects = ignoresSideEffects;
         this.patterns = patterns;
         this.inputs = inputs;
         assert !ignoresSideEffects || FloatingNode.class.isAssignableFrom(nodeClass);
     }
 
-    public MatchPattern(Class<? extends Node> nodeClass, String name, MatchPattern first, Position[] inputs, boolean singleUser, boolean ignoresSideEffects) {
+    public MatchPattern(Class<? extends Node> nodeClass, String name, MatchPattern first, Position[] inputs,
-        this(nodeClass, name, singleUser, ignoresSideEffects, new MatchPattern[]{first}, inputs);
+                    boolean singleUser, boolean consumable, boolean ignoresSideEffects) {
+        this(nodeClass, name, singleUser, consumable, ignoresSideEffects, new MatchPattern[]{first}, inputs);
     }
 
-    public MatchPattern(Class<? extends Node> nodeClass, String name, MatchPattern first, MatchPattern second, Position[] inputs, boolean singleUser, boolean ignoresSideEffects) {
+    public MatchPattern(Class<? extends Node> nodeClass, String name, MatchPattern first, MatchPattern second,
-        this(nodeClass, name, singleUser, ignoresSideEffects, new MatchPattern[]{first, second}, inputs);
+                    Position[] inputs, boolean singleUser, boolean consumable, boolean ignoresSideEffects) {
+        this(nodeClass, name, singleUser, consumable, ignoresSideEffects, new MatchPattern[]{first, second}, inputs);
     }
 
-    public MatchPattern(Class<? extends Node> nodeClass, String name, MatchPattern first, MatchPattern second, MatchPattern third, Position[] inputs, boolean singleUser, boolean ignoresSideEffects) {
+    public MatchPattern(Class<? extends Node> nodeClass, String name, MatchPattern first, MatchPattern second, MatchPattern third,
-        this(nodeClass, name, singleUser, ignoresSideEffects, new MatchPattern[]{first, second, third}, inputs);
+                    Position[] inputs, boolean singleUser, boolean consumable, boolean ignoresSideEffects) {
+        this(nodeClass, name, singleUser, consumable, ignoresSideEffects, new MatchPattern[]{first, second, third}, inputs);
     }
 
     Class<? extends Node> nodeClass() {
@@ -234,8 +242,9 @@ public class MatchPattern {
         if (result != Result.OK) {
             return result;
         }
-        if (singleUser) {
+
-            result = context.consume(node, ignoresSideEffects, atRoot);
+        if (consumable) {
+            result = context.consume(node, ignoresSideEffects, atRoot, singleUser);
             if (result != Result.OK) {
                 return result;
             }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/match/MatchableNode.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -66,4 +66,9 @@ public @interface MatchableNode {
     boolean shareable() default false;
 
     boolean ignoresSideEffects() default false;
+
+    /**
+     * Can a node be consumed by a matched rule regardless of whether it is shareable.
+     */
+    boolean consumable() default true;
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/GraalError.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -55,6 +55,10 @@ public class GraalError extends Error {
         throw new GraalError(cause);
     }
 
+    public static RuntimeException shouldNotReachHere(Throwable cause, String msg) {
+        throw new GraalError(cause, "should not reach here: %s", msg);
+    }
+
     /**
      * Checks a given condition and throws a {@link GraalError} if it is false. Guarantees are
      * stronger than assertions in that they are always checked. Error messages for guarantee

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/IgvDumpChannel.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -81,7 +81,7 @@ final class IgvDumpChannel implements WritableByteChannel {
 
     WritableByteChannel channel() throws IOException {
         if (closed) {
-            throw new IOException();
+            throw new IOException("already closed");
         }
         if (sharedChannel == null) {
             PrintGraphTarget target = DebugOptions.PrintGraph.getValue(options);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/Node.java

@@ -154,14 +154,18 @@ public abstract class Node implements Cloneable, Formattable, NodeInterface {
     /**
      * Annotates a method that can be replaced by a compiler intrinsic. A (resolved) call to the
      * annotated method will be processed by a generated {@code InvocationPlugin} that calls either
-     * a factory method or a constructor corresponding with the annotated method.
+     * a factory method or a constructor corresponding with the annotated method. By default the
+     * intrinsics are implemented by invoking the constructor but a factory method may be used
+     * instead. To use a factory method the class implementing the intrinsic must be annotated with
+     * {@link NodeIntrinsicFactory}. To ease error checking of NodeIntrinsics all intrinsics are
+     * expected to be implemented in the same way, so it's not possible to mix constructor and
+     * factory intrinsification in the same class.
      * <p>
      * A factory method corresponding to an annotated method is a static method named
      * {@code intrinsify} defined in the class denoted by {@link #value()}. In order, its signature
      * is as follows:
      * <ol>
      * <li>A {@code GraphBuilderContext} parameter.</li>
-     * <li>A {@code ResolvedJavaMethod} parameter.</li>
      * <li>A sequence of zero or more {@linkplain InjectedNodeParameter injected} parameters.</li>
      * <li>Remaining parameters that match the declared parameters of the annotated method.</li>
      * </ol>
@@ -199,6 +203,15 @@ public abstract class Node implements Cloneable, Formattable, NodeInterface {
         boolean hasSideEffect() default false;
     }
 
+    /**
+     * Marker annotation indicating that the class uses factory methods instead of constructors for
+     * intrinsification.
+     */
+    @java.lang.annotation.Retention(RetentionPolicy.RUNTIME)
+    @java.lang.annotation.Target(ElementType.TYPE)
+    public @interface NodeIntrinsicFactory {
+    }
+
     /**
      * Marker for a node that can be replaced by another node via global value numbering. A
      * {@linkplain NodeClass#isLeafNode() leaf} node can be replaced by another node of the same

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/NodeClass.java

@@ -30,10 +30,8 @@ import static org.graalvm.compiler.graph.Edges.translateInto;
 import static org.graalvm.compiler.graph.Graph.isModificationCountsEnabled;
 import static org.graalvm.compiler.graph.InputEdges.translateInto;
 import static org.graalvm.compiler.graph.Node.WithAllEdges;
-import static org.graalvm.compiler.serviceprovider.GraalUnsafeAccess.getUnsafe;
 
 import java.lang.annotation.Annotation;
-import java.lang.invoke.MethodHandles;
 import java.lang.reflect.AnnotatedElement;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
@@ -54,7 +52,6 @@ import org.graalvm.compiler.debug.CounterKey;
 import org.graalvm.compiler.debug.DebugCloseable;
 import org.graalvm.compiler.debug.DebugContext;
 import org.graalvm.compiler.debug.GraalError;
-import org.graalvm.compiler.debug.TTY;
 import org.graalvm.compiler.debug.TimerKey;
 import org.graalvm.compiler.graph.Edges.Type;
 import org.graalvm.compiler.graph.Graph.DuplicationReplacement;
@@ -71,6 +68,7 @@ import org.graalvm.compiler.nodeinfo.NodeCycles;
 import org.graalvm.compiler.nodeinfo.NodeInfo;
 import org.graalvm.compiler.nodeinfo.NodeSize;
 import org.graalvm.compiler.nodeinfo.Verbosity;
+import org.graalvm.compiler.serviceprovider.GraalUnsafeAccess;
 
 import sun.misc.Unsafe;
 
@@ -84,7 +82,7 @@ import sun.misc.Unsafe;
  */
 public final class NodeClass<T> extends FieldIntrospection<T> {
 
-    private static final Unsafe UNSAFE = getUnsafe();
+    private static final Unsafe UNSAFE = GraalUnsafeAccess.getUnsafe();
     // Timers for creation of a NodeClass instance
     private static final TimerKey Init_FieldScanning = DebugContext.timer("NodeClass.Init.FieldScanning");
     private static final TimerKey Init_FieldScanningInner = DebugContext.timer("NodeClass.Init.FieldScanning.Inner");
@@ -132,40 +130,12 @@ public final class NodeClass<T> extends FieldIntrospection<T> {
     }
 
     public static <T> NodeClass<T> get(Class<T> clazz) {
-        int numTries = 0;
-        while (true) {
-            @SuppressWarnings("removal")
-            boolean shouldBeInitializedBefore = UNSAFE.shouldBeInitialized(clazz);
-
         NodeClass<T> result = getUnchecked(clazz);
-            if (result != null || clazz == NODE_CLASS) {
+        if (result == null && clazz != NODE_CLASS) {
-                return result;
+            throw GraalError.shouldNotReachHere("TYPE field not initialized for class " + clazz.getTypeName());
-            }
-
-            /*
-             * GR-9537: We observed a transient problem with TYPE fields being null. Retry a couple
-             * of times and print something to the log so that we can gather more diagnostic
-             * information without failing gates.
-             */
-            numTries++;
-            @SuppressWarnings("removal")
-            boolean shouldBeInitializedAfter = UNSAFE.shouldBeInitialized(clazz);
-            String msg = "GR-9537 Reflective field access of TYPE field returned null. This is probably a bug in HotSpot class initialization. " +
-                            " clazz: " + clazz.getTypeName() + ", numTries: " + numTries +
-                            ", shouldBeInitializedBefore: " + shouldBeInitializedBefore + ", shouldBeInitializedAfter: " + shouldBeInitializedAfter;
-            if (numTries <= 100) {
-                TTY.println(msg);
-                try {
-                    MethodHandles.lookup().ensureInitialized(clazz);
-                } catch (IllegalAccessException e) {
-                    throw new RuntimeException(e);
-                }
-            } else {
-                throw GraalError.shouldNotReachHere(msg);
         }
         return result;
     }
-    }
 
     private static final Class<?> NODE_CLASS = Node.class;
     private static final Class<?> INPUT_LIST_CLASS = NodeInputList.class;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotBackend.java

@@ -395,6 +395,11 @@ public class AArch64HotSpotBackend extends HotSpotHostBackend implements LIRGene
             ForeignCallLinkage linkage = foreignCalls.lookupForeignCall(DEOPT_BLOB_UNPACK);
             masm.adr(lr, 0); // Warning: the argument is an offset from the instruction!
             AArch64Call.directJmp(crb, masm, linkage);
+            if (config.supportsMethodHandleDeoptimizationEntry() && crb.needsMHDeoptHandler()) {
+                crb.recordMark(HotSpotMarkId.DEOPT_MH_HANDLER_ENTRY);
+                masm.adr(lr, 0);
+                AArch64Call.directJmp(crb, masm, linkage);
+            }
         } else {
             // No need to emit the stubs for entries back into the method since
             // it has no calls that can cause such "return" entries

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotBackendFactory.java

@@ -66,7 +66,6 @@ import org.graalvm.compiler.phases.Phase;
 import org.graalvm.compiler.phases.common.AddressLoweringByUsePhase;
 import org.graalvm.compiler.phases.schedule.SchedulePhase;
 import org.graalvm.compiler.phases.tiers.CompilerConfiguration;
-import org.graalvm.compiler.phases.util.Providers;
 import org.graalvm.compiler.replacements.aarch64.AArch64GraphBuilderPlugins;
 import org.graalvm.compiler.serviceprovider.ServiceProvider;
 import org.graalvm.compiler.word.WordTypes;
@@ -151,7 +150,7 @@ public class AArch64HotSpotBackendFactory extends HotSpotBackendFactory {
                 lowerer = createLowerer(graalRuntime, metaAccess, foreignCalls, registers, constantReflection, platformConfigurationProvider, metaAccessExtensionProvider, target);
             }
 
-            Providers p = new Providers(metaAccess, codeCache, constantReflection, constantFieldProvider, foreignCalls, lowerer, null, stampProvider, platformConfigurationProvider,
+            HotSpotProviders p = new HotSpotProviders(metaAccess, codeCache, constantReflection, constantFieldProvider, foreignCalls, lowerer, null, stampProvider, platformConfigurationProvider,
                             metaAccessExtensionProvider);
 
             try (InitTimer rt = timer("create SnippetReflection provider")) {
@@ -172,7 +171,7 @@ public class AArch64HotSpotBackendFactory extends HotSpotBackendFactory {
                 suites = createSuites(config, graalRuntime, compilerConfiguration, plugins, replacements);
             }
             providers = new HotSpotProviders(metaAccess, codeCache, constantReflection, constantFieldProvider, foreignCalls, lowerer, replacements, suites, registers,
-                            snippetReflection, wordTypes, plugins, platformConfigurationProvider, metaAccessExtensionProvider);
+                            snippetReflection, wordTypes, plugins, platformConfigurationProvider, metaAccessExtensionProvider, config);
             replacements.setProviders(providers);
             replacements.maybeInitializeEncoder(options);
         }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotDirectStaticCallOp.java

@@ -36,6 +36,7 @@ import org.graalvm.compiler.lir.aarch64.AArch64Call.DirectCallOp;
 import org.graalvm.compiler.lir.asm.CompilationResultBuilder;
 import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind;
 
+import jdk.vm.ci.hotspot.HotSpotResolvedJavaMethod;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
 import jdk.vm.ci.meta.Value;
 
@@ -69,6 +70,9 @@ final class AArch64HotSpotDirectStaticCallOp extends DirectCallOp {
             // correct inline cache value here.
             crb.recordMark(invokeKind == InvokeKind.Static ? HotSpotMarkId.INVOKESTATIC : HotSpotMarkId.INVOKESPECIAL);
             masm.movNativeAddress(inlineCacheRegister, config.nonOopBits);
+            if (config.supportsMethodHandleDeoptimizationEntry() && config.isMethodHandleCall((HotSpotResolvedJavaMethod) callTarget) && invokeKind != InvokeKind.Static) {
+                crb.setNeedsMHDeoptHandler();
+            }
             super.emitCode(crb, masm);
         }
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotDirectVirtualCallOp.java

@@ -36,6 +36,7 @@ import org.graalvm.compiler.lir.aarch64.AArch64Call.DirectCallOp;
 import org.graalvm.compiler.lir.asm.CompilationResultBuilder;
 import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind;
 
+import jdk.vm.ci.hotspot.HotSpotResolvedJavaMethod;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
 import jdk.vm.ci.meta.Value;
 
@@ -69,6 +70,9 @@ final class AArch64HotSpotDirectVirtualCallOp extends DirectCallOp {
             // correct inline cache value here.
             crb.recordMark(invokeKind == InvokeKind.Virtual ? HotSpotMarkId.INVOKEVIRTUAL : HotSpotMarkId.INVOKEINTERFACE);
             masm.movNativeAddress(inlineCacheRegister, config.nonOopBits);
+            if (config.supportsMethodHandleDeoptimizationEntry() && config.isMethodHandleCall((HotSpotResolvedJavaMethod) callTarget)) {
+                crb.setNeedsMHDeoptHandler();
+            }
             super.emitCode(crb, masm);
         }
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/StubAVXTest.java

@@ -235,7 +235,7 @@ public class StubAVXTest extends LIRTest {
     @Test
     public void test() {
         HotSpotProviders providers = (HotSpotProviders) getProviders();
-        HotSpotForeignCallsProviderImpl foreignCalls = (HotSpotForeignCallsProviderImpl) providers.getForeignCalls();
+        HotSpotForeignCallsProviderImpl foreignCalls = providers.getForeignCalls();
         HotSpotForeignCallLinkage linkage = foreignCalls.registerStubCall(TEST_STUB, HotSpotForeignCallDescriptor.Transition.LEAF_NO_VZERO,
                         HotSpotForeignCallDescriptor.Reexecutability.REEXECUTABLE,
                         COMPUTES_REGISTERS_KILLED);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackend.java

@@ -331,6 +331,10 @@ public class AMD64HotSpotBackend extends HotSpotHostBackend implements LIRGenera
             AMD64Call.directCall(crb, asm, foreignCalls.lookupForeignCall(EXCEPTION_HANDLER), null, false, null);
             crb.recordMark(HotSpotMarkId.DEOPT_HANDLER_ENTRY);
             AMD64Call.directCall(crb, asm, foreignCalls.lookupForeignCall(DEOPT_BLOB_UNPACK), null, false, null);
+            if (config.supportsMethodHandleDeoptimizationEntry() && crb.needsMHDeoptHandler()) {
+                crb.recordMark(HotSpotMarkId.DEOPT_MH_HANDLER_ENTRY);
+                AMD64Call.directCall(crb, asm, foreignCalls.lookupForeignCall(DEOPT_BLOB_UNPACK), null, false, null);
+            }
         } else {
             // No need to emit the stubs for entries back into the method since
             // it has no calls that can cause such "return" entries

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackendFactory.java

@@ -58,7 +58,6 @@ import org.graalvm.compiler.nodes.spi.Replacements;
 import org.graalvm.compiler.options.OptionValues;
 import org.graalvm.compiler.phases.common.AddressLoweringPhase;
 import org.graalvm.compiler.phases.tiers.CompilerConfiguration;
-import org.graalvm.compiler.phases.util.Providers;
 import org.graalvm.compiler.replacements.amd64.AMD64GraphBuilderPlugins;
 import org.graalvm.compiler.serviceprovider.JavaVersionUtil;
 import org.graalvm.compiler.serviceprovider.ServiceProvider;
@@ -143,7 +142,7 @@ public class AMD64HotSpotBackendFactory extends HotSpotBackendFactory {
                 lowerer = createLowerer(graalRuntime, metaAccess, foreignCalls, registers, constantReflection, platformConfigurationProvider, metaAccessExtensionProvider, target);
             }
 
-            Providers p = new Providers(metaAccess, codeCache, constantReflection, constantFieldProvider, foreignCalls, lowerer, null, stampProvider, platformConfigurationProvider,
+            HotSpotProviders p = new HotSpotProviders(metaAccess, codeCache, constantReflection, constantFieldProvider, foreignCalls, lowerer, null, stampProvider, platformConfigurationProvider,
                             metaAccessExtensionProvider);
 
             try (InitTimer rt = timer("create SnippetReflection provider")) {
@@ -164,7 +163,7 @@ public class AMD64HotSpotBackendFactory extends HotSpotBackendFactory {
                 suites = createSuites(config, graalRuntime, compilerConfiguration, plugins, registers, replacements, options);
             }
             providers = new HotSpotProviders(metaAccess, codeCache, constantReflection, constantFieldProvider, foreignCalls, lowerer, replacements, suites, registers,
-                            snippetReflection, wordTypes, plugins, platformConfigurationProvider, metaAccessExtensionProvider);
+                            snippetReflection, wordTypes, plugins, platformConfigurationProvider, metaAccessExtensionProvider, config);
             replacements.setProviders(providers);
             replacements.maybeInitializeEncoder(options);
         }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotDirectStaticCallOp.java

@@ -25,6 +25,7 @@
 package org.graalvm.compiler.hotspot.amd64;
 
 import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler;
+import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
 import org.graalvm.compiler.hotspot.HotSpotMarkId;
 import org.graalvm.compiler.lir.LIRFrameState;
 import org.graalvm.compiler.lir.LIRInstructionClass;
@@ -33,6 +34,7 @@ import org.graalvm.compiler.lir.amd64.AMD64Call.DirectCallOp;
 import org.graalvm.compiler.lir.asm.CompilationResultBuilder;
 import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind;
 
+import jdk.vm.ci.hotspot.HotSpotResolvedJavaMethod;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
 import jdk.vm.ci.meta.Value;
 
@@ -45,11 +47,13 @@ final class AMD64HotSpotDirectStaticCallOp extends DirectCallOp {
     public static final LIRInstructionClass<AMD64HotSpotDirectStaticCallOp> TYPE = LIRInstructionClass.create(AMD64HotSpotDirectStaticCallOp.class);
 
     private final InvokeKind invokeKind;
+    private final GraalHotSpotVMConfig config;
 
-    AMD64HotSpotDirectStaticCallOp(ResolvedJavaMethod target, Value result, Value[] parameters, Value[] temps, LIRFrameState state, InvokeKind invokeKind) {
+    AMD64HotSpotDirectStaticCallOp(ResolvedJavaMethod target, Value result, Value[] parameters, Value[] temps, LIRFrameState state, InvokeKind invokeKind, GraalHotSpotVMConfig config) {
         super(TYPE, target, result, parameters, temps, state);
         assert invokeKind.isDirect();
         this.invokeKind = invokeKind;
+        this.config = config;
     }
 
     @Override
@@ -57,6 +61,9 @@ final class AMD64HotSpotDirectStaticCallOp extends DirectCallOp {
     public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
         try (CompilationResultBuilder.CallContext callContext = crb.openCallContext(invokeKind.isDirect())) {
             crb.recordMark(invokeKind == InvokeKind.Static ? HotSpotMarkId.INVOKESTATIC : HotSpotMarkId.INVOKESPECIAL);
+            if (config.supportsMethodHandleDeoptimizationEntry() && config.isMethodHandleCall((HotSpotResolvedJavaMethod) callTarget) && invokeKind != InvokeKind.Static) {
+                crb.setNeedsMHDeoptHandler();
+            }
             super.emitCode(crb, masm);
         }
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotLoweringProvider.java

@@ -24,6 +24,7 @@
 
 package org.graalvm.compiler.hotspot.amd64;
 
+import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
 import static org.graalvm.compiler.hotspot.HotSpotBackend.Options.GraalArithmeticStubs;
 
 import org.graalvm.compiler.core.amd64.AMD64LoweringProviderMixin;
@@ -70,10 +71,8 @@ public class AMD64HotSpotLoweringProvider extends DefaultHotSpotLoweringProvider
     @Override
     public void initialize(OptionValues options, Iterable<DebugHandlersFactory> factories, HotSpotProviders providers, GraalHotSpotVMConfig config) {
         convertSnippets = new AMD64ConvertSnippets.Templates(options, factories, providers, providers.getSnippetReflection(), providers.getCodeCache().getTarget());
-        if (JavaVersionUtil.JAVA_SPEC <= 8) {
+        if (JavaVersionUtil.JAVA_SPEC >= 11 && GeneratePIC.getValue(options)) {
             // AOT only introduced in JDK 9
-            profileSnippets = null;
-        } else {
             profileSnippets = new ProbabilisticProfileSnippets.Templates(options, factories, providers, providers.getCodeCache().getTarget());
         }
         mathSnippets = new AMD64X87MathSnippets.Templates(options, factories, providers, providers.getSnippetReflection(), providers.getCodeCache().getTarget());

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotNodeLIRBuilder.java

@@ -142,7 +142,7 @@ public class AMD64HotSpotNodeLIRBuilder extends AMD64NodeLIRBuilder implements H
             assert invokeKind.isDirect();
             HotSpotResolvedJavaMethod resolvedMethod = (HotSpotResolvedJavaMethod) callTarget.targetMethod();
             assert resolvedMethod.isConcrete() : "Cannot make direct call to abstract method.";
-            append(new AMD64HotSpotDirectStaticCallOp(callTarget.targetMethod(), result, parameters, temps, callState, invokeKind));
+            append(new AMD64HotSpotDirectStaticCallOp(callTarget.targetMethod(), result, parameters, temps, callState, invokeKind, getGen().config));
         }
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotspotDirectVirtualCallOp.java

@@ -35,6 +35,7 @@ import org.graalvm.compiler.lir.asm.CompilationResultBuilder;
 import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind;
 
 import jdk.vm.ci.amd64.AMD64;
+import jdk.vm.ci.hotspot.HotSpotResolvedJavaMethod;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
 import jdk.vm.ci.meta.Value;
 
@@ -65,6 +66,9 @@ final class AMD64HotspotDirectVirtualCallOp extends DirectCallOp {
             crb.recordMark(invokeKind == InvokeKind.Virtual ? HotSpotMarkId.INVOKEVIRTUAL : HotSpotMarkId.INVOKEINTERFACE);
             // This must be emitted exactly like this to ensure it's patchable
             masm.movq(AMD64.rax, config.nonOopBits);
+            if (config.supportsMethodHandleDeoptimizationEntry() && config.isMethodHandleCall((HotSpotResolvedJavaMethod) callTarget)) {
+                crb.setNeedsMHDeoptHandler();
+            }
             super.emitCall(crb, masm);
         }
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CheckGraalIntrinsics.java

@@ -378,9 +378,11 @@ public class CheckGraalIntrinsics extends GraalTest {
         }
 
         if (isJDK10OrHigher()) {
+            if (!(arch instanceof AArch64)) {
                 add(toBeInvestigated,
                                 "java/lang/Math.multiplyHigh(JJ)J");
             }
+        }
 
         if (isJDK11OrHigher()) {
             // Relevant for Java flight recorder
@@ -397,9 +399,12 @@ public class CheckGraalIntrinsics extends GraalTest {
         }
 
         if (isJDK13OrHigher()) {
+            if (!(arch instanceof AArch64)) {
                 add(toBeInvestigated,
                                 "java/lang/Math.abs(I)I",
-                            "java/lang/Math.abs(J)J",
+                                "java/lang/Math.abs(J)J");
+            }
+            add(toBeInvestigated,
                             "java/lang/Math.max(DD)D",
                             "java/lang/Math.max(FF)F",
                             "java/lang/Math.min(DD)D",

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CompileTheWorld.java

@@ -1102,7 +1102,7 @@ public final class CompileTheWorld {
         public static final OptionKey<String> Config = new OptionKey<>(null);
         public static final OptionKey<Boolean> MultiThreaded = new OptionKey<>(false);
         public static final OptionKey<Integer> Threads = new OptionKey<>(0);
-        public static final OptionKey<Boolean> InvalidateInstalledCode = new OptionKey<>(false);
+        public static final OptionKey<Boolean> InvalidateInstalledCode = new OptionKey<>(true);
 
         // @formatter:off
         static final ReflectionOptionDescriptors DESCRIPTORS = new ReflectionOptionDescriptors(Options.class,

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/CommunityCompilerConfigurationFactory.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
 package org.graalvm.compiler.hotspot;
 
 import org.graalvm.compiler.core.phases.CommunityCompilerConfiguration;
+import org.graalvm.compiler.options.OptionValues;
 import org.graalvm.compiler.phases.tiers.CompilerConfiguration;
 import org.graalvm.compiler.serviceprovider.ServiceProvider;
 
@@ -50,4 +51,9 @@ public class CommunityCompilerConfigurationFactory extends CompilerConfiguration
     public CompilerConfiguration createCompilerConfiguration() {
         return new CommunityCompilerConfiguration();
     }
+
+    @Override
+    public Instrumentation createInstrumentation(OptionValues options) {
+        return new DefaultInstrumentation();
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/CompilerConfigurationFactory.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -96,6 +96,8 @@ public abstract class CompilerConfigurationFactory implements Comparable<Compile
 
     public abstract CompilerConfiguration createCompilerConfiguration();
 
+    public abstract Instrumentation createInstrumentation(OptionValues options);
+
     /**
      * Collect the set of available {@linkplain HotSpotBackendFactory backends} for this compiler
      * configuration.

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/DefaultInstrumentation.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package org.graalvm.compiler.hotspot;
+
+/**
+ * Default version of {@link Instrumentation}.
+ */
+public class DefaultInstrumentation implements Instrumentation {
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/EconomyCompilerConfigurationFactory.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
 package org.graalvm.compiler.hotspot;
 
 import org.graalvm.compiler.core.phases.EconomyCompilerConfiguration;
+import org.graalvm.compiler.options.OptionValues;
 import org.graalvm.compiler.phases.tiers.CompilerConfiguration;
 import org.graalvm.compiler.serviceprovider.ServiceProvider;
 
@@ -52,4 +53,9 @@ public class EconomyCompilerConfigurationFactory extends CompilerConfigurationFa
         // the economy configuration only differs in the frontend, it reuses the "community" backend
         return new DefaultBackendMap(CommunityCompilerConfigurationFactory.NAME);
     }
+
+    @Override
+    public Instrumentation createInstrumentation(OptionValues options) {
+        return new DefaultInstrumentation();
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/EncodedSnippets.java

@@ -28,6 +28,8 @@ import static jdk.vm.ci.runtime.JVMCI.getRuntime;
 import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE;
 import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.INLINE_AFTER_PARSING;
 
+import java.util.function.Consumer;
+
 import jdk.internal.vm.compiler.collections.UnmodifiableEconomicMap;
 import org.graalvm.compiler.api.runtime.GraalJVMCICompiler;
 import org.graalvm.compiler.api.runtime.GraalRuntime;
@@ -51,7 +53,6 @@ import org.graalvm.compiler.options.OptionValues;
 import org.graalvm.compiler.phases.util.Providers;
 import org.graalvm.compiler.replacements.ConstantBindingParameterPlugin;
 import org.graalvm.compiler.replacements.PEGraphDecoder;
-import org.graalvm.compiler.replacements.ReplacementsImpl;
 
 import jdk.vm.ci.meta.ResolvedJavaField;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
@@ -61,52 +62,54 @@ import jdk.vm.ci.meta.UnresolvedJavaMethod;
 import jdk.vm.ci.meta.UnresolvedJavaType;
 
 public class EncodedSnippets {
+    static class GraphData {
+        int startOffset;
+        String originalMethod;
+
+        SnippetParameterInfo info;
+
+        GraphData(int startOffset, String originalMethod, SnippetParameterInfo info) {
+            this.startOffset = startOffset;
+            this.originalMethod = originalMethod;
+            this.info = info;
+        }
+    }
+
     private final byte[] snippetEncoding;
     private final Object[] snippetObjects;
     private final NodeClass<?>[] snippetNodeClasses;
-    private final UnmodifiableEconomicMap<String, Integer> snippetStartOffsets;
+    private final UnmodifiableEconomicMap<String, GraphData> graphDatas;
-    private final UnmodifiableEconomicMap<String, String> originalMethods;
-    private UnmodifiableEconomicMap<String, SnippetParameterInfo> snippetParameterInfos;
 
-    EncodedSnippets(byte[] snippetEncoding, Object[] snippetObjects, NodeClass<?>[] snippetNodeClasses, UnmodifiableEconomicMap<String, Integer> snippetStartOffsets,
+    EncodedSnippets(byte[] snippetEncoding, Object[] snippetObjects, NodeClass<?>[] snippetNodeClasses, UnmodifiableEconomicMap<String, GraphData> graphDatas) {
-                    UnmodifiableEconomicMap<String, String> originalMethods, UnmodifiableEconomicMap<String, SnippetParameterInfo> snippetParameterInfos) {
         this.snippetEncoding = snippetEncoding;
         this.snippetObjects = snippetObjects;
         this.snippetNodeClasses = snippetNodeClasses;
-        this.snippetStartOffsets = snippetStartOffsets;
+        this.graphDatas = graphDatas;
-        this.originalMethods = originalMethods;
-        this.snippetParameterInfos = snippetParameterInfos;
-    }
-
-    public byte[] getSnippetEncoding() {
-        return snippetEncoding;
     }
 
     public NodeClass<?>[] getSnippetNodeClasses() {
         return snippetNodeClasses;
     }
 
-    public UnmodifiableEconomicMap<String, Integer> getSnippetStartOffsets() {
+    public void visitImmutable(Consumer<Object> visitor) {
-        return snippetStartOffsets;
+        visitor.accept(snippetEncoding);
+        visitor.accept(snippetNodeClasses);
+        visitor.accept(graphDatas);
     }
 
-    public UnmodifiableEconomicMap<String, String> getOriginalMethods() {
+    StructuredGraph getMethodSubstitutionGraph(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original, HotSpotReplacementsImpl replacements, IntrinsicContext.CompilationContext context,
-        return originalMethods;
-    }
-
-    StructuredGraph getMethodSubstitutionGraph(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original, ReplacementsImpl replacements, IntrinsicContext.CompilationContext context,
                     StructuredGraph.AllowAssumptions allowAssumptions, Cancellable cancellable, OptionValues options) {
         IntrinsicContext.CompilationContext contextToUse = context;
         if (context == IntrinsicContext.CompilationContext.ROOT_COMPILATION) {
             contextToUse = IntrinsicContext.CompilationContext.ROOT_COMPILATION_ENCODING;
         }
-        Integer startOffset = snippetStartOffsets.get(plugin.toString() + contextToUse);
+        GraphData data = graphDatas.get(plugin.toString() + contextToUse);
-        if (startOffset == null) {
+        if (data == null) {
             throw GraalError.shouldNotReachHere("plugin graph not found: " + plugin + " with " + contextToUse);
         }
 
         ResolvedJavaType accessingClass = replacements.getProviders().getMetaAccess().lookupJavaType(plugin.getDeclaringClass());
-        return decodeGraph(original, accessingClass, startOffset, replacements, contextToUse, allowAssumptions, cancellable, options);
+        return decodeGraph(original, accessingClass, data.startOffset, replacements, contextToUse, allowAssumptions, cancellable, options);
     }
 
     /**
@@ -121,7 +124,7 @@ public class EncodedSnippets {
     private StructuredGraph decodeGraph(ResolvedJavaMethod method,
                     ResolvedJavaType accessingClass,
                     int startOffset,
-                    ReplacementsImpl replacements,
+                    HotSpotReplacementsImpl replacements,
                     IntrinsicContext.CompilationContext context,
                     StructuredGraph.AllowAssumptions allowAssumptions,
                     Cancellable cancellable,
@@ -129,24 +132,23 @@ public class EncodedSnippets {
         Providers providers = replacements.getProviders();
         EncodedGraph encodedGraph = new SymbolicEncodedGraph(snippetEncoding, startOffset, snippetObjects, snippetNodeClasses,
                         methodKey(method), accessingClass, method.getDeclaringClass());
-        try (DebugContext debug = replacements.openDebugContext("SVMSnippet_", method, options)) {
+        try (DebugContext debug = replacements.openSnippetDebugContext("LibgraalSnippet_", method, options)) {
-            boolean isSubstitution = true;
+            StructuredGraph result = new StructuredGraph.Builder(options, debug, allowAssumptions).cancellable(cancellable).method(method).setIsSubstitution(true).build();
-            StructuredGraph result = new StructuredGraph.Builder(options, debug, allowAssumptions).cancellable(cancellable).method(method).setIsSubstitution(isSubstitution).build();
+            PEGraphDecoder graphDecoder = new SubstitutionGraphDecoder(providers, result, replacements, null, method, context, encodedGraph, true);
-            PEGraphDecoder graphDecoder = new SubstitutionGraphDecoder(providers, result, replacements, null, method, context, encodedGraph);
 
-            graphDecoder.decode(method, isSubstitution, encodedGraph.trackNodeSourcePosition());
+            graphDecoder.decode(method, result.isSubstitution(), encodedGraph.trackNodeSourcePosition());
 
             assert result.verify();
             return result;
         }
     }
 
-    StructuredGraph getEncodedSnippet(ResolvedJavaMethod method, ReplacementsImpl replacements, Object[] args, StructuredGraph.AllowAssumptions allowAssumptions, OptionValues options) {
+    StructuredGraph getEncodedSnippet(ResolvedJavaMethod method, HotSpotReplacementsImpl replacements, Object[] args, StructuredGraph.AllowAssumptions allowAssumptions, OptionValues options) {
-        Integer startOffset = null;
+        GraphData data = null;
-        if (snippetStartOffsets != null) {
+        if (graphDatas != null) {
-            startOffset = snippetStartOffsets.get(methodKey(method));
+            data = graphDatas.get(methodKey(method));
         }
-        if (startOffset == null) {
+        if (data == null) {
             if (IS_IN_NATIVE_IMAGE) {
                 throw GraalError.shouldNotReachHere("snippet not found: " + method.format("%H.%n(%p)"));
             } else {
@@ -154,31 +156,39 @@ public class EncodedSnippets {
             }
         }
 
-        SymbolicEncodedGraph encodedGraph = new SymbolicEncodedGraph(snippetEncoding, startOffset, snippetObjects, snippetNodeClasses,
+        SymbolicEncodedGraph encodedGraph = new SymbolicEncodedGraph(snippetEncoding, data.startOffset, snippetObjects, snippetNodeClasses, data.originalMethod, method.getDeclaringClass());
-                        originalMethods.get(methodKey(method)), method.getDeclaringClass());
+        return decodeSnippetGraph(encodedGraph, method, replacements, args, allowAssumptions, options, IS_IN_NATIVE_IMAGE);
-        return decodeSnippetGraph(encodedGraph, method, replacements, args, allowAssumptions, options);
     }
 
     public SnippetParameterInfo getSnippetParameterInfo(ResolvedJavaMethod method) {
-        SnippetParameterInfo info = snippetParameterInfos.get(methodKey(method));
+        GraphData data = null;
+        if (graphDatas != null) {
+            data = graphDatas.get(methodKey(method));
+        }
+        assert data != null : method;
+        SnippetParameterInfo info = data.info;
         assert info != null;
         return info;
     }
 
     public boolean isSnippet(ResolvedJavaMethod method) {
-        return snippetParameterInfos.get(methodKey(method)) != null;
+        GraphData data = null;
+        if (graphDatas != null) {
+            data = graphDatas.get(methodKey(method));
+        }
+        return data != null && data.info != null;
     }
 
     @SuppressWarnings("try")
-    private static StructuredGraph decodeSnippetGraph(SymbolicEncodedGraph encodedGraph, ResolvedJavaMethod method, ReplacementsImpl replacements, Object[] args,
+    static StructuredGraph decodeSnippetGraph(SymbolicEncodedGraph encodedGraph, ResolvedJavaMethod method, HotSpotReplacementsImpl replacements, Object[] args,
-                    StructuredGraph.AllowAssumptions allowAssumptions, OptionValues options) {
+                    StructuredGraph.AllowAssumptions allowAssumptions, OptionValues options, boolean mustSucceed) {
         Providers providers = replacements.getProviders();
         ParameterPlugin parameterPlugin = null;
         if (args != null) {
             parameterPlugin = new ConstantBindingParameterPlugin(args, providers.getMetaAccess(), replacements.snippetReflection);
         }
 
-        try (DebugContext debug = replacements.openDebugContext("SVMSnippet_", method, options)) {
+        try (DebugContext debug = replacements.openSnippetDebugContext("SVMSnippet_", method, options)) {
             // @formatter:off
             boolean isSubstitution = true;
             StructuredGraph result = new StructuredGraph.Builder(options, debug, allowAssumptions)
@@ -188,7 +198,7 @@ public class EncodedSnippets {
                     .build();
             // @formatter:on
             try (DebugContext.Scope scope = debug.scope("DecodeSnippetGraph", result)) {
-                PEGraphDecoder graphDecoder = new SubstitutionGraphDecoder(providers, result, replacements, parameterPlugin, method, INLINE_AFTER_PARSING, encodedGraph);
+                PEGraphDecoder graphDecoder = new SubstitutionGraphDecoder(providers, result, replacements, parameterPlugin, method, INLINE_AFTER_PARSING, encodedGraph, mustSucceed);
 
                 graphDecoder.decode(method, isSubstitution, encodedGraph.trackNodeSourcePosition());
                 debug.dump(DebugContext.VERBOSE_LEVEL, result, "After decoding");
@@ -201,18 +211,20 @@ public class EncodedSnippets {
         }
     }
 
-    public static class SubstitutionGraphDecoder extends PEGraphDecoder {
+    static class SubstitutionGraphDecoder extends PEGraphDecoder {
         private final ResolvedJavaMethod method;
         private final EncodedGraph encodedGraph;
         private IntrinsicContext intrinsic;
+        private final boolean mustSucceed;
 
-        SubstitutionGraphDecoder(Providers providers, StructuredGraph result, ReplacementsImpl replacements, ParameterPlugin parameterPlugin, ResolvedJavaMethod method,
+        SubstitutionGraphDecoder(Providers providers, StructuredGraph result, HotSpotReplacementsImpl replacements, ParameterPlugin parameterPlugin, ResolvedJavaMethod method,
-                        IntrinsicContext.CompilationContext context, EncodedGraph encodedGraph) {
+                        IntrinsicContext.CompilationContext context, EncodedGraph encodedGraph, boolean mustSucceed) {
             super(providers.getCodeCache().getTarget().arch, result, providers, null,
                             replacements.getGraphBuilderPlugins().getInvocationPlugins(), new InlineInvokePlugin[0], parameterPlugin,
                             null, null, null);
             this.method = method;
             this.encodedGraph = encodedGraph;
+            this.mustSucceed = mustSucceed;
             intrinsic = new IntrinsicContext(method, null, replacements.getDefaultReplacementBytecodeProvider(), context, false);
         }
 
@@ -230,9 +242,14 @@ public class EncodedSnippets {
         }
 
         @Override
-        protected IntrinsicContext getIntrinsic() {
+        public IntrinsicContext getIntrinsic() {
             return intrinsic;
         }
+
+        @Override
+        protected boolean pluginReplacementMustSucceed() {
+            return mustSucceed;
+        }
     }
 
     static class SymbolicEncodedGraph extends EncodedGraph {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/GraalHotSpotVMConfig.java

@@ -886,6 +886,7 @@ public class GraalHotSpotVMConfig extends GraalHotSpotVMConfigAccess {
     public final int OSR_ENTRY = getConstant("CodeInstaller::OSR_ENTRY", Integer.class);
     public final int EXCEPTION_HANDLER_ENTRY = getConstant("CodeInstaller::EXCEPTION_HANDLER_ENTRY", Integer.class);
     public final int DEOPT_HANDLER_ENTRY = getConstant("CodeInstaller::DEOPT_HANDLER_ENTRY", Integer.class);
+    public final int DEOPT_MH_HANDLER_ENTRY = getConstant("CodeInstaller::DEOPT_MH_HANDLER_ENTRY", Integer.class, -1, (JVMCI ? jvmciGE(JVMCI_20_2_b01) : false));
     public final int FRAME_COMPLETE = getConstant("CodeInstaller::FRAME_COMPLETE", Integer.class, -1, (JVMCI ? jvmciGE(JVMCI_20_1_b01) : JDK_8245443));
     public final int INVOKEINTERFACE = getConstant("CodeInstaller::INVOKEINTERFACE", Integer.class);
     public final int INVOKEVIRTUAL = getConstant("CodeInstaller::INVOKEVIRTUAL", Integer.class);
@@ -899,6 +900,10 @@ public class GraalHotSpotVMConfig extends GraalHotSpotVMConfigAccess {
     public final int CARD_TABLE_SHIFT = getConstant("CodeInstaller::CARD_TABLE_SHIFT", Integer.class);
     public final int CARD_TABLE_ADDRESS = getConstant("CodeInstaller::CARD_TABLE_ADDRESS", Integer.class);
     public final int INVOKE_INVALID = getConstant("CodeInstaller::INVOKE_INVALID", Integer.class);
+    public final int VMINTRINSIC_FIRST_MH_SIG_POLY = getConstant("vmIntrinsics::FIRST_MH_SIG_POLY", Integer.class, -1, (JVMCI ? jvmciGE(JVMCI_20_2_b01) : false));
+    public final int VMINTRINSIC_LAST_MH_SIG_POLY = getConstant("vmIntrinsics::LAST_MH_SIG_POLY", Integer.class, -1, (JVMCI ? jvmciGE(JVMCI_20_2_b01) : false));
+    public final int VMINTRINSIC_INVOKE_GENERIC = getConstant("vmIntrinsics::_invokeGeneric", Integer.class, -1, (JVMCI ? jvmciGE(JVMCI_20_2_b01) : false));
+    public final int VMINTRINSIC_COMPILED_LAMBDA_FORM = getConstant("vmIntrinsics::_compiledLambdaForm", Integer.class, -1, (JVMCI ? jvmciGE(JVMCI_20_2_b01) : false));
 
     public final boolean CPU_HAS_INTEL_JCC_ERRATUM = getFieldValue("VM_Version::_has_intel_jcc_erratum", Boolean.class, "bool",
                     true, "amd64".equals(osArch) && (JVMCI ? jvmciGE(JVMCI_20_1_b01) : JDK >= 15));
@@ -926,4 +931,17 @@ public class GraalHotSpotVMConfig extends GraalHotSpotVMConfigAccess {
         assert checkNullAllocationStubs();
         return true;
     }
+
+    public boolean isMethodHandleCall(HotSpotResolvedJavaMethod targetMethod) {
+        int intrinsicId = targetMethod.intrinsicId();
+        return ((intrinsicId >= VMINTRINSIC_FIRST_MH_SIG_POLY && intrinsicId <= VMINTRINSIC_LAST_MH_SIG_POLY) // MethodHandles::is_signature_polymorphic
+                        && intrinsicId != VMINTRINSIC_INVOKE_GENERIC) // MethodHandles::is_signature_polymorphic_intrinsic
+                        || intrinsicId == VMINTRINSIC_COMPILED_LAMBDA_FORM; // ciMethod::is_compiled_lambda_form
+    }
+
+    public boolean supportsMethodHandleDeoptimizationEntry() {
+        return DEOPT_MH_HANDLER_ENTRY != -1 && VMINTRINSIC_FIRST_MH_SIG_POLY != -1 && VMINTRINSIC_LAST_MH_SIG_POLY != -1 && VMINTRINSIC_INVOKE_GENERIC != -1 &&
+                        VMINTRINSIC_COMPILED_LAMBDA_FORM != -1;
+    }
+
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/GraalHotSpotVMConfigAccess.java

@@ -127,6 +127,7 @@ public class GraalHotSpotVMConfigAccess {
     public final String osArch;
 
     protected static final Version JVMCI_0_55 = new Version2(0, 55);
+    protected static final Version JVMCI_20_2_b01 = new Version3(20, 2, 1);
     protected static final Version JVMCI_20_1_b01 = new Version3(20, 1, 1);
     protected static final Version JVMCI_20_0_b03 = new Version3(20, 0, 3);
     protected static final Version JVMCI_19_3_b03 = new Version3(19, 3, 3);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotBackendFactory.java

@@ -26,13 +26,13 @@ package org.graalvm.compiler.hotspot;
 
 import org.graalvm.compiler.bytecode.BytecodeProvider;
 import org.graalvm.compiler.hotspot.meta.HotSpotGraalConstantFieldProvider;
+import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
 import org.graalvm.compiler.hotspot.meta.HotSpotMetaAccessExtensionProvider;
 import org.graalvm.compiler.hotspot.meta.HotSpotPlatformConfigurationProvider;
 import org.graalvm.compiler.hotspot.meta.HotSpotSnippetReflectionProvider;
 import org.graalvm.compiler.hotspot.meta.HotSpotStampProvider;
 import org.graalvm.compiler.hotspot.word.HotSpotWordTypes;
 import org.graalvm.compiler.phases.tiers.CompilerConfiguration;
-import org.graalvm.compiler.phases.util.Providers;
 import org.graalvm.compiler.replacements.classfile.ClassfileBytecodeProvider;
 
 import jdk.vm.ci.code.Architecture;
@@ -64,7 +64,7 @@ public abstract class HotSpotBackendFactory {
         return new HotSpotMetaAccessExtensionProvider();
     }
 
-    protected HotSpotReplacementsImpl createReplacements(TargetDescription target, Providers p, HotSpotSnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider) {
+    protected HotSpotReplacementsImpl createReplacements(TargetDescription target, HotSpotProviders p, HotSpotSnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider) {
         return new HotSpotReplacementsImpl(p, snippetReflection, bytecodeProvider, target);
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalRuntime.java

@@ -24,20 +24,22 @@
 
 package org.graalvm.compiler.hotspot;
 
-import static jdk.vm.ci.common.InitTimer.timer;
+import jdk.vm.ci.code.Architecture;
-import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntime.runtime;
+import jdk.vm.ci.code.stack.StackIntrospection;
-import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
+import jdk.vm.ci.common.InitTimer;
-import static org.graalvm.compiler.core.common.GraalOptions.HotSpotPrintInlining;
+import jdk.vm.ci.hotspot.HotSpotCompilationRequest;
-import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigAccess.JDK;
+import jdk.vm.ci.hotspot.HotSpotJVMCIRuntime;
-
+import jdk.vm.ci.hotspot.HotSpotResolvedJavaMethod;
-import java.io.PrintStream;
+import jdk.vm.ci.hotspot.HotSpotResolvedJavaType;
-import java.util.ArrayList;
+import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
-import java.util.EnumMap;
+import jdk.vm.ci.hotspot.HotSpotVMConfigStore;
-import java.util.List;
+import jdk.vm.ci.meta.JavaKind;
-import java.util.Map;
+import jdk.vm.ci.meta.MetaAccessProvider;
-import java.util.concurrent.atomic.AtomicReference;
+import jdk.vm.ci.meta.ResolvedJavaMethod;
-import java.util.function.Supplier;
+import jdk.vm.ci.meta.ResolvedJavaType;
-
+import jdk.vm.ci.runtime.JVMCI;
+import jdk.vm.ci.runtime.JVMCIBackend;
+import jdk.vm.ci.services.Services;
 import jdk.internal.vm.compiler.collections.EconomicMap;
 import jdk.internal.vm.compiler.collections.EconomicSet;
 import jdk.internal.vm.compiler.collections.Equivalence;
@@ -78,22 +80,19 @@ import org.graalvm.compiler.replacements.SnippetCounter.Group;
 import org.graalvm.compiler.runtime.RuntimeProvider;
 import org.graalvm.compiler.serviceprovider.GraalServices;
 
-import jdk.vm.ci.code.Architecture;
+import java.io.PrintStream;
-import jdk.vm.ci.code.stack.StackIntrospection;
+import java.util.ArrayList;
-import jdk.vm.ci.common.InitTimer;
+import java.util.EnumMap;
-import jdk.vm.ci.hotspot.HotSpotCompilationRequest;
+import java.util.List;
-import jdk.vm.ci.hotspot.HotSpotJVMCIRuntime;
+import java.util.Map;
-import jdk.vm.ci.hotspot.HotSpotResolvedJavaMethod;
+import java.util.concurrent.atomic.AtomicReference;
-import jdk.vm.ci.hotspot.HotSpotResolvedJavaType;
+import java.util.function.Supplier;
-import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
+
-import jdk.vm.ci.hotspot.HotSpotVMConfigStore;
+import static jdk.vm.ci.common.InitTimer.timer;
-import jdk.vm.ci.meta.JavaKind;
+import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntime.runtime;
-import jdk.vm.ci.meta.MetaAccessProvider;
+import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
-import jdk.vm.ci.meta.ResolvedJavaMethod;
+import static org.graalvm.compiler.core.common.GraalOptions.HotSpotPrintInlining;
-import jdk.vm.ci.meta.ResolvedJavaType;
+import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigAccess.JDK;
-import jdk.vm.ci.runtime.JVMCI;
-import jdk.vm.ci.runtime.JVMCIBackend;
-import jdk.vm.ci.services.Services;
 
 //JaCoCo Exclude
 
@@ -138,6 +137,8 @@ public final class HotSpotGraalRuntime implements HotSpotGraalRuntimeProvider {
 
     private final GraalHotSpotVMConfig config;
 
+    private final Instrumentation instrumentation;
+
     /**
      * The options can be {@linkplain #setOptionValues(String[], String[]) updated} by external
      * interfaces such as JMX. This comes with the risk that inconsistencies can arise as an
@@ -178,6 +179,8 @@ public final class HotSpotGraalRuntime implements HotSpotGraalRuntimeProvider {
         CompilerConfiguration compilerConfiguration = compilerConfigurationFactory.createCompilerConfiguration();
         compilerConfigurationName = compilerConfigurationFactory.getName();
 
+        this.instrumentation = compilerConfigurationFactory.createInstrumentation(options);
+
         if (IS_AOT) {
             management = AOT_INJECTED_MANAGEMENT == null ? null : AOT_INJECTED_MANAGEMENT.get();
         } else {
@@ -427,6 +430,11 @@ public final class HotSpotGraalRuntime implements HotSpotGraalRuntimeProvider {
         return compilerConfigurationName;
     }
 
+    @Override
+    public Instrumentation getInstrumentation() {
+        return instrumentation;
+    }
+
     private long runtimeStartTime;
 
     /**
@@ -492,7 +500,7 @@ public final class HotSpotGraalRuntime implements HotSpotGraalRuntimeProvider {
     /**
      * Substituted by
      * {@code com.oracle.svm.graal.hotspot.libgraal.Target_org_graalvm_compiler_hotspot_HotSpotGraalRuntime}
-     * to call {@code org.graalvm.nativeimage.VMRuntime.shutdown()}.
+     * to call {@code jdk.internal.vm.compiler.nativeimage.VMRuntime.shutdown()}.
      */
     private static void shutdownLibGraal() {
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalRuntimeProvider.java

@@ -107,4 +107,9 @@ public interface HotSpotGraalRuntimeProvider extends GraalRuntime, RuntimeProvid
      * which configuration is in use.
      */
     String getCompilerConfigurationName();
+
+    /**
+     * Returns the instance holding the instrumentation data structures.
+     */
+    Instrumentation getInstrumentation();
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotHostBackend.java

@@ -101,7 +101,7 @@ public abstract class HotSpotHostBackend extends HotSpotBackend implements LIRGe
     @SuppressWarnings("try")
     public void completeInitialization(HotSpotJVMCIRuntime jvmciRuntime, OptionValues options) {
         final HotSpotProviders providers = getProviders();
-        HotSpotHostForeignCallsProvider foreignCalls = (HotSpotHostForeignCallsProvider) providers.getForeignCalls();
+        HotSpotHostForeignCallsProvider foreignCalls = providers.getForeignCalls();
         final HotSpotLoweringProvider lowerer = (HotSpotLoweringProvider) providers.getLowerer();
 
         try (InitTimer st = timer("foreignCalls.initialize")) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotMarkId.java

@@ -42,6 +42,7 @@ public enum HotSpotMarkId implements CompilationResult.MarkId {
     OSR_ENTRY(false),
     EXCEPTION_HANDLER_ENTRY(false),
     DEOPT_HANDLER_ENTRY(false),
+    DEOPT_MH_HANDLER_ENTRY(false),
     FRAME_COMPLETE(true, true),
     INVOKEINTERFACE(false),
     INVOKEVIRTUAL(false),

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotReplacementsImpl.java

@@ -37,6 +37,7 @@ import org.graalvm.compiler.core.common.CompilationIdentifier;
 import org.graalvm.compiler.debug.DebugContext;
 import org.graalvm.compiler.debug.GraalError;
 import org.graalvm.compiler.graph.NodeSourcePosition;
+import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
 import org.graalvm.compiler.hotspot.meta.HotSpotWordOperationPlugin;
 import org.graalvm.compiler.hotspot.word.HotSpotOperation;
 import org.graalvm.compiler.nodes.Cancellable;
@@ -50,7 +51,6 @@ import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin;
 import org.graalvm.compiler.nodes.graphbuilderconf.MethodSubstitutionPlugin;
 import org.graalvm.compiler.nodes.spi.SnippetParameterInfo;
 import org.graalvm.compiler.options.OptionValues;
-import org.graalvm.compiler.phases.util.Providers;
 import org.graalvm.compiler.printer.GraalDebugHandlersFactory;
 import org.graalvm.compiler.replacements.ReplacementsImpl;
 
@@ -63,15 +63,20 @@ import jdk.vm.ci.meta.ResolvedJavaMethod;
  * them.
  */
 public class HotSpotReplacementsImpl extends ReplacementsImpl {
-    public HotSpotReplacementsImpl(Providers providers, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider, TargetDescription target) {
+    public HotSpotReplacementsImpl(HotSpotProviders providers, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider, TargetDescription target) {
         super(new GraalDebugHandlersFactory(snippetReflection), providers, snippetReflection, bytecodeProvider, target);
     }
 
-    HotSpotReplacementsImpl(HotSpotReplacementsImpl replacements, Providers providers) {
+    HotSpotReplacementsImpl(HotSpotReplacementsImpl replacements, HotSpotProviders providers) {
         super(new GraalDebugHandlersFactory(replacements.snippetReflection), providers, replacements.snippetReflection,
                         replacements.getDefaultReplacementBytecodeProvider(), replacements.target);
     }
 
+    @Override
+    public HotSpotProviders getProviders() {
+        return (HotSpotProviders) super.getProviders();
+    }
+
     public void maybeInitializeEncoder(OptionValues options) {
         if (IS_IN_NATIVE_IMAGE) {
             return;
@@ -85,14 +90,6 @@ public class HotSpotReplacementsImpl extends ReplacementsImpl {
         }
     }
 
-    /**
-     * Returns true if this Replacements is being used for preparation of snippets and substitutions
-     * for libgraal.
-     */
-    public boolean isEncodingSnippets() {
-        return false;
-    }
-
     @Override
     public Class<? extends GraphBuilderPlugin> getIntrinsifyingPlugin(ResolvedJavaMethod method) {
         if (!IS_IN_NATIVE_IMAGE) {
@@ -189,6 +186,7 @@ public class HotSpotReplacementsImpl extends ReplacementsImpl {
 
     @Override
     public void registerSnippet(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean trackNodeSourcePosition, OptionValues options) {
+        assert method.isStatic() || receiver != null : "must have a constant type for the receiver";
         if (!IS_IN_NATIVE_IMAGE) {
             assert !snippetRegistrationClosed : "Cannot register snippet after registration is closed: " + method.format("%H.%n(%p)");
             assert registeredSnippets.add(method) : "Cannot register snippet twice: " + method.format("%H.%n(%p)");
@@ -303,4 +301,12 @@ public class HotSpotReplacementsImpl extends ReplacementsImpl {
         return null;
     }
 
+    @SuppressWarnings("unchecked")
+    @Override
+    public <T> T getInjectedArgument(Class<T> capability) {
+        if (capability.equals(GraalHotSpotVMConfig.class)) {
+            return (T) getProviders().getConfig();
+        }
+        return super.getInjectedArgument(capability);
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/Instrumentation.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package org.graalvm.compiler.hotspot;
+
+/**
+ * Interface in charge of holding the instrumentation data structures.
+ */
+public interface Instrumentation {
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/JVMCIVersionCheck.java

@@ -43,7 +43,7 @@ import java.util.regex.Pattern;
  */
 public final class JVMCIVersionCheck {
 
-    private static final Version JVMCI_MIN_VERSION = new Version3(20, 1, 2);
+    private static final Version JVMCI_MIN_VERSION = new Version3(20, 2, 1);
 
     public interface Version {
         boolean isLessThan(Version other);
@@ -180,6 +180,9 @@ public final class JVMCIVersionCheck {
         }
     }
 
+    public static final String JVMCI8_RELEASES_URL = "https://github.com/graalvm/graal-jvmci-8/releases";
+    public static final String JVMCI11_RELEASES_URL = "https://github.com/graalvm/labs-openjdk-11/releases";
+
     private void failVersionCheck(boolean exit, String reason, Object... args) {
         Formatter errorMessage = new Formatter().format(reason, args);
         String javaHome = props.get("java.home");
@@ -189,10 +192,10 @@ public final class JVMCIVersionCheck {
         errorMessage.format("Currently used Java home directory is %s.%n", javaHome);
         errorMessage.format("Currently used VM configuration is: %s%n", vmName);
         if (javaSpecVersion.compareTo("1.9") < 0) {
-            errorMessage.format("Download the latest JVMCI JDK 8 from https://github.com/graalvm/openjdk8-jvmci-builder/releases");
+            errorMessage.format("Download the latest JVMCI JDK 8 from " + JVMCI8_RELEASES_URL);
         } else {
             if (javaSpecVersion.compareTo("11") == 0 && vmVersion.contains("-jvmci-")) {
-                errorMessage.format("Download the latest Labs OpenJDK 11 from https://github.com/graalvm/labs-openjdk-11/releases");
+                errorMessage.format("Download the latest Labs OpenJDK 11 from " + JVMCI11_RELEASES_URL);
             } else {
                 errorMessage.format("Download JDK 11 or later.");
             }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/SymbolicSnippetEncoder.java

@@ -56,7 +56,6 @@ import org.graalvm.compiler.core.common.type.SymbolicJVMCIReference;
 import org.graalvm.compiler.debug.DebugContext;
 import org.graalvm.compiler.debug.GraalError;
 import org.graalvm.compiler.graph.Node;
-import org.graalvm.compiler.graph.NodeClass;
 import org.graalvm.compiler.graph.NodeMap;
 import org.graalvm.compiler.graph.NodeSourcePosition;
 import org.graalvm.compiler.hotspot.EncodedSnippets.GraalCapability;
@@ -67,6 +66,7 @@ import org.graalvm.compiler.hotspot.EncodedSnippets.SymbolicResolvedJavaMethodBy
 import org.graalvm.compiler.hotspot.EncodedSnippets.SymbolicStampPair;
 import org.graalvm.compiler.hotspot.meta.HotSpotForeignCallsProvider;
 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
+import org.graalvm.compiler.hotspot.word.HotSpotWordTypes;
 import org.graalvm.compiler.java.BytecodeParser;
 import org.graalvm.compiler.java.GraphBuilderPhase;
 import org.graalvm.compiler.nodeinfo.Verbosity;
@@ -90,17 +90,15 @@ import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin;
 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins;
 import org.graalvm.compiler.nodes.graphbuilderconf.MethodSubstitutionPlugin;
 import org.graalvm.compiler.nodes.graphbuilderconf.NodePlugin;
-import org.graalvm.compiler.nodes.graphbuilderconf.ParameterPlugin;
 import org.graalvm.compiler.nodes.java.AccessFieldNode;
 import org.graalvm.compiler.nodes.java.MethodCallTargetNode;
+import org.graalvm.compiler.nodes.spi.CoreProviders;
 import org.graalvm.compiler.nodes.spi.SnippetParameterInfo;
 import org.graalvm.compiler.nodes.virtual.VirtualObjectNode;
 import org.graalvm.compiler.options.OptionValues;
 import org.graalvm.compiler.phases.OptimisticOptimizations;
 import org.graalvm.compiler.phases.schedule.SchedulePhase;
 import org.graalvm.compiler.phases.util.Providers;
-import org.graalvm.compiler.replacements.ConstantBindingParameterPlugin;
-import org.graalvm.compiler.replacements.PEGraphDecoder;
 import org.graalvm.compiler.replacements.ReplacementsImpl;
 import org.graalvm.compiler.replacements.SnippetCounter;
 import org.graalvm.compiler.replacements.SnippetIntegerHistogram;
@@ -149,11 +147,109 @@ public class SymbolicSnippetEncoder {
      */
     private int encodedGraphs = 0;
 
+    abstract static class GraphKey {
+        final ResolvedJavaMethod method;
+        final ResolvedJavaMethod original;
+
+        GraphKey(ResolvedJavaMethod method, ResolvedJavaMethod original) {
+            this.method = method;
+            this.original = original;
+        }
+
+        public abstract String keyString();
+
+    }
+
+    static class SnippetKey extends GraphKey {
+
+        SnippetKey(ResolvedJavaMethod method, ResolvedJavaMethod original) {
+            super(method, original);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            SnippetKey that = (SnippetKey) o;
+            return Objects.equals(method, that.method) &&
+                            Objects.equals(original, that.original);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(method, original);
+        }
+
+        @Override
+        public String keyString() {
+            return methodKey(method);
+        }
+
+        @Override
+        public String toString() {
+            return "SnippetKey{" +
+                            "method=" + method +
+                            ", original=" + original +
+                            '}';
+        }
+    }
+
+    static class MethodSubstitutionKey extends GraphKey {
+        final IntrinsicContext.CompilationContext context;
+        final MethodSubstitutionPlugin plugin;
+
+        MethodSubstitutionKey(ResolvedJavaMethod method, ResolvedJavaMethod original, IntrinsicContext.CompilationContext context, MethodSubstitutionPlugin plugin) {
+            super(method, original);
+            this.context = context;
+            this.plugin = plugin;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            MethodSubstitutionKey that = (MethodSubstitutionKey) o;
+            return Objects.equals(original, that.original) &&
+                            context == that.context &&
+                            Objects.equals(plugin, that.plugin);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(original, context, plugin);
+        }
+
+        @Override
+        public String keyString() {
+            return plugin.toString() + context;
+        }
+
+        @Override
+        public String toString() {
+            return "MethodSubstitutionKey{" +
+                            "method=" + method +
+                            ", original=" + original +
+                            ", context=" + context +
+                            ", plugin=" + plugin +
+                            '}';
+        }
+    }
+
     /**
      * All the graphs parsed so far.
      */
     private EconomicMap<String, StructuredGraph> preparedSnippetGraphs = EconomicMap.create();
 
+    private EconomicMap<String, GraphKey> keyToMethod = EconomicMap.create();
+
     private EconomicMap<String, SnippetParameterInfo> snippetParameterInfos = EconomicMap.create();
 
     private EconomicSet<MethodSubstitutionPlugin> knownPlugins = EconomicSet.create();
@@ -172,9 +268,8 @@ public class SymbolicSnippetEncoder {
     }
 
     public void clearSnippetParameterNames() {
-        MapCursor<String, SnippetParameterInfo> cursor = snippetParameterInfos.getEntries();
+        for (SnippetParameterInfo info : snippetParameterInfos.getValues()) {
-        while (cursor.advance()) {
+            info.clearNames();
-            cursor.getValue().clearNames();
         }
     }
 
@@ -187,11 +282,6 @@ public class SymbolicSnippetEncoder {
                 return InlineInfo.DO_NOT_INLINE_NO_EXCEPTION;
             }
 
-            if (snippetReplacements.getIntrinsifyingPlugin(method) != null) {
-                delayedInvocationPluginMethods.add(method);
-                return InlineInfo.DO_NOT_INLINE_NO_EXCEPTION;
-            }
-
             // Force inlining when parsing replacements
             return createIntrinsicInlineInfo(method, snippetReplacements.getDefaultReplacementBytecodeProvider());
         }
@@ -202,21 +292,6 @@ public class SymbolicSnippetEncoder {
         }
     }
 
-    public static class SnippetInvocationPlugins extends InvocationPlugins {
-
-        SnippetInvocationPlugins(InvocationPlugins invocationPlugins) {
-            super(invocationPlugins);
-        }
-
-        @Override
-        public InvocationPlugin lookupInvocation(ResolvedJavaMethod method) {
-            if (method.getAnnotation(Fold.class) != null) {
-                return null;
-            }
-            return super.lookupInvocation(method);
-        }
-    }
-
     /**
      * This plugin disables the snippet counter machinery.
      */
@@ -246,12 +321,12 @@ public class SymbolicSnippetEncoder {
     SymbolicSnippetEncoder(HotSpotReplacementsImpl replacements) {
         this.originalReplacements = replacements;
         GraphBuilderConfiguration.Plugins plugins = replacements.getGraphBuilderPlugins();
-        SnippetInvocationPlugins invocationPlugins = new SnippetInvocationPlugins(plugins.getInvocationPlugins());
+        InvocationPlugins invocationPlugins = plugins.getInvocationPlugins();
         GraphBuilderConfiguration.Plugins copy = new GraphBuilderConfiguration.Plugins(plugins, invocationPlugins);
         copy.clearInlineInvokePlugins();
         copy.appendInlineInvokePlugin(new SnippetInlineInvokePlugin());
         copy.appendNodePlugin(new SnippetCounterPlugin());
-        HotSpotProviders providers = (HotSpotProviders) replacements.getProviders().copyWith(new HotSpotSubstrateConstantReflectionProvider(replacements.getProviders().getConstantReflection()));
+        HotSpotProviders providers = replacements.getProviders().copyWith(new HotSpotSubstrateConstantReflectionProvider(replacements.getProviders().getConstantReflection()));
         this.snippetReplacements = new HotSpotSnippetReplacementsImpl(replacements, providers.copyWith(copy));
         this.snippetReplacements.setGraphBuilderPlugins(copy);
     }
@@ -278,8 +353,10 @@ public class SymbolicSnippetEncoder {
         assert method.getAnnotation(MethodSubstitution.class) != null : "MethodSubstitution must be annotated with @" + MethodSubstitution.class.getSimpleName();
         String originalMethodString = plugin.originalMethodAsString();
         StructuredGraph subst = buildGraph(method, original, originalMethodString, null, true, false, context, options);
-        originalMethods.put(methodKey(method), originalMethodString);
+        MethodSubstitutionKey key = new MethodSubstitutionKey(method, original, context, plugin);
-        preparedSnippetGraphs.put(plugin.toString() + context, subst);
+        originalMethods.put(key.keyString(), originalMethodString);
+        preparedSnippetGraphs.put(key.keyString(), subst);
+        keyToMethod.put(key.keyString(), key);
     }
 
     private StructuredGraph buildGraph(ResolvedJavaMethod method, ResolvedJavaMethod original, String originalMethodString, Object receiver, boolean requireInlining, boolean trackNodeSourcePosition,
@@ -300,7 +377,7 @@ public class SymbolicSnippetEncoder {
         if (context == IntrinsicContext.CompilationContext.ROOT_COMPILATION) {
             contextToUse = IntrinsicContext.CompilationContext.ROOT_COMPILATION_ENCODING;
         }
-        try (DebugContext debug = openDebugContext("SymbolicSnippetEncoder_", method, options)) {
+        try (DebugContext debug = snippetReplacements.openSnippetDebugContext("SymbolicSnippetEncoder_", method, options)) {
             StructuredGraph graph = snippetReplacements.makeGraph(debug, snippetReplacements.getDefaultReplacementBytecodeProvider(), method, args, original, trackNodeSourcePosition, null,
                             contextToUse);
 
@@ -311,51 +388,19 @@ public class SymbolicSnippetEncoder {
                     throw GraalError.shouldNotReachHere("method " + callee.format("%H.%n") + " not inlined in snippet " + method.getName() + " (maybe not final?)");
                 }
             }
-            assert verifySnippetEncodeDecode(debug, method, original, originalMethodString, trackNodeSourcePosition, graph);
+            assert verifySnippetEncodeDecode(debug, method, original, originalMethodString, args, trackNodeSourcePosition, graph);
             debug.dump(DebugContext.VERBOSE_LEVEL, graph, "After buildGraph");
             return graph;
         }
     }
 
     @SuppressWarnings("try")
-    private static StructuredGraph decodeSnippetGraph(SymbolicEncodedGraph encodedGraph, ResolvedJavaMethod method, ReplacementsImpl replacements, Object[] args,
+    private boolean verifySnippetEncodeDecode(DebugContext debug, ResolvedJavaMethod method, ResolvedJavaMethod original, String originalMethodString, Object[] args, boolean trackNodeSourcePosition,
-                    StructuredGraph.AllowAssumptions allowAssumptions, OptionValues options) {
-        Providers providers = replacements.getProviders();
-        ParameterPlugin parameterPlugin = null;
-        if (args != null) {
-            parameterPlugin = new ConstantBindingParameterPlugin(args, providers.getMetaAccess(), replacements.snippetReflection);
-        }
-
-        try (DebugContext debug = replacements.openDebugContext("SVMSnippet_", method, options)) {
-            // @formatter:off
-            boolean isSubstitution = true;
-            StructuredGraph result = new StructuredGraph.Builder(options, debug, allowAssumptions)
-                    .method(method)
-                    .trackNodeSourcePosition(encodedGraph.trackNodeSourcePosition())
-                    .setIsSubstitution(isSubstitution)
-                    .build();
-            // @formatter:on
-            try (DebugContext.Scope scope = debug.scope("DecodeSnippetGraph", result)) {
-                PEGraphDecoder graphDecoder = new EncodedSnippets.SubstitutionGraphDecoder(providers, result, replacements, parameterPlugin, method, INLINE_AFTER_PARSING, encodedGraph);
-
-                graphDecoder.decode(method, isSubstitution, encodedGraph.trackNodeSourcePosition());
-                debug.dump(DebugContext.VERBOSE_LEVEL, result, "After decoding");
-
-                assert result.verify();
-                return result;
-            } catch (Throwable t) {
-                throw debug.handle(t);
-            }
-        }
-    }
-
-    @SuppressWarnings("try")
-    private boolean verifySnippetEncodeDecode(DebugContext debug, ResolvedJavaMethod method, ResolvedJavaMethod original, String originalMethodString, boolean trackNodeSourcePosition,
                     StructuredGraph graph) {
         // Verify the encoding and decoding process
         EncodedGraph encodedGraph = GraphEncoder.encodeSingleGraph(graph, HotSpotJVMCIRuntime.runtime().getHostJVMCIBackend().getTarget().arch);
 
-        HotSpotProviders originalProvider = (HotSpotProviders) snippetReplacements.getProviders();
+        HotSpotProviders originalProvider = snippetReplacements.getProviders();
 
         SnippetReflectionProvider snippetReflection = originalProvider.getSnippetReflection();
         SymbolicSnippetEncoder.HotSpotSubstrateConstantReflectionProvider constantReflection = new SymbolicSnippetEncoder.HotSpotSubstrateConstantReflectionProvider(
@@ -363,7 +408,7 @@ public class SymbolicSnippetEncoder {
         HotSpotProviders newProviders = new HotSpotProviders(originalProvider.getMetaAccess(), originalProvider.getCodeCache(), constantReflection,
                         originalProvider.getConstantFieldProvider(), originalProvider.getForeignCalls(), originalProvider.getLowerer(), null, originalProvider.getSuites(),
                         originalProvider.getRegisters(), snippetReflection, originalProvider.getWordTypes(), originalProvider.getGraphBuilderPlugins(),
-                        originalProvider.getPlatformConfigurationProvider(), originalProvider.getMetaAccessExtensionProvider());
+                        originalProvider.getPlatformConfigurationProvider(), originalProvider.getMetaAccessExtensionProvider(), originalProvider.getConfig());
         HotSpotSnippetReplacementsImpl filteringReplacements = new HotSpotSnippetReplacementsImpl(newProviders, snippetReflection,
                         originalProvider.getReplacements().getDefaultReplacementBytecodeProvider(), originalProvider.getCodeCache().getTarget());
         filteringReplacements.setGraphBuilderPlugins(originalProvider.getReplacements().getGraphBuilderPlugins());
@@ -371,11 +416,11 @@ public class SymbolicSnippetEncoder {
             for (int i = 0; i < encodedGraph.getNumObjects(); i++) {
                 filterSnippetObject(encodedGraph.getObject(i));
             }
-            StructuredGraph snippet = filteringReplacements.makeGraph(debug, filteringReplacements.getDefaultReplacementBytecodeProvider(), method, null, original,
+            StructuredGraph snippet = filteringReplacements.makeGraph(debug, filteringReplacements.getDefaultReplacementBytecodeProvider(), method, args, original,
                             trackNodeSourcePosition, null);
             SymbolicEncodedGraph symbolicGraph = new SymbolicEncodedGraph(encodedGraph, method.getDeclaringClass(), originalMethodString);
-            StructuredGraph decodedSnippet = decodeSnippetGraph(symbolicGraph, original != null ? original : method, originalReplacements, null,
+            StructuredGraph decodedSnippet = EncodedSnippets.decodeSnippetGraph(symbolicGraph, original != null ? original : method, originalReplacements, null,
-                            StructuredGraph.AllowAssumptions.ifNonNull(graph.getAssumptions()), graph.getOptions());
+                            StructuredGraph.AllowAssumptions.ifNonNull(graph.getAssumptions()), graph.getOptions(), false);
             String snippetString = getCanonicalGraphString(snippet, true, false);
             String decodedSnippetString = getCanonicalGraphString(decodedSnippet, true, false);
             if (snippetString.equals(decodedSnippetString)) {
@@ -430,14 +475,16 @@ public class SymbolicSnippetEncoder {
     synchronized void registerSnippet(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean trackNodeSourcePosition, OptionValues options) {
         if (IS_BUILDING_NATIVE_IMAGE || UseEncodedGraphs.getValue(options)) {
             assert method.getAnnotation(Snippet.class) != null : "Snippet must be annotated with @" + Snippet.class.getSimpleName();
-            String key = methodKey(method);
+            SnippetKey key = new SnippetKey(method, original);
-            if (!preparedSnippetGraphs.containsKey(key)) {
+            String keyString = key.keyString();
+            if (!preparedSnippetGraphs.containsKey(keyString)) {
                 if (original != null) {
-                    originalMethods.put(key, methodKey(original));
+                    originalMethods.put(keyString, methodKey(original));
                 }
                 StructuredGraph snippet = buildGraph(method, original, null, receiver, true, trackNodeSourcePosition, INLINE_AFTER_PARSING, options);
-                preparedSnippetGraphs.put(key, snippet);
+                preparedSnippetGraphs.put(keyString, snippet);
-                snippetParameterInfos.put(key, new SnippetParameterInfo(method));
+                snippetParameterInfos.put(keyString, new SnippetParameterInfo(method));
+                keyToMethod.put(keyString, key);
             }
         }
 
@@ -450,26 +497,22 @@ public class SymbolicSnippetEncoder {
         }
         encoder.finishPrepare();
 
-        byte[] snippetEncoding;
+        EconomicMap<String, EncodedSnippets.GraphData> graphDatas = EconomicMap.create();
-        Object[] snippetObjects;
-        NodeClass<?>[] snippetNodeClasses;
-        EconomicMap<String, Integer> snippetStartOffsets;
-
-        snippetStartOffsets = EconomicMap.create();
         MapCursor<String, StructuredGraph> cursor = preparedSnippetGraphs.getEntries();
         while (cursor.advance()) {
-            snippetStartOffsets.put(cursor.getKey(), encoder.encode(cursor.getValue()));
+            EncodedSnippets.GraphData data = new EncodedSnippets.GraphData(encoder.encode(cursor.getValue()), originalMethods.get(cursor.getKey()), snippetParameterInfos.get(cursor.getKey()));
+            graphDatas.put(cursor.getKey(), data);
         }
-        snippetEncoding = encoder.getEncoding();
+
-        snippetObjects = encoder.getObjects();
+        byte[] snippetEncoding = encoder.getEncoding();
-        snippetNodeClasses = encoder.getNodeClasses();
+        Object[] snippetObjects = encoder.getObjects();
         for (int i = 0; i < snippetObjects.length; i++) {
             Object o = filterSnippetObject(snippetObjects[i]);
             debug.log("snippetObjects[%d] = %s -> %s", i, o != null ? o.getClass().getSimpleName() : null, o);
             snippetObjects[i] = o;
         }
-        debug.log("Encoded %d snippet preparedSnippetGraphs using %d bytes with %d objects", snippetStartOffsets.size(), snippetEncoding.length, snippetObjects.length);
+        debug.log("Encoded %d snippet preparedSnippetGraphs using %d bytes with %d objects", graphDatas.size(), snippetEncoding.length, snippetObjects.length);
-        return new EncodedSnippets(snippetEncoding, snippetObjects, snippetNodeClasses, snippetStartOffsets, originalMethods, snippetParameterInfos);
+        return new EncodedSnippets(snippetEncoding, snippetObjects, encoder.getNodeClasses(), graphDatas);
     }
 
     /**
@@ -594,7 +637,8 @@ public class SymbolicSnippetEncoder {
             // Filter these out for now. These can't easily be handled because these positions
             // description snippet methods which might not be available in the runtime.
             return null;
-        } else if (o instanceof HotSpotForeignCallsProvider || o instanceof GraalHotSpotVMConfig) {
+        } else if (o instanceof HotSpotForeignCallsProvider || o instanceof GraalHotSpotVMConfig || o instanceof HotSpotWordTypes || o instanceof TargetDescription ||
+                        o instanceof SnippetReflectionProvider) {
             return new GraalCapability(o.getClass());
         } else if (o instanceof Stamp) {
             SymbolicJVMCIReference<?> ref = ((Stamp) o).makeSymbolic();
@@ -731,11 +775,11 @@ public class SymbolicSnippetEncoder {
      * as the parser for these snippets.
      */
     class HotSpotSnippetReplacementsImpl extends HotSpotReplacementsImpl {
-        HotSpotSnippetReplacementsImpl(HotSpotReplacementsImpl replacements, Providers providers) {
+        HotSpotSnippetReplacementsImpl(HotSpotReplacementsImpl replacements, HotSpotProviders providers) {
             super(replacements, providers);
         }
 
-        HotSpotSnippetReplacementsImpl(Providers providers, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider, TargetDescription target) {
+        HotSpotSnippetReplacementsImpl(HotSpotProviders providers, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider, TargetDescription target) {
             super(providers, snippetReflection, bytecodeProvider, target);
         }
 
@@ -764,7 +808,7 @@ public class SymbolicSnippetEncoder {
     }
 
     class HotSpotSnippetGraphBuilderPhase extends GraphBuilderPhase.Instance {
-        HotSpotSnippetGraphBuilderPhase(Providers theProviders, GraphBuilderConfiguration graphBuilderConfig, OptimisticOptimizations optimisticOpts, IntrinsicContext initialIntrinsicContext) {
+        HotSpotSnippetGraphBuilderPhase(CoreProviders theProviders, GraphBuilderConfiguration graphBuilderConfig, OptimisticOptimizations optimisticOpts, IntrinsicContext initialIntrinsicContext) {
             super(theProviders, graphBuilderConfig, optimisticOpts, initialIntrinsicContext);
         }
 
@@ -787,6 +831,11 @@ public class SymbolicSnippetEncoder {
             return plugin.isGeneratedFromFoldOrNodeIntrinsic();
         }
 
+        @Override
+        public boolean shouldDeferPlugin(GeneratedInvocationPlugin plugin) {
+            return plugin.isGeneratedFromFoldOrNodeIntrinsic();
+        }
+
         @Override
         protected boolean canInlinePartialIntrinsicExit() {
             return false;
@@ -797,10 +846,7 @@ public class SymbolicSnippetEncoder {
             if (intrinsicContext != null && intrinsicContext.isCallToOriginal(targetMethod)) {
                 return false;
             }
-            if (targetMethod.getAnnotation(Fold.class) != null) {
+
-                // Always defer Fold until decode time but NodeIntrinsics may fold if they are able.
-                return false;
-            }
             InvocationPlugin plugin = graphBuilderConfig.getPlugins().getInvocationPlugins().lookupInvocation(targetMethod);
             if (plugin != null && conditionalPlugins.contains(plugin)) {
                 // Because supporting arbitrary plugins in the context of encoded graphs is complex

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java

@@ -26,6 +26,7 @@ package org.graalvm.compiler.hotspot.meta;
 
 import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE;
 import static org.graalvm.compiler.core.common.GraalOptions.AlwaysInlineVTableStubs;
+import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
 import static org.graalvm.compiler.core.common.GraalOptions.InlineVTableStubs;
 import static org.graalvm.compiler.core.common.GraalOptions.OmitHotExceptionStacktrace;
 import static org.graalvm.compiler.hotspot.meta.HotSpotForeignCallsProviderImpl.OSR_MIGRATION_END;
@@ -229,7 +230,7 @@ public abstract class DefaultHotSpotLoweringProvider extends DefaultJavaLowering
         allocationSnippets = new HotSpotAllocationSnippets.Templates(options, factories, runtime, providers, target, config);
         monitorSnippets = new MonitorSnippets.Templates(options, factories, runtime, providers, target, config.useFastLocking);
         g1WriteBarrierSnippets = new HotSpotG1WriteBarrierSnippets.Templates(options, factories, runtime, providers, target, config);
-        serialWriteBarrierSnippets = new HotSpotSerialWriteBarrierSnippets.Templates(options, factories, runtime, providers, target, config);
+        serialWriteBarrierSnippets = new HotSpotSerialWriteBarrierSnippets.Templates(options, factories, runtime, providers, target);
         exceptionObjectSnippets = new LoadExceptionObjectSnippets.Templates(options, factories, providers, target);
         assertionSnippets = new AssertionSnippets.Templates(options, factories, providers, target);
         arraycopySnippets = new ArrayCopySnippets.Templates(new HotSpotArraycopySnippets(), options, factories, runtime, providers, providers.getSnippetReflection(), target);
@@ -238,12 +239,12 @@ public abstract class DefaultHotSpotLoweringProvider extends DefaultJavaLowering
         resolveConstantSnippets = new ResolveConstantSnippets.Templates(options, factories, providers, target);
         objectCloneSnippets = new ObjectCloneSnippets.Templates(options, factories, providers, target);
         foreignCallSnippets = new ForeignCallSnippets.Templates(options, factories, providers, target);
+        if (JavaVersionUtil.JAVA_SPEC >= 11) {
             objectSnippets = new ObjectSnippets.Templates(options, factories, providers, target);
+        }
         unsafeSnippets = new UnsafeSnippets.Templates(options, factories, providers, target);
-        if (JavaVersionUtil.JAVA_SPEC <= 8) {
+        if (JavaVersionUtil.JAVA_SPEC >= 11 && GeneratePIC.getValue(options)) {
             // AOT only introduced in JDK 9
-            profileSnippets = null;
-        } else {
             profileSnippets = new ProfileSnippets.Templates(options, factories, providers, target);
         }
     }
@@ -419,6 +420,9 @@ public abstract class DefaultHotSpotLoweringProvider extends DefaultJavaLowering
             } else if (n instanceof KlassBeingInitializedCheckNode) {
                 allocationSnippets.lower((KlassBeingInitializedCheckNode) n, tool);
             } else if (n instanceof FastNotifyNode) {
+                if (JavaVersionUtil.JAVA_SPEC < 11) {
+                    throw GraalError.shouldNotReachHere("FastNotify is not support prior to 11");
+                }
                 if (graph.getGuardsStage() == GuardsStage.AFTER_FSA) {
                     objectSnippets.lower(n, tool);
                 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotForeignCallsProviderImpl.java

@@ -179,7 +179,9 @@ public abstract class HotSpotForeignCallsProviderImpl implements HotSpotForeignC
                     HotSpotForeignCallDescriptor descriptor,
                     long address,
                     boolean prependThread) {
-        if (address != 0) {
+        if (address == 0) {
+            throw new IllegalArgumentException("Can't link foreign call with zero address");
+        }
         ForeignCallStub stub = new ForeignCallStub(options, jvmciRuntime, providers, address, descriptor, prependThread);
         HotSpotForeignCallLinkage linkage = stub.getLinkage();
         HotSpotForeignCallLinkage targetLinkage = stub.getTargetLinkage();
@@ -187,7 +189,6 @@ public abstract class HotSpotForeignCallsProviderImpl implements HotSpotForeignC
         register(linkage);
         register(targetLinkage);
     }
-    }
 
     public static final boolean PREPEND_THREAD = true;
     public static final boolean DONT_PREPEND_THREAD = !PREPEND_THREAD;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java

@@ -84,13 +84,13 @@ import org.graalvm.compiler.nodes.calc.IntegerConvertNode;
 import org.graalvm.compiler.nodes.calc.LeftShiftNode;
 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
 import org.graalvm.compiler.nodes.graphbuilderconf.ForeignCallPlugin;
+import org.graalvm.compiler.nodes.graphbuilderconf.GeneratedPluginFactory;
 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.Plugins;
 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext;
 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin;
 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin.Receiver;
 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins;
 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins.Registration;
-import org.graalvm.compiler.nodes.graphbuilderconf.NodeIntrinsicPluginFactory;
 import org.graalvm.compiler.nodes.java.DynamicNewInstanceNode;
 import org.graalvm.compiler.nodes.memory.OnHeapMemoryAccess.BarrierType;
 import org.graalvm.compiler.nodes.memory.ReadNode;
@@ -152,8 +152,8 @@ public class HotSpotGraphBuilderPlugins {
         InvocationPlugins invocationPlugins = new HotSpotInvocationPlugins(graalRuntime, config, compilerConfiguration);
 
         Plugins plugins = new Plugins(invocationPlugins);
-        NodeIntrinsificationProvider nodeIntrinsificationProvider = new NodeIntrinsificationProvider(metaAccess, snippetReflection, foreignCalls, wordTypes, target);
         if (!IS_IN_NATIVE_IMAGE) {
+            // In libgraal all word related operations have been fully processed so this is unneeded
             HotSpotWordOperationPlugin wordOperationPlugin = new HotSpotWordOperationPlugin(snippetReflection, wordTypes);
             HotSpotNodePlugin nodePlugin = new HotSpotNodePlugin(wordOperationPlugin, config, wordTypes);
 
@@ -161,7 +161,7 @@ public class HotSpotGraphBuilderPlugins {
             plugins.appendNodePlugin(nodePlugin);
         }
         if (!GeneratePIC.getValue(options)) {
-            plugins.appendNodePlugin(new MethodHandlePlugin(constantReflection.getMethodHandleAccess(), true));
+            plugins.appendNodePlugin(new MethodHandlePlugin(constantReflection.getMethodHandleAccess(), !config.supportsMethodHandleDeoptimizationEntry()));
         }
         plugins.appendInlineInvokePlugin(replacements);
         if (InlineDuringParsing.getValue(options)) {
@@ -205,12 +205,24 @@ public class HotSpotGraphBuilderPlugins {
                 registerArrayPlugins(invocationPlugins, replacements);
                 registerStringPlugins(invocationPlugins, replacements);
                 registerArraysSupportPlugins(invocationPlugins, config, replacements);
-
-                for (NodeIntrinsicPluginFactory factory : GraalServices.load(NodeIntrinsicPluginFactory.class)) {
-                    factory.registerPlugins(invocationPlugins, nodeIntrinsificationProvider);
-                }
             }
         });
+        if (!IS_IN_NATIVE_IMAGE) {
+            // In libgraal all NodeIntrinsics been converted into special nodes so the plugins
+            // aren't needed.
+            NodeIntrinsificationProvider nodeIntrinsificationProvider = new NodeIntrinsificationProvider(metaAccess, snippetReflection, foreignCalls, wordTypes, target);
+            invocationPlugins.defer(new Runnable() {
+
+                @Override
+                public void run() {
+
+                    for (GeneratedPluginFactory factory : GraalServices.load(GeneratedPluginFactory.class)) {
+                        factory.registerPlugins(invocationPlugins, nodeIntrinsificationProvider);
+                    }
+
+                }
+            });
+        }
         return plugins;
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java

@@ -128,6 +128,7 @@ import org.graalvm.compiler.nodes.NamedLocationIdentity;
 import org.graalvm.compiler.nodes.extended.BytecodeExceptionNode.BytecodeExceptionKind;
 import org.graalvm.compiler.options.OptionValues;
 import org.graalvm.compiler.replacements.arraycopy.ArrayCopyForeignCalls;
+import org.graalvm.compiler.serviceprovider.JavaVersionUtil;
 import org.graalvm.compiler.word.Word;
 import org.graalvm.compiler.word.WordTypes;
 import jdk.internal.vm.compiler.word.LocationIdentity;
@@ -361,8 +362,12 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall
         linkForeignCall(options, providers, createDescriptor(REGISTER_FINALIZER, SAFEPOINT, NOT_REEXECUTABLE, any()), c.registerFinalizerAddress, PREPEND_THREAD);
         linkForeignCall(options, providers, MONITORENTER, c.monitorenterAddress, PREPEND_THREAD);
         linkForeignCall(options, providers, MONITOREXIT, c.monitorexitAddress, PREPEND_THREAD);
+        if (JavaVersionUtil.JAVA_SPEC >= 11) {
             linkForeignCall(options, providers, NOTIFY, c.notifyAddress, PREPEND_THREAD);
             linkForeignCall(options, providers, NOTIFY_ALL, c.notifyAllAddress, PREPEND_THREAD);
+        } else {
+            assert c.notifyAddress == 0 : "unexpected value";
+        }
         linkForeignCall(options, providers, LOG_PRINTF, c.logPrintfAddress, PREPEND_THREAD);
         linkForeignCall(options, providers, LOG_OBJECT, c.logObjectAddress, PREPEND_THREAD);
         linkForeignCall(options, providers, LOG_PRIMITIVE, c.logPrimitiveAddress, PREPEND_THREAD);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotProviders.java

@@ -26,14 +26,19 @@ package org.graalvm.compiler.hotspot.meta;
 
 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
 import org.graalvm.compiler.core.common.spi.ConstantFieldProvider;
+import org.graalvm.compiler.core.common.spi.ForeignCallsProvider;
 import org.graalvm.compiler.core.common.spi.MetaAccessExtensionProvider;
+import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
 import org.graalvm.compiler.hotspot.word.HotSpotWordTypes;
 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.Plugins;
 import org.graalvm.compiler.nodes.spi.LoweringProvider;
+import org.graalvm.compiler.nodes.spi.PlatformConfigurationProvider;
 import org.graalvm.compiler.nodes.spi.Replacements;
+import org.graalvm.compiler.nodes.spi.StampProvider;
 import org.graalvm.compiler.phases.tiers.SuitesProvider;
 import org.graalvm.compiler.phases.util.Providers;
 
+import jdk.vm.ci.code.CodeCacheProvider;
 import jdk.vm.ci.hotspot.HotSpotCodeCacheProvider;
 import jdk.vm.ci.meta.ConstantReflectionProvider;
 import jdk.vm.ci.meta.MetaAccessProvider;
@@ -46,20 +51,40 @@ public class HotSpotProviders extends Providers {
 
     private final SuitesProvider suites;
     private final HotSpotRegistersProvider registers;
-    private final SnippetReflectionProvider snippetReflection;
-    private final HotSpotWordTypes wordTypes;
     private final Plugins graphBuilderPlugins;
+    private final GraalHotSpotVMConfig config;
 
-    public HotSpotProviders(MetaAccessProvider metaAccess, HotSpotCodeCacheProvider codeCache, ConstantReflectionProvider constantReflection, ConstantFieldProvider constantField,
+    public HotSpotProviders(MetaAccessProvider metaAccess,
-                    HotSpotForeignCallsProvider foreignCalls, LoweringProvider lowerer, Replacements replacements, SuitesProvider suites, HotSpotRegistersProvider registers,
+                    HotSpotCodeCacheProvider codeCache,
-                    SnippetReflectionProvider snippetReflection, HotSpotWordTypes wordTypes, Plugins graphBuilderPlugins, HotSpotPlatformConfigurationProvider platformConfigurationProvider,
+                    ConstantReflectionProvider constantReflection,
-                    MetaAccessExtensionProvider metaAccessExtensionProvider) {
+                    ConstantFieldProvider constantField,
-        super(metaAccess, codeCache, constantReflection, constantField, foreignCalls, lowerer, replacements, new HotSpotStampProvider(), platformConfigurationProvider, metaAccessExtensionProvider);
+                    HotSpotHostForeignCallsProvider foreignCalls,
+                    LoweringProvider lowerer,
+                    Replacements replacements,
+                    SuitesProvider suites,
+                    HotSpotRegistersProvider registers,
+                    SnippetReflectionProvider snippetReflection,
+                    HotSpotWordTypes wordTypes,
+                    Plugins graphBuilderPlugins,
+                    PlatformConfigurationProvider platformConfigurationProvider,
+                    MetaAccessExtensionProvider metaAccessExtensionProvider,
+                    GraalHotSpotVMConfig config) {
+        super(metaAccess, codeCache, constantReflection, constantField, foreignCalls, lowerer, replacements, new HotSpotStampProvider(), platformConfigurationProvider, metaAccessExtensionProvider,
+                        snippetReflection, wordTypes);
         this.suites = suites;
         this.registers = registers;
-        this.snippetReflection = snippetReflection;
-        this.wordTypes = wordTypes;
         this.graphBuilderPlugins = graphBuilderPlugins;
+        this.config = config;
+    }
+
+    public HotSpotProviders(MetaAccessProvider metaAccess, CodeCacheProvider codeCache, ConstantReflectionProvider constantReflection, ConstantFieldProvider constantField,
+                    ForeignCallsProvider foreignCalls, LoweringProvider lowerer, Replacements replacements, StampProvider stampProvider, PlatformConfigurationProvider platformConfigurationProvider,
+                    MetaAccessExtensionProvider metaAccessExtensionProvider) {
+        super(metaAccess, codeCache, constantReflection, constantField, foreignCalls, lowerer, replacements, stampProvider, platformConfigurationProvider, metaAccessExtensionProvider, null, null);
+        this.suites = null;
+        this.registers = null;
+        this.graphBuilderPlugins = null;
+        this.config = null;
     }
 
     @Override
@@ -68,8 +93,8 @@ public class HotSpotProviders extends Providers {
     }
 
     @Override
-    public HotSpotForeignCallsProvider getForeignCalls() {
+    public HotSpotHostForeignCallsProvider getForeignCalls() {
-        return (HotSpotForeignCallsProvider) super.getForeignCalls();
+        return (HotSpotHostForeignCallsProvider) super.getForeignCalls();
     }
 
     public SuitesProvider getSuites() {
@@ -80,47 +105,46 @@ public class HotSpotProviders extends Providers {
         return registers;
     }
 
-    public SnippetReflectionProvider getSnippetReflection() {
-        return snippetReflection;
-    }
-
     public Plugins getGraphBuilderPlugins() {
         return graphBuilderPlugins;
     }
 
+    @Override
     public HotSpotWordTypes getWordTypes() {
-        return wordTypes;
+        return (HotSpotWordTypes) super.getWordTypes();
+    }
+
+    public GraalHotSpotVMConfig getConfig() {
+        return config;
     }
 
     @Override
     public HotSpotPlatformConfigurationProvider getPlatformConfigurationProvider() {
-        return (HotSpotPlatformConfigurationProvider) super.getPlatformConfigurationProvider();
+        return (HotSpotPlatformConfigurationProvider) platformConfigurationProvider;
     }
 
     @Override
-    public Providers copyWith(ConstantReflectionProvider substitution) {
+    public HotSpotProviders copyWith(ConstantReflectionProvider substitution) {
-        assert this.getClass() == HotSpotProviders.class : "must override in " + getClass();
         return new HotSpotProviders(getMetaAccess(), getCodeCache(), substitution, getConstantFieldProvider(), getForeignCalls(), getLowerer(), getReplacements(), getSuites(),
-                        getRegisters(), getSnippetReflection(), getWordTypes(), getGraphBuilderPlugins(), getPlatformConfigurationProvider(), getMetaAccessExtensionProvider());
+                        getRegisters(), getSnippetReflection(), getWordTypes(), getGraphBuilderPlugins(), getPlatformConfigurationProvider(), getMetaAccessExtensionProvider(), config);
     }
 
     @Override
-    public Providers copyWith(ConstantFieldProvider substitution) {
+    public HotSpotProviders copyWith(ConstantFieldProvider substitution) {
-        assert this.getClass() == HotSpotProviders.class : "must override in " + getClass();
+        return new HotSpotProviders(getMetaAccess(), getCodeCache(), getConstantReflection(), substitution, getForeignCalls(), getLowerer(), getReplacements(),
-        return new HotSpotProviders(getMetaAccess(), getCodeCache(), getConstantReflection(), substitution, getForeignCalls(), getLowerer(), getReplacements(), getSuites(),
+                        getSuites(),
-                        getRegisters(), getSnippetReflection(), getWordTypes(), getGraphBuilderPlugins(), getPlatformConfigurationProvider(), getMetaAccessExtensionProvider());
+                        getRegisters(), getSnippetReflection(), getWordTypes(), getGraphBuilderPlugins(), getPlatformConfigurationProvider(), getMetaAccessExtensionProvider(), config);
     }
 
     @Override
-    public Providers copyWith(Replacements substitution) {
+    public HotSpotProviders copyWith(Replacements substitution) {
-        assert this.getClass() == HotSpotProviders.class : "must override in " + getClass();
+        return new HotSpotProviders(getMetaAccess(), getCodeCache(), getConstantReflection(), getConstantFieldProvider(), getForeignCalls(), getLowerer(), substitution,
-        return new HotSpotProviders(getMetaAccess(), getCodeCache(), getConstantReflection(), getConstantFieldProvider(), getForeignCalls(), getLowerer(), substitution, getSuites(),
+                        getSuites(), getRegisters(), getSnippetReflection(), getWordTypes(), getGraphBuilderPlugins(), getPlatformConfigurationProvider(), getMetaAccessExtensionProvider(), config);
-                        getRegisters(), getSnippetReflection(), getWordTypes(), getGraphBuilderPlugins(), getPlatformConfigurationProvider(), getMetaAccessExtensionProvider());
     }
 
-    public Providers copyWith(Plugins substitution) {
+    public HotSpotProviders copyWith(Plugins substitution) {
-        assert this.getClass() == HotSpotProviders.class : "must override in " + getClass();
+        return new HotSpotProviders(getMetaAccess(), getCodeCache(), getConstantReflection(), getConstantFieldProvider(), getForeignCalls(), getLowerer(), getReplacements(),
-        return new HotSpotProviders(getMetaAccess(), getCodeCache(), getConstantReflection(), getConstantFieldProvider(), getForeignCalls(), getLowerer(), getReplacements(), getSuites(),
+                        getSuites(), getRegisters(), getSnippetReflection(), getWordTypes(), substitution, getPlatformConfigurationProvider(), getMetaAccessExtensionProvider(), config);
-                        getRegisters(), getSnippetReflection(), getWordTypes(), substitution, getPlatformConfigurationProvider(), getMetaAccessExtensionProvider());
     }
+
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/GraalHotSpotVMConfigNode.java

@@ -32,13 +32,13 @@ import org.graalvm.compiler.core.common.type.Stamp;
 import org.graalvm.compiler.core.common.type.StampFactory;
 import org.graalvm.compiler.debug.GraalError;
 import org.graalvm.compiler.graph.Node;
+import org.graalvm.compiler.graph.Node.NodeIntrinsicFactory;
 import org.graalvm.compiler.graph.NodeClass;
 import org.graalvm.compiler.graph.spi.Canonicalizable;
 import org.graalvm.compiler.graph.spi.CanonicalizerTool;
 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
 import org.graalvm.compiler.hotspot.HotSpotLIRGenerator;
 import org.graalvm.compiler.hotspot.HotSpotMarkId;
-import org.graalvm.compiler.hotspot.HotSpotReplacementsImpl;
 import org.graalvm.compiler.nodeinfo.NodeInfo;
 import org.graalvm.compiler.nodes.ConstantNode;
 import org.graalvm.compiler.nodes.calc.FloatingNode;
@@ -47,13 +47,13 @@ import org.graalvm.compiler.nodes.spi.LIRLowerable;
 import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool;
 
 import jdk.vm.ci.meta.JavaKind;
-import jdk.vm.ci.meta.ResolvedJavaMethod;
 import jdk.vm.ci.meta.Value;
 
 /**
  * Represents {@link GraalHotSpotVMConfig} values that may change after compilation.
  */
 @NodeInfo(cycles = CYCLES_1, size = SIZE_1)
+@NodeIntrinsicFactory
 public class GraalHotSpotVMConfigNode extends FloatingNode implements LIRLowerable, Canonicalizable {
     public static final NodeClass<GraalHotSpotVMConfigNode> TYPE = NodeClass.create(GraalHotSpotVMConfigNode.class);
 
@@ -109,11 +109,8 @@ public class GraalHotSpotVMConfigNode extends FloatingNode implements LIRLowerab
         return loadIntConfigValue(HotSpotMarkId.LOG_OF_HEAP_REGION_GRAIN_BYTES);
     }
 
-    @SuppressWarnings("unused")
+    public static boolean intrinsify(GraphBuilderContext b, @InjectedNodeParameter Stamp returnStamp, @InjectedNodeParameter GraalHotSpotVMConfig config, HotSpotMarkId mark) {
-    public static boolean intrinsify(GraphBuilderContext b, ResolvedJavaMethod method, @InjectedNodeParameter Stamp returnStamp, @InjectedNodeParameter GraalHotSpotVMConfig config,
+        if (b.getReplacements().isEncodingSnippets()) {
-                    HotSpotMarkId mark) {
-        HotSpotReplacementsImpl replacements = (HotSpotReplacementsImpl) b.getReplacements();
-        if (replacements.isEncodingSnippets()) {
             // This plugin must be deferred so that these constants aren't embedded in libgraal
             return false;
         }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/ClassGetHubNode.java

@@ -30,6 +30,7 @@ import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_1;
 import org.graalvm.compiler.core.common.calc.CanonicalCondition;
 import org.graalvm.compiler.core.common.type.Stamp;
 import org.graalvm.compiler.graph.Node;
+import org.graalvm.compiler.graph.Node.NodeIntrinsicFactory;
 import org.graalvm.compiler.graph.NodeClass;
 import org.graalvm.compiler.graph.spi.Canonicalizable;
 import org.graalvm.compiler.graph.spi.CanonicalizerTool;
@@ -56,7 +57,6 @@ import jdk.vm.ci.meta.ConstantReflectionProvider;
 import jdk.vm.ci.meta.JavaConstant;
 import jdk.vm.ci.meta.JavaKind;
 import jdk.vm.ci.meta.MetaAccessProvider;
-import jdk.vm.ci.meta.ResolvedJavaMethod;
 import jdk.vm.ci.meta.ResolvedJavaType;
 
 /**
@@ -66,6 +66,7 @@ import jdk.vm.ci.meta.ResolvedJavaType;
  * {@link ReadNode#canonicalizeRead(ValueNode, AddressNode, LocationIdentity, CanonicalizerTool)}.
  */
 @NodeInfo(cycles = CYCLES_1, size = SIZE_1)
+@NodeIntrinsicFactory
 public final class ClassGetHubNode extends FloatingNode implements Lowerable, Canonicalizable, ConvertNode {
     public static final NodeClass<ClassGetHubNode> TYPE = NodeClass.create(ClassGetHubNode.class);
     @Input protected ValueNode clazz;
@@ -79,8 +80,7 @@ public final class ClassGetHubNode extends FloatingNode implements Lowerable, Ca
         return canonical(null, metaAccess, constantReflection, allUsagesAvailable, KlassPointerStamp.klass(), clazz);
     }
 
-    @SuppressWarnings("unused")
+    public static boolean intrinsify(GraphBuilderContext b, ValueNode clazz) {
-    public static boolean intrinsify(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode clazz) {
         ValueNode clazzValue = create(clazz, b.getMetaAccess(), b.getConstantReflection(), false);
         b.push(JavaKind.Object, b.append(clazzValue));
         return true;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotAllocationSnippets.java

@@ -151,7 +151,7 @@ public class HotSpotAllocationSnippets extends AllocationSnippets {
                     @ConstantParameter long size,
                     @ConstantParameter boolean fillContents,
                     @ConstantParameter boolean emitMemoryBarrier,
-                    @ConstantParameter AllocationProfilingData profilingData) {
+                    @ConstantParameter HotSpotAllocationProfilingData profilingData) {
         Object result = allocateInstanceImpl(hub.asWord(), prototypeMarkWord, WordFactory.unsigned(size), fillContents, emitMemoryBarrier, true, profilingData);
         return piCastToSnippetReplaceeStamp(result);
     }
@@ -166,7 +166,7 @@ public class HotSpotAllocationSnippets extends AllocationSnippets {
                     @ConstantParameter boolean emitMemoryBarrier,
                     @ConstantParameter boolean maybeUnroll,
                     @ConstantParameter boolean supportsBulkZeroing,
-                    @ConstantParameter AllocationProfilingData profilingData) {
+                    @ConstantParameter HotSpotAllocationProfilingData profilingData) {
         Object result = allocateArrayImpl(hub.asWord(), prototypeMarkWord, length, headerSize, log2ElementSize, fillContents, headerSize, emitMemoryBarrier, maybeUnroll, supportsBulkZeroing,
                         profilingData);
         return piArrayCastToSnippetReplaceeStamp(result, length);
@@ -178,7 +178,7 @@ public class HotSpotAllocationSnippets extends AllocationSnippets {
                     @ConstantParameter long size,
                     @ConstantParameter boolean fillContents,
                     @ConstantParameter boolean emitMemoryBarrier,
-                    @ConstantParameter AllocationProfilingData profilingData) {
+                    @ConstantParameter HotSpotAllocationProfilingData profilingData) {
         // Klass must be initialized by the time the first instance is allocated, therefore we can
         // just load it from the corresponding cell and avoid the resolution check. We have to use a
         // fixed load though, to prevent it from floating above the initialization.
@@ -192,7 +192,7 @@ public class HotSpotAllocationSnippets extends AllocationSnippets {
                     Class<?> classClass,
                     @ConstantParameter boolean fillContents,
                     @ConstantParameter boolean emitMemoryBarrier,
-                    @ConstantParameter AllocationProfilingData profilingData) {
+                    @ConstantParameter HotSpotAllocationProfilingData profilingData) {
         if (probability(DEOPT_PROBABILITY, type == null)) {
             DeoptimizeNode.deopt(None, RuntimeConstraint);
         }
@@ -240,7 +240,7 @@ public class HotSpotAllocationSnippets extends AllocationSnippets {
                     @ConstantParameter boolean emitMemoryBarrier,
                     @ConstantParameter boolean maybeUnroll,
                     @ConstantParameter boolean supportsBulkZeroing,
-                    @ConstantParameter AllocationProfilingData profilingData) {
+                    @ConstantParameter HotSpotAllocationProfilingData profilingData) {
         // Primitive array types are eagerly pre-resolved. We can use a floating load.
         KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub);
         return allocateArrayImpl(picHub.asWord(), prototypeMarkWord, length, headerSize, log2ElementSize, fillContents, headerSize, emitMemoryBarrier, maybeUnroll, supportsBulkZeroing, profilingData);
@@ -256,7 +256,7 @@ public class HotSpotAllocationSnippets extends AllocationSnippets {
                     @ConstantParameter boolean emitMemoryBarrier,
                     @ConstantParameter boolean maybeUnroll,
                     @ConstantParameter boolean supportsBulkZeroing,
-                    @ConstantParameter AllocationProfilingData profilingData) {
+                    @ConstantParameter HotSpotAllocationProfilingData profilingData) {
         // Array type would be resolved by dominating resolution.
         KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub);
         return allocateArrayImpl(picHub.asWord(), prototypeMarkWord, length, headerSize, log2ElementSize, fillContents, headerSize, emitMemoryBarrier, maybeUnroll, supportsBulkZeroing, profilingData);
@@ -272,7 +272,7 @@ public class HotSpotAllocationSnippets extends AllocationSnippets {
                     @ConstantParameter JavaKind knownElementKind,
                     @ConstantParameter int knownLayoutHelper,
                     @ConstantParameter boolean supportsBulkZeroing,
-                    @ConstantParameter AllocationProfilingData profilingData) {
+                    @ConstantParameter HotSpotAllocationProfilingData profilingData) {
         /*
          * We only need the dynamic check for void when we have no static information from
          * knownElementKind.
@@ -637,7 +637,7 @@ public class HotSpotAllocationSnippets extends AllocationSnippets {
             threadBeingInitializedCheck = snippet(HotSpotAllocationSnippets.class, "threadBeingInitializedCheck", null, receiver);
         }
 
-        private AllocationProfilingData getProfilingData(OptionValues localOptions, String path, ResolvedJavaType type) {
+        private HotSpotAllocationProfilingData getProfilingData(OptionValues localOptions, String path, ResolvedJavaType type) {
             if (ProfileAllocations.getValue(localOptions)) {
                 // Create one object per snippet instantiation - this kills the snippet caching as
                 // we need to add the object as a constant to the snippet.

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotG1WriteBarrierSnippets.java

@@ -55,6 +55,7 @@ import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
 import org.graalvm.compiler.replacements.gc.G1WriteBarrierSnippets;
 import org.graalvm.compiler.word.Word;
+import jdk.internal.vm.compiler.word.Pointer;
 import jdk.internal.vm.compiler.word.WordFactory;
 
 import jdk.vm.ci.code.Register;
@@ -68,11 +69,9 @@ public final class HotSpotG1WriteBarrierSnippets extends G1WriteBarrierSnippets
     public static final HotSpotForeignCallDescriptor VALIDATE_OBJECT = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, REEXECUTABLE, NO_LOCATIONS, "validate_object", boolean.class, Word.class,
                     Word.class);
 
-    private final GraalHotSpotVMConfig config;
     private final Register threadRegister;
 
-    public HotSpotG1WriteBarrierSnippets(GraalHotSpotVMConfig config, HotSpotRegistersProvider registers) {
+    public HotSpotG1WriteBarrierSnippets(HotSpotRegistersProvider registers) {
-        this.config = config;
         this.threadRegister = registers.getThreadRegister();
     }
 
@@ -127,13 +126,10 @@ public final class HotSpotG1WriteBarrierSnippets extends G1WriteBarrierSnippets
     }
 
     @Override
-    protected Word cardTableAddress() {
+    protected Word cardTableAddress(Pointer oop) {
-        return WordFactory.unsigned(GraalHotSpotVMConfigNode.cardTableAddress());
+        Word cardTable = WordFactory.unsigned(GraalHotSpotVMConfigNode.cardTableAddress());
-    }
+        int cardTableShift = HotSpotReplacementsUtil.cardTableShift(INJECTED_VMCONFIG);
-
+        return cardTable.add(oop.unsignedShiftRight(cardTableShift));
-    @Override
-    protected int cardTableShift() {
-        return HotSpotReplacementsUtil.cardTableShift(INJECTED_VMCONFIG);
     }
 
     @Override
@@ -158,7 +154,7 @@ public final class HotSpotG1WriteBarrierSnippets extends G1WriteBarrierSnippets
 
     @Override
     protected boolean verifyBarrier() {
-        return ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED || config.verifyBeforeGC || config.verifyAfterGC;
+        return ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED || HotSpotReplacementsUtil.verifyBeforeOrAfterGC(INJECTED_VMCONFIG);
     }
 
     @Override
@@ -204,7 +200,7 @@ public final class HotSpotG1WriteBarrierSnippets extends G1WriteBarrierSnippets
             super(options, factories, providers, providers.getSnippetReflection(), target);
             this.lowerer = new HotspotG1WriteBarrierLowerer(config, factory);
 
-            HotSpotG1WriteBarrierSnippets receiver = new HotSpotG1WriteBarrierSnippets(config, providers.getRegisters());
+            HotSpotG1WriteBarrierSnippets receiver = new HotSpotG1WriteBarrierSnippets(providers.getRegisters());
             g1PreWriteBarrier = snippet(G1WriteBarrierSnippets.class, "g1PreWriteBarrier", null, receiver, GC_INDEX_LOCATION, GC_LOG_LOCATION, SATB_QUEUE_MARKING_LOCATION, SATB_QUEUE_INDEX_LOCATION,
                             SATB_QUEUE_BUFFER_LOCATION);
             g1ReferentReadBarrier = snippet(G1WriteBarrierSnippets.class, "g1ReferentReadBarrier", null, receiver, GC_INDEX_LOCATION, GC_LOG_LOCATION, SATB_QUEUE_MARKING_LOCATION,

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotReplacementsUtil.java

@@ -605,6 +605,11 @@ public class HotSpotReplacementsUtil {
         return WordFactory.unsigned(ComputeObjectAddressNode.get(a, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int)));
     }
 
+    @Fold
+    public static boolean verifyBeforeOrAfterGC(@InjectedParameter GraalHotSpotVMConfig config) {
+        return config.verifyBeforeGC || config.verifyAfterGC;
+    }
+
     /**
      * Idiom for making {@link GraalHotSpotVMConfig} a constant.
      */

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotSerialWriteBarrierSnippets.java

@@ -27,7 +27,6 @@ package org.graalvm.compiler.hotspot.replacements;
 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
 
 import org.graalvm.compiler.debug.DebugHandlersFactory;
-import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
 import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode;
 import org.graalvm.compiler.nodes.gc.SerialArrayRangeWriteBarrier;
@@ -45,10 +44,8 @@ import jdk.internal.vm.compiler.word.WordFactory;
 import jdk.vm.ci.code.TargetDescription;
 
 public class HotSpotSerialWriteBarrierSnippets extends SerialWriteBarrierSnippets {
-    private final GraalHotSpotVMConfig config;
 
-    public HotSpotSerialWriteBarrierSnippets(GraalHotSpotVMConfig config) {
+    public HotSpotSerialWriteBarrierSnippets() {
-        this.config = config;
     }
 
     @Override
@@ -63,12 +60,12 @@ public class HotSpotSerialWriteBarrierSnippets extends SerialWriteBarrierSnippet
 
     @Override
     public boolean verifyBarrier() {
-        return ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED || config.verifyBeforeGC || config.verifyAfterGC;
+        return ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED || HotSpotReplacementsUtil.verifyBeforeOrAfterGC(INJECTED_VMCONFIG);
     }
 
     @Override
     protected byte dirtyCardValue() {
-        return config.dirtyCardValue;
+        return HotSpotReplacementsUtil.dirtyCardValue(INJECTED_VMCONFIG);
     }
 
     public static class Templates extends AbstractTemplates {
@@ -78,11 +75,11 @@ public class HotSpotSerialWriteBarrierSnippets extends SerialWriteBarrierSnippet
 
         private final SerialWriteBarrierLowerer lowerer;
 
-        public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, Group.Factory factory, HotSpotProviders providers, TargetDescription target, GraalHotSpotVMConfig config) {
+        public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, Group.Factory factory, HotSpotProviders providers, TargetDescription target) {
             super(options, factories, providers, providers.getSnippetReflection(), target);
             this.lowerer = new SerialWriteBarrierLowerer(factory);
 
-            HotSpotSerialWriteBarrierSnippets receiver = new HotSpotSerialWriteBarrierSnippets(config);
+            HotSpotSerialWriteBarrierSnippets receiver = new HotSpotSerialWriteBarrierSnippets();
             serialImpreciseWriteBarrier = snippet(SerialWriteBarrierSnippets.class, "serialImpreciseWriteBarrier", null, receiver, GC_CARD_LOCATION);
             serialPreciseWriteBarrier = snippet(SerialWriteBarrierSnippets.class, "serialPreciseWriteBarrier", null, receiver, GC_CARD_LOCATION);
             serialArrayRangeWriteBarrier = snippet(SerialWriteBarrierSnippets.class, "serialArrayRangeWriteBarrier", null, receiver, GC_CARD_LOCATION);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/KlassLayoutHelperNode.java

@@ -31,6 +31,7 @@ import org.graalvm.compiler.core.common.type.ObjectStamp;
 import org.graalvm.compiler.core.common.type.Stamp;
 import org.graalvm.compiler.core.common.type.StampFactory;
 import org.graalvm.compiler.graph.Node;
+import org.graalvm.compiler.graph.Node.NodeIntrinsicFactory;
 import org.graalvm.compiler.graph.NodeClass;
 import org.graalvm.compiler.graph.spi.Canonicalizable;
 import org.graalvm.compiler.graph.spi.CanonicalizerTool;
@@ -49,7 +50,6 @@ import jdk.vm.ci.meta.Constant;
 import jdk.vm.ci.meta.ConstantReflectionProvider;
 import jdk.vm.ci.meta.JavaKind;
 import jdk.vm.ci.meta.MetaAccessProvider;
-import jdk.vm.ci.meta.ResolvedJavaMethod;
 import jdk.vm.ci.meta.ResolvedJavaType;
 
 /**
@@ -57,6 +57,7 @@ import jdk.vm.ci.meta.ResolvedJavaType;
  * information in {@code klass}.
  */
 @NodeInfo(cycles = CYCLES_1, size = SIZE_1)
+@NodeIntrinsicFactory
 public final class KlassLayoutHelperNode extends FloatingNode implements Canonicalizable, Lowerable {
 
     public static final NodeClass<KlassLayoutHelperNode> TYPE = NodeClass.create(KlassLayoutHelperNode.class);
@@ -74,8 +75,7 @@ public final class KlassLayoutHelperNode extends FloatingNode implements Canonic
         return canonical(null, config, klass, stamp, constantReflection, metaAccess);
     }
 
-    @SuppressWarnings("unused")
+    public static boolean intrinsify(GraphBuilderContext b, @InjectedNodeParameter GraalHotSpotVMConfig config, ValueNode klass) {
-    public static boolean intrinsify(GraphBuilderContext b, ResolvedJavaMethod method, @InjectedNodeParameter GraalHotSpotVMConfig config, ValueNode klass) {
         ValueNode valueNode = create(config, klass, b.getConstantReflection(), b.getMetaAccess());
         b.push(JavaKind.Int, b.append(valueNode));
         return true;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BytecodeParser.java

@@ -349,6 +349,7 @@ import org.graalvm.compiler.nodes.MergeNode;
 import org.graalvm.compiler.nodes.NodeView;
 import org.graalvm.compiler.nodes.ParameterNode;
 import org.graalvm.compiler.nodes.PiNode;
+import org.graalvm.compiler.nodes.PluginReplacementNode;
 import org.graalvm.compiler.nodes.ReturnNode;
 import org.graalvm.compiler.nodes.StartNode;
 import org.graalvm.compiler.nodes.StateSplit;
@@ -396,6 +397,7 @@ import org.graalvm.compiler.nodes.extended.LoadHubNode;
 import org.graalvm.compiler.nodes.extended.MembarNode;
 import org.graalvm.compiler.nodes.extended.StateSplitProxyNode;
 import org.graalvm.compiler.nodes.graphbuilderconf.ClassInitializationPlugin;
+import org.graalvm.compiler.nodes.graphbuilderconf.GeneratedInvocationPlugin;
 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration;
 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.BytecodeExceptionMode;
 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext;
@@ -2173,7 +2175,22 @@ public class BytecodeParser implements GraphBuilderContext {
         }
     }
 
-    @SuppressWarnings("try")
+    @Override
+    public void replacePlugin(GeneratedInvocationPlugin plugin, ResolvedJavaMethod targetMethod, ValueNode[] args, PluginReplacementNode.ReplacementFunction replacementFunction) {
+        assert replacementFunction != null;
+        JavaType returnType = maybeEagerlyResolve(targetMethod.getSignature().getReturnType(method.getDeclaringClass()), targetMethod.getDeclaringClass());
+        StampPair returnStamp = getReplacements().getGraphBuilderPlugins().getOverridingStamp(this, returnType, false);
+        if (returnStamp == null) {
+            returnStamp = StampFactory.forDeclaredType(getAssumptions(), returnType, false);
+        }
+        ValueNode node = new PluginReplacementNode(returnStamp.getTrustedStamp(), args, replacementFunction, plugin.getClass().getSimpleName());
+        if (returnType.getJavaKind() == JavaKind.Void) {
+            add(node);
+        } else {
+            addPush(returnType.getJavaKind(), node);
+        }
+    }
+
     protected boolean tryInvocationPlugin(InvokeKind invokeKind, ValueNode[] args, ResolvedJavaMethod targetMethod, JavaKind resultType) {
         InvocationPlugin plugin = graphBuilderConfig.getPlugins().getInvocationPlugins().lookupInvocation(targetMethod);
         if (plugin != null) {
@@ -2183,6 +2200,15 @@ public class BytecodeParser implements GraphBuilderContext {
                 return false;
             }
 
+            if (applyInvocationPlugin(invokeKind, args, targetMethod, resultType, plugin)) {
+                return !plugin.isDecorator();
+            }
+        }
+        return false;
+    }
+
+    @SuppressWarnings("try")
+    protected boolean applyInvocationPlugin(InvokeKind invokeKind, ValueNode[] args, ResolvedJavaMethod targetMethod, JavaKind resultType, InvocationPlugin plugin) {
         InvocationPluginReceiver pluginReceiver = invocationPluginReceiver.init(targetMethod, args);
         assert invokeKind.isDirect() : "Cannot apply invocation plugin on an indirect call site.";
 
@@ -2190,12 +2216,11 @@ public class BytecodeParser implements GraphBuilderContext {
         try (DebugCloseable context = openNodeContext(targetMethod)) {
             if (plugin.execute(this, targetMethod, pluginReceiver, args)) {
                 assert assertions.check(true);
-                    return !plugin.isDecorator();
+                return true;
             } else {
                 assert assertions.check(false);
             }
         }
-        }
         return false;
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/lang/Math_abs.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,12 +32,11 @@ import jdk.vm.ci.meta.ResolvedJavaMethod;
 /*
  */
 public class Math_abs extends UnaryMath {
-
     @SuppressWarnings("serial")
     public static class NaN extends Throwable {
     }
 
-    public static double test(double arg) throws NaN {
+    public static double testAbsD(double arg) throws NaN {
         double v = Math.abs(arg);
         if (Double.isNaN(v)) {
             // NaN can't be tested against itself
@@ -48,43 +47,73 @@ public class Math_abs extends UnaryMath {
 
     @Test
     public void run0() throws Throwable {
-        runTest("test", 5.0d);
+        runTest("testAbsD", 5.0d);
     }
 
     @Test
     public void run1() throws Throwable {
-        runTest("test", -5.0d);
+        runTest("testAbsD", -5.0d);
     }
 
     @Test
     public void run2() throws Throwable {
-        runTest("test", 0.0d);
+        runTest("testAbsD", 0.0d);
     }
 
     @Test
     public void run3() throws Throwable {
-        runTest("test", -0.0d);
+        runTest("testAbsD", -0.0d);
     }
 
     @Test
     public void run4() throws Throwable {
-        runTest("test", java.lang.Double.NEGATIVE_INFINITY);
+        runTest("testAbsD", java.lang.Double.NEGATIVE_INFINITY);
     }
 
     @Test
     public void run5() throws Throwable {
-        runTest("test", java.lang.Double.POSITIVE_INFINITY);
+        runTest("testAbsD", java.lang.Double.POSITIVE_INFINITY);
     }
 
     @Test
     public void run6() throws Throwable {
-        runTest("test", java.lang.Double.NaN);
+        runTest("testAbsD", java.lang.Double.NaN);
     }
 
     @Test
     public void run7() {
         OptionValues options = getInitialOptions();
-        ResolvedJavaMethod method = getResolvedJavaMethod("test");
+        ResolvedJavaMethod method = getResolvedJavaMethod("testAbsD");
         testManyValues(options, method);
     }
+
+    public static int testAbsI(int arg) {
+        return Math.abs(arg);
+    }
+
+    public static long testAbsL(long arg) {
+        return Math.abs(arg);
+    }
+
+    @Test
+    public void run8() {
+        runTest("testAbsI", Integer.MIN_VALUE);
+        runTest("testAbsI", -326543323);
+        runTest("testAbsI", -21325);
+        runTest("testAbsI", -0);
+        runTest("testAbsI", 5432);
+        runTest("testAbsI", 352438548);
+        runTest("testAbsI", Integer.MAX_VALUE);
+    }
+
+    @Test
+    public void run9() {
+        runTest("testAbsL", Long.MIN_VALUE);
+        runTest("testAbsL", -425423654342L);
+        runTest("testAbsL", -21543224L);
+        runTest("testAbsL", -0L);
+        runTest("testAbsL", 1325488L);
+        runTest("testAbsL", 313567897765L);
+        runTest("testAbsL", Long.MAX_VALUE);
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64BitFieldOp.java

@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2019, Arm Limited and affiliates. All rights reserved.
+ * Copyright (c) 2019, 2020, Arm Limited and affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -42,6 +42,8 @@ import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG;
  */
 public class AArch64BitFieldOp extends AArch64LIRInstruction {
     public enum BitFieldOpCode {
+        SBFX,
+        SBFIZ,
         UBFX,
         UBFIZ,
     }
@@ -68,8 +70,14 @@ public class AArch64BitFieldOp extends AArch64LIRInstruction {
     protected void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
         Register dst = asRegister(result);
         Register src = asRegister(input);
-        final int size = input.getPlatformKind().getSizeInBytes() * Byte.SIZE;
+        final int size = result.getPlatformKind().getSizeInBytes() * Byte.SIZE;
         switch (opcode) {
+            case SBFX:
+                masm.sbfm(size, dst, src, lsb, lsb + width - 1);
+                break;
+            case SBFIZ:
+                masm.sbfm(size, dst, src, size - lsb, width - 1);
+                break;
             case UBFX:
                 masm.ubfm(size, dst, src, lsb, lsb + width - 1);
                 break;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/vector/AMD64VectorShuffle.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/alloc/lsra/LinearScanWalker.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2009, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2009, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -673,9 +673,20 @@ class LinearScanWalker extends IntervalWalker {
         splitBeforeUsage(interval, minSplitPos, registerAvailableUntil);
     }
 
-    void splitAndSpillInterval(Interval interval) {
+    /**
+     * Split and spill the given interval. The interval is unconditionally split into a left and
+     * right part. However, if the {@code mustHaveRegUsePos} of the supplied (selected in the
+     * caller) register is later than the entire range of the left interval after splitting, we can
+     * allocate the interval to register {@code reg} without spilling it eagerly.
+     *
+     * @param interval the {@linkplain Interval} to split and spill
+     * @param reg a register selected in the caller most suitable for allocating {@code interval}
+     *            to, only used if the left interval after splitting can be allocated to reg since
+     *            the first {@code mustHaveRegUsePos} of {@code reg} is later
+     * @param mustHaveRegUsePos the first must have usage of the register
+     */
+    void splitAndSpillInterval(Interval interval, Register reg, int mustHaveRegUsePos) {
         assert interval.state == State.Active || interval.state == State.Inactive : "other states not allowed";
-
         int currentPos = currentPosition;
         if (interval.state == State.Inactive) {
             // the interval is currently inactive, so no spill slot is needed for now.
@@ -695,7 +706,40 @@ class LinearScanWalker extends IntervalWalker {
             splitBeforeUsage(interval, minSplitPos, maxSplitPos);
 
             assert interval.nextUsage(RegisterPriority.MustHaveRegister, currentPos) == Integer.MAX_VALUE : "the remaining part is spilled to stack and therefore has no register";
+
+            if (interval.to() >= mustHaveRegUsePos) {
                 splitForSpilling(interval);
+            } else {
+                /*
+                 * Only need to split'n'spill if the register selected has a usage in the current
+                 * interval's range.
+                 *
+                 * Special case loop phi inputs: if we have a loop phi that has no usage inside the
+                 * loop (and the phi has a usage far away in range) we are tempted to spill the left
+                 * interval at definition of the loop phi, however this can cause move resolution to
+                 * insert spill moves to the stack (phi resolve) inside the body of the loop for
+                 * loop end phi inputs.
+                 *
+                 * In this special scenario we have an interval (loop phi forward end input) that
+                 * has no usage inside the loop just far away from the loop, but the register we
+                 * selected has its first usage outside of the range, so instead of eagerly spilling
+                 * here we use the register and hope it suffices to keep the loop phi in register
+                 * altogether, if not possible we can still spill the register and re-use it
+                 * (hopefully at a better position).
+                 */
+                assert reg != null;
+                boolean needSplit = blockPos[reg.number] <= interval.to();
+                int splitPos = blockPos[reg.number];
+                assert splitPos > 0 : "invalid splitPos";
+                assert needSplit || splitPos > interval.from() : "splitting interval at from";
+                interval.assignLocation(reg.asValue(interval.kind()));
+                if (needSplit) {
+                    // register not available for full interval : so split it
+                    splitWhenPartialRegisterAvailable(interval, splitPos);
+                }
+                // perform splitting and spilling for all affected intervals
+                splitAndSpillIntersectingIntervals(reg);
+            }
         }
     }
 
@@ -789,11 +833,10 @@ class LinearScanWalker extends IntervalWalker {
 
     void splitAndSpillIntersectingIntervals(Register reg) {
         assert reg != null : "no register assigned";
-
         for (int i = 0; i < spillIntervals[reg.number].size(); i++) {
             Interval interval = spillIntervals[reg.number].get(i);
             removeFromList(interval);
-            splitAndSpillInterval(interval);
+            splitAndSpillInterval(interval, null, -1/* unconditionally split and spill */);
         }
     }
 
@@ -872,8 +915,12 @@ class LinearScanWalker extends IntervalWalker {
                         throw new OutOfRegistersException("LinearScan: no register found", description);
                     }
 
-                    splitAndSpillInterval(interval);
+                    splitAndSpillInterval(interval, reg, regUsePos);
                     return;
+                } else {
+                    if (debug.isLogEnabled()) {
+                        debug.log("not able to spill current interval. firstUsage(register): %d, usePos: %d", firstUsage, regUsePos);
+                    }
                 }
                 break;
             }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/asm/CompilationResultBuilder.java

@@ -189,6 +189,12 @@ public class CompilationResultBuilder {
         return !uncompressedNullRegister.equals(Register.None) && JavaConstant.NULL_POINTER.equals(nullConstant);
     }
 
+    /**
+     * This flag indicates whether the assembler should emit a separate deoptimization handler for
+     * method handle invocations.
+     */
+    private boolean needsMHDeoptHandler = false;
+
     public CompilationResultBuilder(CodeCacheProvider codeCache,
                     ForeignCallsProvider foreignCalls,
                     FrameMap frameMap,
@@ -729,4 +735,12 @@ public class CompilationResultBuilder {
         }
         return currentCallContext;
     }
+
+    public void setNeedsMHDeoptHandler() {
+        this.needsMHDeoptHandler = true;
+    }
+
+    public boolean needsMHDeoptHandler() {
+        return needsMHDeoptHandler;
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop.phases/src/org/graalvm/compiler/loop/phases/ConvertDeoptimizeToGuardPhase.java

@@ -64,7 +64,6 @@ import org.graalvm.compiler.nodes.util.GraphUtil;
 import org.graalvm.compiler.phases.BasePhase;
 import org.graalvm.compiler.phases.common.DeadCodeEliminationPhase;
 import org.graalvm.compiler.phases.common.LazyValue;
-import org.graalvm.compiler.phases.util.Providers;
 
 import jdk.vm.ci.meta.Constant;
 import jdk.vm.ci.meta.DeoptimizationAction;
@@ -83,8 +82,6 @@ import jdk.vm.ci.meta.DeoptimizationAction;
  */
 public class ConvertDeoptimizeToGuardPhase extends BasePhase<CoreProviders> {
 
-    private static final Providers EMPTY_PROVIDERS = new Providers(null, null, null, null, null, null, null, null, null, null);
-
     @Override
     @SuppressWarnings("try")
     protected void run(final StructuredGraph graph, CoreProviders context) {
@@ -228,7 +225,8 @@ public class ConvertDeoptimizeToGuardPhase extends BasePhase<CoreProviders> {
                             FixedNode next = pred.next();
                             pred.setNext(guard);
                             guard.setNext(next);
-                            SimplifierTool simplifierTool = GraphUtil.getDefaultSimplifier(providers != null ? providers : EMPTY_PROVIDERS, false, graph.getAssumptions(), graph.getOptions());
+                            assert providers != null;
+                            SimplifierTool simplifierTool = GraphUtil.getDefaultSimplifier(providers, false, graph.getAssumptions(), graph.getOptions());
                             survivingSuccessor.simplify(simplifierTool);
                         }
                     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/IfNode.java

@@ -732,12 +732,12 @@ public final class IfNode extends ControlSplitNode implements Simplifiable, LIRL
      * Check it these two blocks end up at the same place. Meeting at the same merge, or
      * deoptimizing in the same way.
      */
-    private static boolean sameDestination(AbstractBeginNode succ1, AbstractBeginNode succ2) {
+    public static boolean sameDestination(AbstractBeginNode succ1, AbstractBeginNode succ2) {
         Node next1 = succ1.next();
         Node next2 = succ2.next();
-        if (next1 instanceof EndNode && next2 instanceof EndNode) {
+        if (next1 instanceof AbstractEndNode && next2 instanceof AbstractEndNode) {
-            EndNode end1 = (EndNode) next1;
+            AbstractEndNode end1 = (AbstractEndNode) next1;
-            EndNode end2 = (EndNode) next2;
+            AbstractEndNode end2 = (AbstractEndNode) next2;
             if (end1.merge() == end2.merge()) {
                 for (PhiNode phi : end1.merge().phis()) {
                     if (phi.valueAt(end1) != phi.valueAt(end2)) {
@@ -1052,18 +1052,26 @@ public final class IfNode extends ControlSplitNode implements Simplifiable, LIRL
      */
     private boolean isSafeConditionalInput(ValueNode value, AbstractBeginNode successor) {
         assert successor.hasNoUsages();
-        if (value.isConstant() || value instanceof ParameterNode || condition.inputs().contains(value)) {
+        if (value.isConstant() || condition.inputs().contains(value)) {
-            // Assume constants are cheap to evaluate and Parameters are always evaluated. Any input
+            // Assume constants are cheap to evaluate. Any input to the condition itself is also
-            // to the condition itself is also unconditionally evaluated.
+            // unconditionally evaluated.
             return true;
         }
 
-        if (value instanceof FixedNode && graph().isAfterFixedReadPhase()) {
+        if (graph().isAfterFixedReadPhase()) {
+            if (value instanceof ParameterNode) {
+                // Assume Parameters are always evaluated but only apply this logic to graphs after
+                // inlining. Checking for ParameterNode causes it to apply to graphs which are going
+                // to be inlined into other graphs which is incorrect.
+                return true;
+            }
+            if (value instanceof FixedNode) {
                 List<Node> nodes = getNodesForBlock(successor);
                 // The successor block is empty so assume that this input evaluated before the
                 // condition.
                 return nodes != null && nodes.size() == 2;
             }
+        }
         return false;
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/PiNode.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,6 +34,7 @@ import org.graalvm.compiler.core.common.type.StampFactory;
 import org.graalvm.compiler.core.common.type.TypeReference;
 import org.graalvm.compiler.debug.DebugContext;
 import org.graalvm.compiler.graph.Node;
+import org.graalvm.compiler.graph.Node.NodeIntrinsicFactory;
 import org.graalvm.compiler.graph.NodeClass;
 import org.graalvm.compiler.graph.spi.Canonicalizable;
 import org.graalvm.compiler.graph.spi.CanonicalizerTool;
@@ -50,7 +51,6 @@ import org.graalvm.compiler.nodes.type.StampTool;
 import org.graalvm.compiler.nodes.virtual.VirtualObjectNode;
 
 import jdk.vm.ci.meta.JavaKind;
-import jdk.vm.ci.meta.ResolvedJavaMethod;
 import jdk.vm.ci.meta.ResolvedJavaType;
 
 //JaCoCo Exclude
@@ -64,6 +64,7 @@ import jdk.vm.ci.meta.ResolvedJavaType;
  * also the scheduling restriction enforced by the guard, will go away.
  */
 @NodeInfo(cycles = CYCLES_0, size = SIZE_0)
+@NodeIntrinsicFactory
 public class PiNode extends FloatingGuardedNode implements LIRLowerable, Virtualizable, Canonicalizable, ValueProxy {
 
     public static final NodeClass<PiNode> TYPE = NodeClass.create(PiNode.class);
@@ -124,8 +125,7 @@ public class PiNode extends FloatingGuardedNode implements LIRLowerable, Virtual
         return new PiNode(object, stamp, guard);
     }
 
-    @SuppressWarnings("unused")
+    public static boolean intrinsify(GraphBuilderContext b, ValueNode object, ValueNode guard) {
-    public static boolean intrinsify(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode object, ValueNode guard) {
         Stamp stamp = AbstractPointerStamp.pointerNonNull(object.stamp(NodeView.DEFAULT));
         ValueNode value = canonical(object, stamp, (GuardingNode) guard, null);
         if (value == null) {
@@ -135,8 +135,7 @@ public class PiNode extends FloatingGuardedNode implements LIRLowerable, Virtual
         return true;
     }
 
-    @SuppressWarnings("unused")
+    public static boolean intrinsify(GraphBuilderContext b, ValueNode object, ResolvedJavaType toType, boolean exactType, boolean nonNull) {
-    public static boolean intrinsify(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode object, ResolvedJavaType toType, boolean exactType, boolean nonNull) {
         Stamp stamp = StampFactory.object(exactType ? TypeReference.createExactTrusted(toType) : TypeReference.createWithoutAssumptions(toType),
                         nonNull || StampTool.isPointerNonNull(object.stamp(NodeView.DEFAULT)));
         ValueNode value = canonical(object, stamp, null, null);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/PluginReplacementNode.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package org.graalvm.compiler.nodes;
+
+import java.util.Map;
+
+import org.graalvm.compiler.core.common.type.Stamp;
+import org.graalvm.compiler.graph.NodeClass;
+import org.graalvm.compiler.graph.NodeInputList;
+import org.graalvm.compiler.nodeinfo.NodeCycles;
+import org.graalvm.compiler.nodeinfo.NodeInfo;
+import org.graalvm.compiler.nodeinfo.NodeSize;
+import org.graalvm.compiler.nodeinfo.Verbosity;
+import org.graalvm.compiler.nodes.graphbuilderconf.GeneratedPluginInjectionProvider;
+import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext;
+
+@NodeInfo(nameTemplate = "PluginReplacement/{p#pluginName}", cycles = NodeCycles.CYCLES_IGNORED, size = NodeSize.SIZE_IGNORED)
+public final class PluginReplacementNode extends FixedWithNextNode {
+    public static final NodeClass<PluginReplacementNode> TYPE = NodeClass.create(PluginReplacementNode.class);
+
+    @Input protected NodeInputList<ValueNode> args;
+    private final ReplacementFunction function;
+    private final String pluginName;
+
+    public PluginReplacementNode(Stamp stamp, ValueNode[] args, ReplacementFunction function, String pluginName) {
+        super(TYPE, stamp);
+        this.args = new NodeInputList<>(this, args);
+        this.function = function;
+        this.pluginName = pluginName;
+    }
+
+    public boolean replace(GraphBuilderContext b, GeneratedPluginInjectionProvider injection) {
+        return function.replace(b, injection, stamp, args);
+    }
+
+    public interface ReplacementFunction {
+        boolean replace(GraphBuilderContext b, GeneratedPluginInjectionProvider injection, Stamp stamp, NodeInputList<ValueNode> args);
+    }
+
+    @Override
+    public String toString(Verbosity verbosity) {
+        if (verbosity == Verbosity.Short) {
+            return super.toString(verbosity) + "/" + pluginName;
+        }
+        return super.toString(verbosity);
+    }
+
+    @Override
+    public Map<Object, Object> getDebugProperties(Map<Object, Object> map) {
+        map.put("name", pluginName);
+        return super.getDebugProperties(map);
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/cfg/ControlFlowGraph.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -442,6 +442,7 @@ public final class ControlFlowGraph implements AbstractControlFlowGraph<Block> {
             assert nodeToBlock.get(cur) == null;
             nodeToBlock.set(cur, block);
             FixedNode next = cur.next();
+            assert next != null : cur;
             if (next instanceof AbstractBeginNode) {
                 block.endNode = cur;
                 return;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/BytecodeExceptionNode.java

@@ -64,6 +64,7 @@ public final class BytecodeExceptionNode extends AbstractMemoryCheckpoint implem
         OUT_OF_BOUNDS(2, ArrayIndexOutOfBoundsException.class),
         CLASS_CAST(2, ClassCastException.class),
         ARRAY_STORE(1, ArrayStoreException.class),
+        ILLEGAL_ARGUMENT_EXCEPTION(1, IllegalArgumentException.class),
         DIVISION_BY_ZERO(0, ArithmeticException.class),
         INTEGER_EXACT_OVERFLOW(0, ArithmeticException.class),
         LONG_EXACT_OVERFLOW(0, ArithmeticException.class);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/ForeignCallNode.java

@@ -36,6 +36,7 @@ import org.graalvm.compiler.core.common.spi.ForeignCallSignature;
 import org.graalvm.compiler.core.common.spi.ForeignCallsProvider;
 import org.graalvm.compiler.core.common.type.Stamp;
 import org.graalvm.compiler.core.common.type.StampFactory;
+import org.graalvm.compiler.graph.Node.NodeIntrinsicFactory;
 import org.graalvm.compiler.graph.NodeClass;
 import org.graalvm.compiler.graph.NodeInputList;
 import org.graalvm.compiler.nodeinfo.NodeInfo;
@@ -47,8 +48,6 @@ import org.graalvm.compiler.nodes.memory.AbstractMemoryCheckpoint;
 
 import jdk.vm.ci.code.BytecodeFrame;
 import jdk.vm.ci.meta.JavaKind;
-import jdk.vm.ci.meta.ResolvedJavaMethod;
-import jdk.vm.ci.meta.ResolvedJavaType;
 
 /**
  * Node for a {@linkplain ForeignCallDescriptor foreign} call.
@@ -61,6 +60,7 @@ import jdk.vm.ci.meta.ResolvedJavaType;
           size = SIZE_2,
           sizeRationale = "Rough estimation of the call operation itself.")
 // @formatter:on
+@NodeIntrinsicFactory
 public class ForeignCallNode extends AbstractMemoryCheckpoint implements ForeignCall {
     public static final NodeClass<ForeignCallNode> TYPE = NodeClass.create(ForeignCallNode.class);
 
@@ -70,17 +70,17 @@ public class ForeignCallNode extends AbstractMemoryCheckpoint implements Foreign
     protected final ForeignCallDescriptor descriptor;
     protected int bci = BytecodeFrame.UNKNOWN_BCI;
 
-    public static boolean intrinsify(GraphBuilderContext b, ResolvedJavaMethod targetMethod, @InjectedNodeParameter Stamp returnStamp, @InjectedNodeParameter ForeignCallsProvider foreignCalls,
+    public static boolean intrinsify(GraphBuilderContext b, @InjectedNodeParameter Stamp returnStamp, @InjectedNodeParameter ForeignCallsProvider foreignCalls,
                     ForeignCallSignature signature, ValueNode... arguments) {
         ForeignCallDescriptor descriptor = foreignCalls.getDescriptor(signature);
-        return doIntrinsify(b, targetMethod, returnStamp, descriptor, arguments, false);
+        return doIntrinsify(b, returnStamp, descriptor, arguments, false);
     }
 
-    public static boolean intrinsify(GraphBuilderContext b, ResolvedJavaMethod targetMethod, @InjectedNodeParameter Stamp returnStamp, ForeignCallDescriptor descriptor, ValueNode... arguments) {
+    public static boolean intrinsify(GraphBuilderContext b, @InjectedNodeParameter Stamp returnStamp, ForeignCallDescriptor descriptor, ValueNode... arguments) {
-        return doIntrinsify(b, targetMethod, returnStamp, descriptor, arguments, false);
+        return doIntrinsify(b, returnStamp, descriptor, arguments, false);
     }
 
-    static boolean doIntrinsify(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Stamp returnStamp, ForeignCallDescriptor descriptor, ValueNode[] arguments, boolean withException) {
+    static boolean doIntrinsify(GraphBuilderContext b, Stamp returnStamp, ForeignCallDescriptor descriptor, ValueNode[] arguments, boolean withException) {
         ForeignCall node;
         if (withException) {
             node = new ForeignCallWithExceptionNode(descriptor, arguments);
@@ -89,8 +89,6 @@ public class ForeignCallNode extends AbstractMemoryCheckpoint implements Foreign
         }
         node.asNode().setStamp(returnStamp);
 
-        assert verifyDescriptor(b, targetMethod, descriptor);
-
         /*
          * Need to update the BCI of a ForeignCallNode so that it gets the stateDuring in the case
          * that the foreign call can deoptimize. As with all deoptimization, we need a state in a
@@ -101,7 +99,7 @@ public class ForeignCallNode extends AbstractMemoryCheckpoint implements Foreign
             node.setBci(nonIntrinsicAncestor.bci());
         }
 
-        JavaKind returnKind = targetMethod.getSignature().getReturnKind();
+        JavaKind returnKind = returnStamp.getStackKind();
         if (returnKind == JavaKind.Void) {
             b.add(node.asNode());
         } else {
@@ -111,17 +109,6 @@ public class ForeignCallNode extends AbstractMemoryCheckpoint implements Foreign
         return true;
     }
 
-    static boolean verifyDescriptor(GraphBuilderContext b, ResolvedJavaMethod targetMethod, ForeignCallDescriptor descriptor) {
-        int parameters = 1;
-        for (Class<?> arg : descriptor.getArgumentTypes()) {
-            ResolvedJavaType res = b.getMetaAccess().lookupJavaType(arg);
-            ResolvedJavaType parameterType = (ResolvedJavaType) targetMethod.getSignature().getParameterType(parameters, targetMethod.getDeclaringClass());
-            assert parameterType.equals(res) : descriptor + ": parameter " + parameters + " mismatch: " + res + " != " + parameterType;
-            parameters++;
-        }
-        return true;
-    }
-
     public ForeignCallNode(ForeignCallsProvider foreignCalls, ForeignCallSignature signature, ValueNode... arguments) {
         this(TYPE, foreignCalls.getDescriptor(signature), arguments);
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/ForeignCallWithExceptionNode.java

@@ -34,6 +34,7 @@ import java.util.Arrays;
 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
 import org.graalvm.compiler.core.common.type.Stamp;
 import org.graalvm.compiler.core.common.type.StampFactory;
+import org.graalvm.compiler.graph.Node.NodeIntrinsicFactory;
 import org.graalvm.compiler.graph.NodeClass;
 import org.graalvm.compiler.graph.NodeInputList;
 import org.graalvm.compiler.graph.spi.Simplifiable;
@@ -52,7 +53,6 @@ import org.graalvm.compiler.nodes.util.GraphUtil;
 
 import jdk.vm.ci.code.BytecodeFrame;
 import jdk.vm.ci.meta.JavaKind;
-import jdk.vm.ci.meta.ResolvedJavaMethod;
 
 /**
  * Node for a {@linkplain ForeignCallDescriptor foreign} call with an {@linkplain WithExceptionNode
@@ -66,6 +66,7 @@ import jdk.vm.ci.meta.ResolvedJavaMethod;
           size = SIZE_2,
           sizeRationale = "Rough estimation of the call operation itself.")
 // @formatter:on
+@NodeIntrinsicFactory
 public class ForeignCallWithExceptionNode extends WithExceptionNode implements ForeignCall, Simplifiable {
     public static final NodeClass<ForeignCallWithExceptionNode> TYPE = NodeClass.create(ForeignCallWithExceptionNode.class);
 
@@ -76,8 +77,8 @@ public class ForeignCallWithExceptionNode extends WithExceptionNode implements F
     protected final ForeignCallDescriptor descriptor;
     protected int bci = BytecodeFrame.UNKNOWN_BCI;
 
-    public static boolean intrinsify(GraphBuilderContext b, ResolvedJavaMethod targetMethod, @InjectedNodeParameter Stamp returnStamp, ForeignCallDescriptor descriptor, ValueNode... arguments) {
+    public static boolean intrinsify(GraphBuilderContext b, @InjectedNodeParameter Stamp returnStamp, ForeignCallDescriptor descriptor, ValueNode... arguments) {
-        return ForeignCallNode.doIntrinsify(b, targetMethod, returnStamp, descriptor, arguments, true);
+        return ForeignCallNode.doIntrinsify(b, returnStamp, descriptor, arguments, true);
     }
 
     public ForeignCallWithExceptionNode(ForeignCallDescriptor descriptor, ValueNode... arguments) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/UnsafeCopyNode.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,20 +26,21 @@ package org.graalvm.compiler.nodes.extended;
 
 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
+import org.graalvm.compiler.graph.Node.NodeIntrinsicFactory;
 import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext;
 import jdk.internal.vm.compiler.word.LocationIdentity;
 
 import jdk.vm.ci.meta.JavaKind;
-import jdk.vm.ci.meta.ResolvedJavaMethod;
 
 /**
  * Copy a value at a location specified as an offset relative to a source object to another location
  * specified as an offset relative to destination object. No null checks are performed.
  */
+@NodeIntrinsicFactory
 public final class UnsafeCopyNode {
 
-    public static boolean intrinsify(GraphBuilderContext b, @SuppressWarnings("unused") ResolvedJavaMethod targetMethod, ValueNode sourceObject, ValueNode sourceOffset, ValueNode destinationObject,
+    public static boolean intrinsify(GraphBuilderContext b, ValueNode sourceObject, ValueNode sourceOffset, ValueNode destinationObject,
                     ValueNode destinationOffset, JavaKind accessKind, LocationIdentity locationIdentity) {
         RawLoadNode value = b.add(new RawLoadNode(sourceObject, sourceOffset, accessKind, locationIdentity));
         b.add(new RawStoreNode(destinationObject, destinationOffset, value, accessKind, locationIdentity));

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GeneratedFoldInvocationPlugin.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package org.graalvm.compiler.nodes.graphbuilderconf;
+
+public abstract class GeneratedFoldInvocationPlugin extends GeneratedInvocationPlugin {
+
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GeneratedInvocationPlugin.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -33,6 +33,7 @@ import java.lang.reflect.Method;
 import org.graalvm.compiler.api.replacements.Fold;
 import org.graalvm.compiler.debug.GraalError;
 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
+import org.graalvm.compiler.nodes.PluginReplacementNode;
 import org.graalvm.compiler.nodes.ValueNode;
 
 import jdk.vm.ci.meta.MetaAccessProvider;
@@ -80,16 +81,24 @@ public abstract class GeneratedInvocationPlugin implements InvocationPlugin {
             return false;
         }
 
-        ResolvedJavaMethod thisExecuteMethod = getExecutedMethod(b);
-        if (b.getMethod().equals(thisExecuteMethod)) {
-            // The "execute" method of this plugin is itself being compiled. In (only) this context,
-            // the injected argument of the call to the @Fold annotated method will be non-null.
         if (IS_BUILDING_NATIVE_IMAGE) {
+            // The use of this plugin in the plugin itself shouldn't be folded since that defeats
+            // the purpose of the fold.
+            ResolvedJavaType foldNodeClass = b.getMetaAccess().lookupJavaType(PluginReplacementNode.ReplacementFunction.class);
+            if (foldNodeClass.isAssignableFrom(b.getMethod().getDeclaringClass())) {
                 return false;
             }
+            ResolvedJavaType foldPluginClass = b.getMetaAccess().lookupJavaType(GeneratedFoldInvocationPlugin.class);
+            if (foldPluginClass.isAssignableFrom(b.getMethod().getDeclaringClass())) {
+                return false;
+            }
+        }
+
+        ResolvedJavaMethod thisExecuteMethod = getExecutedMethod(b);
+        if (b.getMethod().equals(thisExecuteMethod)) {
             return true;
         }
-        throw new AssertionError("must pass null to injected argument of " + foldAnnotatedMethod.format("%H.%n(%p)") + ", not " + arg);
+        throw new AssertionError("must pass null to injected argument of " + foldAnnotatedMethod.format("%H.%n(%p)") + ", not " + arg + " in " + b.getMethod().format("%H.%n(%p)"));
     }
 
     private ResolvedJavaMethod getExecutedMethod(GraphBuilderContext b) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GeneratedNodeIntrinsicInvocationPlugin.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package org.graalvm.compiler.nodes.graphbuilderconf;
+
+import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
+
+import jdk.vm.ci.meta.MetaAccessProvider;
+import jdk.vm.ci.meta.ResolvedJavaMethod;
+import jdk.vm.ci.meta.ResolvedJavaType;
+
+public abstract class GeneratedNodeIntrinsicInvocationPlugin extends GeneratedInvocationPlugin {
+    protected boolean verifyForeignCallDescriptor(GraphBuilderTool b, ResolvedJavaMethod targetMethod, ForeignCallDescriptor descriptor) {
+        MetaAccessProvider metaAccess = b.getMetaAccess();
+        int parameters = 1;
+        for (Class<?> arg : descriptor.getArgumentTypes()) {
+            ResolvedJavaType res = metaAccess.lookupJavaType(arg);
+            ResolvedJavaType parameterType = (ResolvedJavaType) targetMethod.getSignature().getParameterType(parameters, targetMethod.getDeclaringClass());
+            assert parameterType.equals(res) : descriptor + ": parameter " + parameters + " mismatch: " + res + " != " + parameterType;
+            parameters++;
+        }
+        return true;
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GeneratedPluginFactory.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package org.graalvm.compiler.nodes.graphbuilderconf;
+
+public interface GeneratedPluginFactory {
+
+    void registerPlugins(InvocationPlugins plugins, GeneratedPluginInjectionProvider injection);
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GeneratedPluginInjectionProvider.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,9 +26,7 @@ package org.graalvm.compiler.nodes.graphbuilderconf;
 
 import org.graalvm.compiler.core.common.type.Stamp;
 
-public interface NodeIntrinsicPluginFactory {
+public interface GeneratedPluginInjectionProvider {
-
-    public interface InjectionProvider {
 
     <T> T getInjectedArgument(Class<T> type);
 
@@ -40,7 +38,4 @@ public interface NodeIntrinsicPluginFactory {
      *            non-null
      */
     Stamp getInjectedStamp(Class<?> type, boolean nonNull);
-    }
-
-    void registerPlugins(InvocationPlugins plugins, InjectionProvider injection);
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GraphBuilderContext.java

@@ -48,6 +48,7 @@ import org.graalvm.compiler.nodes.Invoke;
 import org.graalvm.compiler.nodes.LogicNode;
 import org.graalvm.compiler.nodes.NodeView;
 import org.graalvm.compiler.nodes.PiNode;
+import org.graalvm.compiler.nodes.PluginReplacementNode;
 import org.graalvm.compiler.nodes.StateSplit;
 import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.nodes.ValueNode;
@@ -377,6 +378,17 @@ public interface GraphBuilderContext extends GraphBuilderTool {
     default AbstractBeginNode genExplicitExceptionEdge(@SuppressWarnings("ununsed") BytecodeExceptionKind exceptionKind) {
         return null;
     }
+
+    /**
+     *
+     * @param plugin
+     * @param targetMethod
+     * @param args
+     * @param replacementFunction
+     */
+    default void replacePlugin(GeneratedInvocationPlugin plugin, ResolvedJavaMethod targetMethod, ValueNode[] args, PluginReplacementNode.ReplacementFunction replacementFunction) {
+        throw GraalError.unimplemented();
+    }
 }
 
 class GraphBuilderContextUtil {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GraphBuilderTool.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -87,4 +87,10 @@ public interface GraphBuilderTool {
         // By default generated plugins must be completely processed during parsing.
         return false;
     }
+
+    @SuppressWarnings("unused")
+    default boolean shouldDeferPlugin(GeneratedInvocationPlugin plugin) {
+        // By default generated plugins must be completely processed during parsing.
+        return false;
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/DynamicNewArrayNode.java

@@ -106,7 +106,7 @@ public class DynamicNewArrayNode extends AbstractNewArrayNode implements Canonic
                 return this;
             }
             ResolvedJavaType type = tool.getConstantReflection().asJavaType(elementType.asConstant());
-            if (type != null && !throwsIllegalArgumentException(type) && tool.getMetaAccessExtensionProvider().canConstantFoldDynamicAllocation(type)) {
+            if (type != null && !throwsIllegalArgumentException(type) && tool.getMetaAccessExtensionProvider().canConstantFoldDynamicAllocation(type.getArrayClass())) {
                 return createNewArrayNode(type);
             }
         }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/InstanceOfNode.java

@@ -34,6 +34,7 @@ import org.graalvm.compiler.core.common.type.ObjectStamp;
 import org.graalvm.compiler.core.common.type.Stamp;
 import org.graalvm.compiler.core.common.type.StampFactory;
 import org.graalvm.compiler.core.common.type.TypeReference;
+import org.graalvm.compiler.graph.Node.NodeIntrinsicFactory;
 import org.graalvm.compiler.graph.NodeClass;
 import org.graalvm.compiler.graph.spi.CanonicalizerTool;
 import org.graalvm.compiler.nodeinfo.NodeInfo;
@@ -54,7 +55,6 @@ import org.graalvm.compiler.nodes.type.StampTool;
 
 import jdk.vm.ci.meta.JavaKind;
 import jdk.vm.ci.meta.JavaTypeProfile;
-import jdk.vm.ci.meta.ResolvedJavaMethod;
 import jdk.vm.ci.meta.ResolvedJavaType;
 import jdk.vm.ci.meta.TriState;
 
@@ -62,6 +62,7 @@ import jdk.vm.ci.meta.TriState;
  * The {@code InstanceOfNode} represents an instanceof test.
  */
 @NodeInfo(cycles = CYCLES_8, size = SIZE_8)
+@NodeIntrinsicFactory
 public class InstanceOfNode extends UnaryOpLogicNode implements Lowerable, Virtualizable {
     public static final NodeClass<InstanceOfNode> TYPE = NodeClass.create(InstanceOfNode.class);
 
@@ -220,8 +221,7 @@ public class InstanceOfNode extends UnaryOpLogicNode implements Lowerable, Virtu
     @NodeIntrinsic
     public static native boolean doInstanceof(@ConstantNodeParameter ResolvedJavaType type, Object object);
 
-    @SuppressWarnings("unused")
+    public static boolean intrinsify(GraphBuilderContext b, ResolvedJavaType type, ValueNode object) {
-    static boolean intrinsify(GraphBuilderContext b, ResolvedJavaMethod method, ResolvedJavaType type, ValueNode object) {
         InstanceOfNode node = new InstanceOfNode(StampFactory.objectNonNull(TypeReference.create(b.getAssumptions(), type)), object, null, null);
         node = b.add(node);
         b.addPush(JavaKind.Int, ConditionalNode.create(node, NodeView.DEFAULT));

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/MethodCallTargetNode.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -38,6 +38,7 @@ import org.graalvm.compiler.nodeinfo.Verbosity;
 import org.graalvm.compiler.nodes.BeginNode;
 import org.graalvm.compiler.nodes.CallTargetNode;
 import org.graalvm.compiler.nodes.FixedGuardNode;
+import org.graalvm.compiler.nodes.FixedNode;
 import org.graalvm.compiler.nodes.Invoke;
 import org.graalvm.compiler.nodes.LogicNode;
 import org.graalvm.compiler.nodes.NodeView;
@@ -177,29 +178,36 @@ public class MethodCallTargetNode extends CallTargetNode implements IterableNode
             return;
         }
 
-        Assumptions assumptions = graph().getAssumptions();
+        if (invokeKind.isInterface()) {
-        /*
+            MethodCallTargetNode result = tryDevirtualizeInterfaceCall(receiver(), targetMethod, profile, graph().getAssumptions(), contextType, this, invoke().asNode());
-         * Even though we are not registering an assumption (see comment below), the optimization is
+            assert result == this;
-         * only valid when speculative optimizations are enabled.
+        }
-         */
+    }
-        if (invokeKind().isIndirect() && invokeKind().isInterface() && assumptions != null) {
 
-            // check if the type of the receiver can narrow the result
+    public static MethodCallTargetNode tryDevirtualizeInterfaceCall(ValueNode receiver, ResolvedJavaMethod targetMethod, JavaTypeProfile profile, Assumptions assumptions, ResolvedJavaType contextType,
-            ValueNode receiver = receiver();
+                    MethodCallTargetNode callTarget, FixedNode insertionPoint) {
+        if (assumptions == null) {
+            /*
+             * Even though we are not registering an assumption (see comment below), the
+             * optimization is only valid when speculative optimizations are enabled.
+             */
+            return callTarget;
+        }
 
         // try to turn a interface call into a virtual call
-            ResolvedJavaType declaredReceiverType = targetMethod().getDeclaringClass();
+        ResolvedJavaType declaredReceiverType = targetMethod.getDeclaringClass();
 
         /*
-             * We need to check the invoke kind to avoid recursive simplification for virtual
+         * We need to check the invoke kind to avoid recursive simplification for virtual interface
-             * interface methods calls.
+         * methods calls.
          */
         if (declaredReceiverType.isInterface()) {
             ResolvedJavaType singleImplementor = declaredReceiverType.getSingleImplementor();
             if (singleImplementor != null && !singleImplementor.equals(declaredReceiverType)) {
                 TypeReference speculatedType = TypeReference.createTrusted(assumptions, singleImplementor);
-                    if (tryCheckCastSingleImplementor(receiver, speculatedType)) {
+                MethodCallTargetNode callTargetResult = tryCheckCastSingleImplementor(receiver, targetMethod, profile, contextType, speculatedType, insertionPoint, callTarget);
-                        return;
+                if (callTargetResult != null) {
+                    return callTargetResult;
                 }
             }
         }
@@ -210,19 +218,24 @@ public class MethodCallTargetNode extends CallTargetNode implements IterableNode
             if (uncheckedStamp != null) {
                 TypeReference speculatedType = StampTool.typeReferenceOrNull(uncheckedStamp);
                 if (speculatedType != null) {
-                        tryCheckCastSingleImplementor(receiver, speculatedType);
+                    MethodCallTargetNode callTargetResult = tryCheckCastSingleImplementor(receiver, targetMethod, profile, contextType, speculatedType, insertionPoint, callTarget);
+                    if (callTargetResult != null) {
+                        return callTargetResult;
                     }
                 }
             }
         }
+        return callTarget;
     }
 
-    private boolean tryCheckCastSingleImplementor(ValueNode receiver, TypeReference speculatedType) {
+    private static MethodCallTargetNode tryCheckCastSingleImplementor(ValueNode receiver, ResolvedJavaMethod targetMethod, JavaTypeProfile profile, ResolvedJavaType contextType,
+                    TypeReference speculatedType,
+                    FixedNode insertionPoint, MethodCallTargetNode callTarget) {
         ResolvedJavaType singleImplementor = speculatedType.getType();
         if (singleImplementor != null) {
-            ResolvedJavaMethod singleImplementorMethod = singleImplementor.resolveConcreteMethod(targetMethod(), invoke().getContextType());
+            ResolvedJavaMethod singleImplementorMethod = singleImplementor.resolveConcreteMethod(targetMethod, contextType);
             if (singleImplementorMethod != null) {
-                /**
+                /*
                  * We have an invoke on an interface with a single implementor. We can replace this
                  * with an invoke virtual.
                  *
@@ -233,22 +246,32 @@ public class MethodCallTargetNode extends CallTargetNode implements IterableNode
                  * an assumption but as we need an instanceof check anyway we can verify both
                  * properties by checking of the receiver is an instance of the single implementor.
                  */
-                AnchoringNode anchor = BeginNode.prevBegin(invoke().asNode());
+                StructuredGraph graph = insertionPoint.graph();
-                LogicNode condition = graph().addOrUniqueWithInputs(InstanceOfNode.create(speculatedType, receiver, getProfile(), anchor));
+                AnchoringNode anchor = BeginNode.prevBegin(insertionPoint);
-                FixedGuardNode guard = graph().add(new FixedGuardNode(condition, DeoptimizationReason.OptimizedTypeCheckViolated, DeoptimizationAction.InvalidateRecompile, false));
+                LogicNode condition = graph.addOrUniqueWithInputs(InstanceOfNode.create(speculatedType, receiver, profile, anchor));
-                graph().addBeforeFixed(invoke().asNode(), guard);
+                FixedGuardNode guard = graph.add(new FixedGuardNode(condition, DeoptimizationReason.OptimizedTypeCheckViolated, DeoptimizationAction.InvalidateRecompile, false));
-                ValueNode valueNode = graph().addOrUnique(new PiNode(receiver, StampFactory.objectNonNull(speculatedType), guard));
+                graph.addBeforeFixed(insertionPoint, guard);
-                arguments().set(0, valueNode);
+                InvokeKind invokeKind;
                 if (speculatedType.isExact()) {
-                    setInvokeKind(InvokeKind.Special);
+                    invokeKind = InvokeKind.Special;
                 } else {
-                    setInvokeKind(InvokeKind.Virtual);
+                    invokeKind = InvokeKind.Virtual;
                 }
-                setTargetMethod(singleImplementorMethod);
+                MethodCallTargetNode callTargetResult = callTarget;
-                return true;
+                ValueNode valueNode = graph.addOrUnique(new PiNode(receiver, StampFactory.objectNonNull(speculatedType), guard));
+                if (callTarget.isAlive()) {
+                    callTarget.arguments().set(0, valueNode);
+                    callTargetResult.setInvokeKind(invokeKind);
+                    callTargetResult.setTargetMethod(singleImplementorMethod);
+                } else {
+                    ValueNode[] arguments = callTarget.arguments().toArray(new ValueNode[callTarget.arguments().size()]);
+                    arguments[0] = valueNode;
+                    callTargetResult = new MethodCallTargetNode(invokeKind, singleImplementorMethod, arguments, callTarget.returnStamp, profile);
+                }
+                return callTargetResult;
             }
         }
-        return false;
+        return null;
     }
 
     public JavaTypeProfile getProfile() {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/CoreProvidersImpl.java

@@ -100,4 +100,20 @@ public class CoreProvidersImpl implements CoreProviders {
     public MetaAccessExtensionProvider getMetaAccessExtensionProvider() {
         return metaAccessExtensionProvider;
     }
+
+    public CoreProvidersImpl copyWith(ConstantReflectionProvider substitution) {
+        assert this.getClass() == CoreProvidersImpl.class : "must override in " + getClass();
+        return new CoreProvidersImpl(metaAccess, substitution, constantFieldProvider, lowerer, replacements, stampProvider, foreignCalls, platformConfigurationProvider, metaAccessExtensionProvider);
+    }
+
+    public CoreProvidersImpl copyWith(ConstantFieldProvider substitution) {
+        assert this.getClass() == CoreProvidersImpl.class : "must override in " + getClass();
+        return new CoreProvidersImpl(metaAccess, constantReflection, substitution, lowerer, replacements, stampProvider, foreignCalls, platformConfigurationProvider, metaAccessExtensionProvider);
+    }
+
+    public CoreProvidersImpl copyWith(Replacements substitution) {
+        assert this.getClass() == CoreProvidersImpl.class : "must override in " + getClass();
+        return new CoreProvidersImpl(metaAccess, constantReflection, constantFieldProvider, lowerer, substitution, stampProvider, foreignCalls, platformConfigurationProvider,
+                        metaAccessExtensionProvider);
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/DelegatingReplacements.java

@@ -27,6 +27,7 @@ package org.graalvm.compiler.nodes.spi;
 import org.graalvm.compiler.api.replacements.SnippetTemplateCache;
 import org.graalvm.compiler.bytecode.BytecodeProvider;
 import org.graalvm.compiler.core.common.CompilationIdentifier;
+import org.graalvm.compiler.core.common.type.Stamp;
 import org.graalvm.compiler.debug.DebugContext;
 import org.graalvm.compiler.graph.NodeSourcePosition;
 import org.graalvm.compiler.nodes.Cancellable;
@@ -56,6 +57,16 @@ public class DelegatingReplacements implements Replacements {
         return delegate.getProviders();
     }
 
+    @Override
+    public <T> T getInjectedArgument(Class<T> type) {
+        return delegate.getInjectedArgument(type);
+    }
+
+    @Override
+    public Stamp getInjectedStamp(Class<?> type, boolean nonNull) {
+        return delegate.getInjectedStamp(type, nonNull);
+    }
+
     @Override
     public GraphBuilderConfiguration.Plugins getGraphBuilderPlugins() {
         return delegate.getGraphBuilderPlugins();
@@ -82,6 +93,11 @@ public class DelegatingReplacements implements Replacements {
         return delegate.isSnippet(method);
     }
 
+    @Override
+    public boolean isEncodingSnippets() {
+        return delegate.isEncodingSnippets();
+    }
+
     @Override
     public void registerSnippet(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean trackNodeSourcePosition, OptionValues options) {
         delegate.registerSnippet(method, original, receiver, trackNodeSourcePosition, options);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/Replacements.java

@@ -33,6 +33,7 @@ import org.graalvm.compiler.graph.NodeSourcePosition;
 import org.graalvm.compiler.nodes.Cancellable;
 import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions;
+import org.graalvm.compiler.nodes.graphbuilderconf.GeneratedPluginInjectionProvider;
 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration;
 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderPlugin;
 import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext;
@@ -45,7 +46,7 @@ import jdk.vm.ci.meta.ResolvedJavaMethod;
 /**
  * Interface for managing replacements.
  */
-public interface Replacements {
+public interface Replacements extends GeneratedPluginInjectionProvider {
 
     CoreProviders getProviders();
 
@@ -84,6 +85,14 @@ public interface Replacements {
      */
     boolean isSnippet(ResolvedJavaMethod method);
 
+    /**
+     * Returns {@code true} if this {@code Replacements} is being used for preparation of snippets
+     * and substitutions for libgraal.
+     */
+    default boolean isEncodingSnippets() {
+        return false;
+    }
+
     /**
      * Registers a method as snippet.
      */

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/util/GraphUtil.java

@@ -41,6 +41,7 @@ import org.graalvm.compiler.bytecode.Bytecode;
 import org.graalvm.compiler.code.SourceStackTraceBailoutException;
 import org.graalvm.compiler.core.common.type.ObjectStamp;
 import org.graalvm.compiler.debug.DebugContext;
+import org.graalvm.compiler.debug.GraalError;
 import org.graalvm.compiler.graph.Graph;
 import org.graalvm.compiler.graph.Node;
 import org.graalvm.compiler.graph.NodeBitMap;
@@ -55,6 +56,7 @@ import org.graalvm.compiler.nodes.AbstractMergeNode;
 import org.graalvm.compiler.nodes.ConstantNode;
 import org.graalvm.compiler.nodes.ControlSinkNode;
 import org.graalvm.compiler.nodes.ControlSplitNode;
+import org.graalvm.compiler.nodes.EndNode;
 import org.graalvm.compiler.nodes.FixedNode;
 import org.graalvm.compiler.nodes.FixedWithNextNode;
 import org.graalvm.compiler.nodes.FrameState;
@@ -63,6 +65,7 @@ import org.graalvm.compiler.nodes.IfNode;
 import org.graalvm.compiler.nodes.LoopBeginNode;
 import org.graalvm.compiler.nodes.LoopEndNode;
 import org.graalvm.compiler.nodes.LoopExitNode;
+import org.graalvm.compiler.nodes.MergeNode;
 import org.graalvm.compiler.nodes.NodeView;
 import org.graalvm.compiler.nodes.PhiNode;
 import org.graalvm.compiler.nodes.PiNode;
@@ -73,9 +76,11 @@ import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.ValuePhiNode;
 import org.graalvm.compiler.nodes.ValueProxyNode;
 import org.graalvm.compiler.nodes.WithExceptionNode;
+import org.graalvm.compiler.nodes.extended.MultiGuardNode;
 import org.graalvm.compiler.nodes.java.LoadIndexedNode;
 import org.graalvm.compiler.nodes.java.MethodCallTargetNode;
 import org.graalvm.compiler.nodes.java.MonitorIdNode;
+import org.graalvm.compiler.nodes.memory.MemoryPhiNode;
 import org.graalvm.compiler.nodes.spi.ArrayLengthProvider;
 import org.graalvm.compiler.nodes.spi.ArrayLengthProvider.FindLengthMode;
 import org.graalvm.compiler.nodes.spi.CoreProviders;
@@ -124,7 +129,7 @@ public class GraphUtil {
         debug.dump(DebugContext.DETAILED_LEVEL, node.graph(), "After fixing merges (killCFG %s)", node);
 
         // Mark non-fixed nodes
-        markUsages(markedNodes);
+        markUsagesForKill(markedNodes);
 
         // Detach marked nodes from non-marked nodes
         for (Node marked : markedNodes) {
@@ -223,19 +228,37 @@ public class GraphUtil {
         }
     }
 
-    private static void markUsages(EconomicSet<Node> markedNodes) {
+    private static void markUsagesForKill(EconomicSet<Node> markedNodes) {
         NodeStack workStack = new NodeStack(markedNodes.size() + 4);
         for (Node marked : markedNodes) {
             workStack.push(marked);
         }
+        ArrayList<MultiGuardNode> unmarkedMultiGuards = new ArrayList<>();
         while (!workStack.isEmpty()) {
             Node marked = workStack.pop();
             for (Node usage : marked.usages()) {
-                if (!markedNodes.contains(usage)) {
+                boolean doMark = true;
+                if (usage instanceof MultiGuardNode) {
+                    // Only mark a MultiGuardNode for deletion if all of its guards are marked for
+                    // deletion. Otherwise, we would kill nodes outside the path to be killed.
+                    MultiGuardNode multiGuard = (MultiGuardNode) usage;
+                    for (Node guard : multiGuard.inputs()) {
+                        if (!markedNodes.contains(guard)) {
+                            doMark = false;
+                            unmarkedMultiGuards.add(multiGuard);
+                        }
+                    }
+                }
+                if (doMark && !markedNodes.contains(usage)) {
                     workStack.push(usage);
                     markedNodes.add(usage);
                 }
             }
+            // Detach unmarked multi guards from the marked node
+            for (MultiGuardNode multiGuard : unmarkedMultiGuards) {
+                multiGuard.replaceFirstInput(marked, null);
+            }
+            unmarkedMultiGuards.clear();
         }
     }
 
@@ -1144,4 +1167,74 @@ public class GraphUtil {
     public static boolean mayRemoveSplit(IfNode ifNode) {
         return GraphUtil.checkFrameState(ifNode.trueSuccessor(), MAX_FRAMESTATE_SEARCH_DEPTH) && GraphUtil.checkFrameState(ifNode.falseSuccessor(), MAX_FRAMESTATE_SEARCH_DEPTH);
     }
+
+    /**
+     * An if node with an empty body at the end of a loop is represented with a {@link LoopEndNode}
+     * at the end of each path. For some optimizations it is more useful to have a representation of
+     * the if statement as a proper diamond with a merge after the two bodies, followed by a
+     * {@link LoopEndNode}. This method tries to transform the given {@code ifNode} into such a
+     * form, introducing new phi nodes for the diamond and patching up the loop's phis accordingly.
+     * On success, the newly introduced loop end node is returned. If the given {@code ifNode} is
+     * not an if statement with empty bodies at the end of the loop, the graph is not modified, and
+     * {@code null} is returned.
+     *
+     * Note that the diamond representation is not canonical and will be undone by the next
+     * application of {@link LoopEndNode#simplify(SimplifierTool)}.
+     */
+    public static LoopEndNode tryToTransformToEmptyLoopDiamond(IfNode ifNode, LoopBeginNode loopBegin) {
+        if (ifNode.trueSuccessor().next() instanceof AbstractEndNode && ifNode.falseSuccessor().next() instanceof AbstractEndNode) {
+            AbstractEndNode trueEnd = (AbstractEndNode) ifNode.trueSuccessor().next();
+            AbstractEndNode falseEnd = (AbstractEndNode) ifNode.falseSuccessor().next();
+            if (trueEnd.merge() == loopBegin && falseEnd.merge() == loopBegin) {
+                StructuredGraph graph = loopBegin.graph();
+                for (PhiNode phi : loopBegin.phis()) {
+                    if (!(phi instanceof ValuePhiNode || phi instanceof MemoryPhiNode)) {
+                        return null;
+                    }
+                }
+
+                EndNode newTrueEnd = graph.add(new EndNode());
+                EndNode newFalseEnd = graph.add(new EndNode());
+                MergeNode merge = graph.add(new MergeNode());
+                merge.addForwardEnd(newTrueEnd);
+                merge.addForwardEnd(newFalseEnd);
+
+                ArrayList<PhiNode> replacementPhis = new ArrayList<>(loopBegin.phis().count());
+                for (PhiNode phi : loopBegin.phis()) {
+                    if (phi instanceof ValuePhiNode) {
+                        ValuePhiNode valuePhi = (ValuePhiNode) phi;
+                        ValuePhiNode newPhi = phi.graph().unique(new ValuePhiNode(valuePhi.stamp(NodeView.DEFAULT), merge, new ValueNode[]{valuePhi.valueAt(trueEnd), valuePhi.valueAt(falseEnd)}));
+                        replacementPhis.add(newPhi);
+                    } else if (phi instanceof MemoryPhiNode) {
+                        MemoryPhiNode memoryPhi = (MemoryPhiNode) phi;
+                        MemoryPhiNode newPhi = phi.graph().unique(new MemoryPhiNode(merge, memoryPhi.getLocationIdentity(), new ValueNode[]{memoryPhi.valueAt(trueEnd), memoryPhi.valueAt(falseEnd)}));
+                        replacementPhis.add(newPhi);
+                    } else {
+                        GraalError.shouldNotReachHere();
+                    }
+                }
+
+                loopBegin.removeEnd(trueEnd);
+                loopBegin.removeEnd(falseEnd);
+                ifNode.trueSuccessor().setNext(newTrueEnd);
+                ifNode.falseSuccessor().setNext(newFalseEnd);
+                trueEnd.safeDelete();
+                falseEnd.safeDelete();
+
+                LoopEndNode newEnd = graph.add(new LoopEndNode(loopBegin));
+                merge.setNext(newEnd);
+                int i = 0;
+                for (PhiNode phi : loopBegin.phis()) {
+                    ValueNode replacementPhi = replacementPhis.get(i);
+                    assert (phi instanceof ValuePhiNode && replacementPhi instanceof ValuePhiNode) || (phi instanceof MemoryPhiNode && replacementPhi instanceof MemoryPhiNode);
+                    phi.addInput(replacementPhi);
+                    i++;
+                }
+                assert i == replacementPhis.size() : "did not consume all values";
+
+                return newEnd;
+            }
+        }
+        return null;
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionsParser.java

@@ -129,7 +129,12 @@ public class OptionsParser {
                     msg.format("%n    %s=<value>", match.getName());
                 }
             }
-            throw new IllegalArgumentException(msg.toString());
+            IllegalArgumentException iae = new IllegalArgumentException(msg.toString());
+            if (isFromLibGraal(iae)) {
+                msg.format("%nIf %s is a libgraal option, it must be specified with '-Dlibgraal.%s' as opposed to '-Dgraal.%s'.", name, name, name);
+                iae = new IllegalArgumentException(msg.toString());
+            }
+            throw iae;
         }
 
         Object value = parseOptionValue(desc, uncheckedValue);
@@ -137,6 +142,15 @@ public class OptionsParser {
         desc.getOptionKey().update(values, value);
     }
 
+    private static boolean isFromLibGraal(Throwable t) {
+        for (StackTraceElement frame : t.getStackTrace()) {
+            if ("jdk.internal.vm.compiler.libgraal.LibGraal".equals(frame.getClassName())) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     /** Parses a given option value with a known descriptor. */
     public static Object parseOptionValue(OptionDescriptor desc, Object uncheckedValue) {
         Class<?> optionType = desc.getOptionValueType();

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/info/AssumptionInlineInfo.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,7 @@ import jdk.internal.vm.compiler.collections.EconomicSet;
 import org.graalvm.compiler.graph.Node;
 import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind;
 import org.graalvm.compiler.nodes.Invoke;
+import org.graalvm.compiler.nodes.spi.CoreProviders;
 import org.graalvm.compiler.phases.common.inlining.InliningUtil;
 import org.graalvm.compiler.phases.util.Providers;
 
@@ -48,7 +49,7 @@ public class AssumptionInlineInfo extends ExactInlineInfo {
     }
 
     @Override
-    public EconomicSet<Node> inline(Providers providers, String reason) {
+    public EconomicSet<Node> inline(CoreProviders providers, String reason) {
         takenAssumption.recordTo(invoke.asNode().graph().getAssumptions());
         return super.inline(providers, reason);
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/info/ExactInlineInfo.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,6 +27,7 @@ package org.graalvm.compiler.phases.common.inlining.info;
 import jdk.internal.vm.compiler.collections.EconomicSet;
 import org.graalvm.compiler.graph.Node;
 import org.graalvm.compiler.nodes.Invoke;
+import org.graalvm.compiler.nodes.spi.CoreProviders;
 import org.graalvm.compiler.phases.common.inlining.info.elem.Inlineable;
 import org.graalvm.compiler.phases.util.Providers;
 
@@ -53,7 +54,7 @@ public class ExactInlineInfo extends AbstractInlineInfo {
     }
 
     @Override
-    public EconomicSet<Node> inline(Providers providers, String reason) {
+    public EconomicSet<Node> inline(CoreProviders providers, String reason) {
         return inline(invoke, concrete, inlineableElement, !suppressNullCheck, reason);
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/info/InlineInfo.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,7 @@ import jdk.internal.vm.compiler.collections.EconomicSet;
 import org.graalvm.compiler.graph.Node;
 import org.graalvm.compiler.nodes.Invoke;
 import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.nodes.spi.CoreProviders;
 import org.graalvm.compiler.options.OptionValues;
 import org.graalvm.compiler.phases.common.CanonicalizerPhase;
 import org.graalvm.compiler.phases.common.inlining.info.elem.Inlineable;
@@ -77,7 +78,7 @@ public interface InlineInfo {
      *
      * @return a collection of nodes that need to be canonicalized after the inlining
      */
-    EconomicSet<Node> inline(Providers providers, String reason);
+    EconomicSet<Node> inline(CoreProviders providers, String reason);
 
     /**
      * Try to make the call static bindable to avoid interface and virtual method calls.

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/info/MultiTypeGuardInlineInfo.java

@@ -53,6 +53,7 @@ import org.graalvm.compiler.nodes.extended.LoadHubNode;
 import org.graalvm.compiler.nodes.java.ExceptionObjectNode;
 import org.graalvm.compiler.nodes.java.MethodCallTargetNode;
 import org.graalvm.compiler.nodes.java.TypeSwitchNode;
+import org.graalvm.compiler.nodes.spi.CoreProviders;
 import org.graalvm.compiler.nodes.spi.StampProvider;
 import org.graalvm.compiler.nodes.util.GraphUtil;
 import org.graalvm.compiler.phases.common.inlining.InliningUtil;
@@ -158,7 +159,7 @@ public class MultiTypeGuardInlineInfo extends AbstractInlineInfo {
     }
 
     @Override
-    public EconomicSet<Node> inline(Providers providers, String reason) {
+    public EconomicSet<Node> inline(CoreProviders providers, String reason) {
         if (hasSingleMethod()) {
             return inlineSingleMethod(graph(), providers.getStampProvider(), providers.getConstantReflection(), reason);
         } else {
@@ -184,7 +185,7 @@ public class MultiTypeGuardInlineInfo extends AbstractInlineInfo {
         return notRecordedTypeProbability > 0;
     }
 
-    private EconomicSet<Node> inlineMultipleMethods(StructuredGraph graph, Providers providers, String reason) {
+    private EconomicSet<Node> inlineMultipleMethods(StructuredGraph graph, CoreProviders providers, String reason) {
         int numberOfMethods = concretes.size();
         FixedNode continuation = invoke.next();
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/info/TypeGuardInlineInfo.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -38,6 +38,7 @@ import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.calc.CompareNode;
 import org.graalvm.compiler.nodes.extended.LoadHubNode;
+import org.graalvm.compiler.nodes.spi.CoreProviders;
 import org.graalvm.compiler.phases.common.inlining.InliningUtil;
 import org.graalvm.compiler.phases.common.inlining.info.elem.Inlineable;
 import org.graalvm.compiler.phases.util.Providers;
@@ -101,7 +102,7 @@ public class TypeGuardInlineInfo extends AbstractInlineInfo {
     }
 
     @Override
-    public EconomicSet<Node> inline(Providers providers, String reason) {
+    public EconomicSet<Node> inline(CoreProviders providers, String reason) {
         createGuard(graph(), providers);
         return inline(invoke, concrete, inlineableElement, false, reason);
     }
@@ -113,7 +114,7 @@ public class TypeGuardInlineInfo extends AbstractInlineInfo {
     }
 
     @SuppressWarnings("try")
-    private void createGuard(StructuredGraph graph, Providers providers) {
+    private void createGuard(StructuredGraph graph, CoreProviders providers) {
         try (DebugCloseable context = invoke.asNode().withNodeSourcePosition()) {
             ValueNode nonNullReceiver = InliningUtil.nonNullReceiver(invoke);
             LoadHubNode receiverHub = graph.unique(new LoadHubNode(providers.getStampProvider(), nonNullReceiver));

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/walker/InliningData.java

@@ -68,7 +68,6 @@ import org.graalvm.compiler.phases.common.inlining.info.elem.Inlineable;
 import org.graalvm.compiler.phases.common.inlining.info.elem.InlineableGraph;
 import org.graalvm.compiler.phases.common.inlining.policy.InliningPolicy;
 import org.graalvm.compiler.phases.tiers.HighTierContext;
-import org.graalvm.compiler.phases.util.Providers;
 
 import jdk.vm.ci.code.BailoutException;
 import jdk.vm.ci.meta.Assumptions.AssumptionResult;
@@ -412,7 +411,7 @@ public class InliningData {
             try (DebugContext.Scope scope = debug.scope("doInline", callerGraph)) {
                 EconomicSet<Node> canonicalizedNodes = EconomicSet.create(Equivalence.IDENTITY);
                 canonicalizedNodes.addAll(calleeInfo.invoke().asNode().usages());
-                EconomicSet<Node> parameterUsages = calleeInfo.inline(new Providers(context), reason);
+                EconomicSet<Node> parameterUsages = calleeInfo.inline(context.getProviders(), reason);
                 canonicalizedNodes.addAll(parameterUsages);
                 counterInliningRuns.increment(debug);
                 debug.dump(DebugContext.DETAILED_LEVEL, callerGraph, "after %s", calleeInfo);
@@ -468,7 +467,7 @@ public class InliningData {
         }
 
         if (context.getOptimisticOptimizations().devirtualizeInvokes(calleeInfo.graph().getOptions())) {
-            calleeInfo.tryToDevirtualizeInvoke(new Providers(context));
+            calleeInfo.tryToDevirtualizeInvoke(context.getProviders());
         }
 
         return false;
@@ -496,7 +495,7 @@ public class InliningData {
      * <p>
      * The {@link InlineInfo} used to get things rolling is kept around in the
      * {@link MethodInvocation}, it will be needed in case of inlining, see
-     * {@link InlineInfo#inline(Providers, String)}
+     * {@link InlineInfo#inline(org.graalvm.compiler.nodes.spi.CoreProviders, String)}
      * </p>
      */
     private void processNextInvoke() {