8235634: Update Graal

Reviewed-by: kvn

src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/Main.java

@@ -224,9 +224,14 @@ public final class Main {
                 System.gc();
             }
 
-            HotSpotGC graal_gc = runtime.getGarbageCollector();
-            int def = graal_gc.ordinal() + 1;
-            String name = "CollectedHeap::" + graal_gc.name();
+            HotSpotGC graalGC = runtime.getGarbageCollector();
+            // Prior to JDK 14, the Graal HotSpotGC enum order matched the JDK CollectedHeap enum
+            // order, so using the ordinal value worked fine. In JDK 14, CMS was removed on the
+            // JDK side, so we need a symbolic lookup of the JDK value.
+            int def = graalGC.ordinal() + 1;
+            // The GC names are spelled the same in both enums, so no clever remapping is needed
+            // here.
+            String name = "CollectedHeap::" + graalGC.name();
             int gc = graalHotSpotVMConfig.getConstant(name, Integer.class, def);
 
             BinaryContainer binaryContainer = new BinaryContainer(graalOptions, graalHotSpotVMConfig, graphBuilderConfig, gc, JVM_VERSION);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.api.directives/src/org/graalvm/compiler/api/directives/GraalDirectives.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -71,6 +71,22 @@ public final class GraalDirectives {
     public static void controlFlowAnchor() {
     }
 
+    /**
+     * A call to this method will force the compiler to assume this instruction has a visible memory
+     * effect killing all memory locations.
+     */
+    public static void sideEffect() {
+
+    }
+
+    /**
+     * A call to this method will force the compiler to assume this instruction has a visible memory
+     * effect killing all memory locations.
+     */
+    public static int sideEffect(@SuppressWarnings("unused") int a) {
+        return 0;
+    }
+
     /**
      * Injects a probability for the given condition into the profiling information of a branch
      * instruction. The probability must be a value between 0.0 and 1.0 (inclusive).

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Assembler.java

@@ -1298,9 +1298,9 @@ public class AMD64Assembler extends AMD64BaseAssembler {
      */
     public static class VexRVMROp extends VexOp {
         // @formatter:off
-        public static final VexRVMROp VPBLENDVB  = new VexRVMROp("VPBLENDVB",  P_66, M_0F3A, W0, 0x4C, VEXOpAssertion.AVX1_2);
-        public static final VexRVMROp VPBLENDVPS = new VexRVMROp("VPBLENDVPS", P_66, M_0F3A, W0, 0x4A, VEXOpAssertion.AVX1);
-        public static final VexRVMROp VPBLENDVPD = new VexRVMROp("VPBLENDVPD", P_66, M_0F3A, W0, 0x4B, VEXOpAssertion.AVX1);
+        public static final VexRVMROp VPBLENDVB = new VexRVMROp("VPBLENDVB", P_66, M_0F3A, W0, 0x4C, VEXOpAssertion.AVX1_2);
+        public static final VexRVMROp VBLENDVPS = new VexRVMROp("VBLENDVPS", P_66, M_0F3A, W0, 0x4A, VEXOpAssertion.AVX1);
+        public static final VexRVMROp VBLENDVPD = new VexRVMROp("VBLENDVPD", P_66, M_0F3A, W0, 0x4B, VEXOpAssertion.AVX1);
         // @formatter:on
 
         protected VexRVMROp(String opcode, int pp, int mmmmm, int w, int op, VEXOpAssertion assertion) {
@@ -3139,14 +3139,14 @@ public class AMD64Assembler extends AMD64BaseAssembler {
         SSEOp.XOR.emit(this, PS, dst, src);
     }
 
-    protected final void decl(Register dst) {
+    public final void decl(Register dst) {
         // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
         prefix(dst);
         emitByte(0xFF);
         emitModRM(1, dst);
     }
 
-    protected final void incl(Register dst) {
+    public final void incl(Register dst) {
         // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
         prefix(dst);
         emitByte(0xFF);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AVXKind.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -111,6 +111,17 @@ public final class AVXKind {
         return getAVXKind(kind.getScalar(), newSize);
     }
 
+    public static AMD64Kind getMaskKind(AMD64Kind kind) {
+        switch (kind.getScalar()) {
+            case SINGLE:
+                return getAVXKind(AMD64Kind.DWORD, kind.getVectorLength());
+            case DOUBLE:
+                return getAVXKind(AMD64Kind.QWORD, kind.getVectorLength());
+            default:
+                return kind;
+        }
+    }
+
     public static AMD64Kind getAVXKind(AMD64Kind base, AVXSize size) {
         for (AMD64Kind ret : AMD64Kind.values()) {
             if (ret.getScalar() == base && ret.getSizeInBytes() == size.getBytes()) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.sparc/src/org/graalvm/compiler/asm/sparc/SPARCAssembler.java

@@ -628,6 +628,13 @@ public abstract class SPARCAssembler extends Assembler {
             return value;
         }
 
+        public int getOpfCCValue() {
+            /*
+             * In the opf_cc encoding for FMOVcc, the third bit is set to indicate icc/xcc.
+             */
+            return (isFloat ? value : (value | 0x4));
+        }
+
         public String getOperator() {
             return operator;
         }
@@ -1613,7 +1620,7 @@ public abstract class SPARCAssembler extends Assembler {
             inst = BitSpec.rd.setBits(inst, rd.encoding());
             inst = BitSpec.op3.setBits(inst, opfLow.op3.value);
             inst = BitSpec.opfCond.setBits(inst, condition.value);
-            inst = BitSpec.opfCC.setBits(inst, cc.value);
+            inst = BitSpec.opfCC.setBits(inst, cc.getOpfCCValue());
             inst = BitSpec.opfLow.setBits(inst, opfLow.value);
             inst = BitSpec.rs2.setBits(inst, rs2.encoding());
             masm.emitInt(inst);
@@ -2193,7 +2200,7 @@ public abstract class SPARCAssembler extends Assembler {
     }
 
     private void fmovcc(ConditionFlag cond, CC cc, Register rs2, Register rd, int opfLow) {
-        int opfCC = cc.value;
+        int opfCC = cc.getOpfCCValue();
         int a = opfCC << 11 | opfLow << 5 | rs2.encoding;
         fmt10(rd.encoding, Fpop2.value, cond.value, a);
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64AddressNode.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -63,6 +63,14 @@ public class AMD64AddressNode extends AddressNode implements Simplifiable, LIRLo
 
     private int displacement;
 
+    /*
+     * If this address has been improved by folding an uncompress operation into it, this is set by
+     * the address lowering to the uncompression scale used by the encoding strategy. It is null
+     * otherwise. This might be different from scale if we lowered an uncompression followed by
+     * further improvements that modify the scale.
+     */
+    private Scale uncompressionScale;
+
     public AMD64AddressNode(ValueNode base) {
         this(base, null);
     }
@@ -72,6 +80,7 @@ public class AMD64AddressNode extends AddressNode implements Simplifiable, LIRLo
         this.base = base;
         this.index = index;
         this.scale = Scale.Times1;
+        this.uncompressionScale = null;
     }
 
     public void canonicalizeIndex(SimplifierTool tool) {
@@ -103,12 +112,14 @@ public class AMD64AddressNode extends AddressNode implements Simplifiable, LIRLo
         AllocatableValue baseValue = base == null ? Value.ILLEGAL : tool.asAllocatable(gen.operand(base));
         AllocatableValue indexValue = index == null ? Value.ILLEGAL : tool.asAllocatable(gen.operand(index));
 
-        AllocatableValue baseReference = LIRKind.derivedBaseFromValue(baseValue);
+        AllocatableValue baseReference = base == null ? null : LIRKind.derivedBaseFromValue(baseValue);
         AllocatableValue indexReference;
         if (index == null) {
             indexReference = null;
         } else if (scale.equals(Scale.Times1)) {
             indexReference = LIRKind.derivedBaseFromValue(indexValue);
+        } else if (scale.equals(uncompressionScale) && LIRKind.isScalarCompressedReference(indexValue.getValueKind())) {
+            indexReference = LIRKind.derivedBaseFromValue(indexValue);
         } else {
             if (LIRKind.isValue(indexValue)) {
                 indexReference = null;
@@ -163,6 +174,10 @@ public class AMD64AddressNode extends AddressNode implements Simplifiable, LIRLo
         this.displacement = displacement;
     }
 
+    public void setUncompressionScale(Scale scale) {
+        this.uncompressionScale = scale;
+    }
+
     @Override
     public long getMaxConstantDisplacement() {
         return displacement;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64ArithmeticLIRGenerator.java

@@ -1305,7 +1305,7 @@ public class AMD64ArithmeticLIRGenerator extends ArithmeticLIRGenerator implemen
                 return;
             } else if (c instanceof VMConstant) {
                 VMConstant vc = (VMConstant) c;
-                if (size == DWORD && !GeneratePIC.getValue(getOptions())) {
+                if (size == DWORD && !GeneratePIC.getValue(getOptions()) && getLIRGen().target().inlineObjects) {
                     getLIRGen().append(new AMD64BinaryConsumer.VMConstOp(CMP.getMIOpcode(DWORD, false), left, vc));
                 } else {
                     getLIRGen().append(new AMD64BinaryConsumer.DataOp(CMP.getRMOpcode(size), size, left, vc));

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64LIRGenerator.java

@@ -409,7 +409,7 @@ public abstract class AMD64LIRGenerator extends LIRGenerator {
         return result;
     }
 
-    private static AVXSize getRegisterSize(Value a) {
+    protected static AVXSize getRegisterSize(Value a) {
         AMD64Kind kind = (AMD64Kind) a.getPlatformKind();
         if (kind.isXMM()) {
             return AVXKind.getRegisterSize(kind);
@@ -479,18 +479,19 @@ public abstract class AMD64LIRGenerator extends LIRGenerator {
         if (JavaConstant.isNull(a.getConstant())) {
             append(new AMD64BinaryConsumer.MemoryConstOp(CMP, size, b, 0, state));
             return true;
-        } else if (a.getConstant() instanceof VMConstant && size == DWORD) {
+        } else if (a.getConstant() instanceof VMConstant && size == DWORD && target().inlineObjects) {
             VMConstant vc = (VMConstant) a.getConstant();
             append(new AMD64BinaryConsumer.MemoryVMConstOp(CMP.getMIOpcode(size, false), b, vc, state));
             return true;
         } else {
-            long value = a.getJavaConstant().asLong();
-            if (NumUtil.is32bit(value)) {
-                append(new AMD64BinaryConsumer.MemoryConstOp(CMP, size, b, (int) value, state));
-                return true;
-            } else {
-                return emitCompareRegMemoryOp(size, asAllocatable(a), b, state);
+            if (a.getConstant() instanceof JavaConstant && a.getJavaConstant().getJavaKind() != JavaKind.Object) {
+                long value = a.getJavaConstant().asLong();
+                if (NumUtil.is32bit(value)) {
+                    append(new AMD64BinaryConsumer.MemoryConstOp(CMP, size, b, (int) value, state));
+                    return true;
+                }
             }
+            return emitCompareRegMemoryOp(size, asAllocatable(a), b, state);
         }
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/LIRKind.java

@@ -421,6 +421,22 @@ public final class LIRKind extends ValueKind<LIRKind> {
         return !isUnknownReference() && (referenceCompressionMask & (1 << idx)) != 0;
     }
 
+    /**
+     * Check whether the given kind is a scalar (i.e., vector length 1) <b>compressed</b> reference.
+     *
+     * @param kind The kind to be checked.
+     * @return true if the given kind is a scalar compressed reference
+     */
+    public static boolean isScalarCompressedReference(ValueKind<?> kind) {
+        if (kind instanceof LIRKind) {
+            LIRKind lirKind = (LIRKind) kind;
+            if (lirKind.getPlatformKind().getVectorLength() == 1 && lirKind.isCompressedReference(0)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     /**
      * Check whether this kind is a value type that doesn't need to be tracked at safepoints.
      */

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/NumUtil.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -105,6 +105,11 @@ public class NumUtil {
         return -0x80000000L <= x && x < 0x80000000L;
     }
 
+    public static byte safeToByte(int v) {
+        assert isByte(v);
+        return (byte) v;
+    }
+
     public static short safeToShort(int v) {
         assert isShort(v);
         return (short) v;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/Stamp.java

@@ -135,6 +135,13 @@ public abstract class Stamp implements SpeculationContextObject {
         return this.equals(this.unrestricted());
     }
 
+    /**
+     * Tests whether this stamp represents a pointer value.
+     */
+    public boolean isPointerStamp() {
+        return this instanceof AbstractPointerStamp;
+    }
+
     /**
      * If this stamp represents a single value, the methods returns this single value. It returns
      * null otherwise.

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/HashMapGetTest.java

@@ -25,25 +25,27 @@
 package org.graalvm.compiler.core.test;
 
 import jdk.vm.ci.meta.ResolvedJavaMethod;
+import org.graalvm.compiler.nodes.FieldLocationIdentity;
 import org.graalvm.compiler.nodes.IfNode;
 import org.graalvm.compiler.nodes.LogicNode;
 import org.graalvm.compiler.nodes.ReturnNode;
 import org.graalvm.compiler.nodes.calc.ObjectEqualsNode;
+import org.graalvm.compiler.nodes.memory.ReadNode;
 import org.junit.Test;
 
+import java.io.IOException;
 import java.util.HashMap;
 
-public class HashMapGetTest extends GraalCompilerTest {
+public class HashMapGetTest extends SubprocessTest {
 
-    public static void mapGet(HashMap<Integer, Integer> map, Integer key) {
+    public static <K, V> void mapGet(HashMap<K, V> map, K key) {
         map.get(key);
     }
 
-    @Test
     public void hashMapTest() {
         HashMap<Integer, Integer> map = new HashMap<>();
         ResolvedJavaMethod get = getResolvedJavaMethod(HashMapGetTest.class, "mapGet");
-        for (int i = 0; i < 5000; i++) {
+        for (int i = 0; i < 10000; i++) {
             mapGet(map, i);
             map.put(i, i);
             mapGet(map, i);
@@ -51,10 +53,18 @@ public class HashMapGetTest extends GraalCompilerTest {
         test(get, null, map, 0);
         for (IfNode ifNode : lastCompiledGraph.getNodes(IfNode.TYPE)) {
             LogicNode condition = ifNode.condition();
-            if (ifNode.getTrueSuccessorProbability() < 0.4 && condition instanceof ObjectEqualsNode) {
-                assertTrue(ifNode.trueSuccessor().next() instanceof ReturnNode, "Expected return but got %s (trueSuccessor: %s)", ifNode.trueSuccessor().next(), ifNode.trueSuccessor());
+            if (ifNode.getTrueSuccessorProbability() < 0.4 && ifNode.predecessor() instanceof ReadNode && condition instanceof ObjectEqualsNode) {
+                ReadNode read = (ReadNode) ifNode.predecessor();
+                if (read.getLocationIdentity() instanceof FieldLocationIdentity && ((FieldLocationIdentity) read.getLocationIdentity()).getField().getName().contains("key")) {
+                    assertTrue(ifNode.trueSuccessor().next() instanceof ReturnNode, "Expected return after %s, got %s", ifNode.trueSuccessor(), ifNode.trueSuccessor().next());
+                }
             }
         }
     }
 
+    @Test
+    public void hashMapTestInSubprocess() throws IOException, InterruptedException {
+        launchSubprocess(this::hashMapTest);
+    }
+
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ObjectSubstitutionsTest.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package org.graalvm.compiler.core.test;
+
+import org.junit.Test;
+
+public class ObjectSubstitutionsTest extends GraalCompilerTest {
+
+    public static int SideEffect;
+
+    public static final void notifySnippet() {
+        synchronized (ObjectSubstitutionsTest.class) {
+            SideEffect = System.identityHashCode(ObjectSubstitutionsTest.class);
+            ObjectSubstitutionsTest.class.notify();
+        }
+    }
+
+    public static final void notifyAllSnippet() {
+        synchronized (ObjectSubstitutionsTest.class) {
+            SideEffect = System.identityHashCode(ObjectSubstitutionsTest.class);
+            ObjectSubstitutionsTest.class.notifyAll();
+        }
+    }
+
+    @Test
+    public void testNotifyEmpty() {
+        test("notifySnippet");
+    }
+
+    @Test
+    public void testNotifyAllEmpty() {
+        test("notifyAllSnippet");
+    }
+
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/DebugContext.java

@@ -1841,6 +1841,25 @@ public final class DebugContext implements AutoCloseable {
         return createTimer(format, arg1, arg2);
     }
 
+    /**
+     * Gets the name to use for a class based on whether it appears to be an obfuscated name. The
+     * heuristic for an obfuscated name is that it is less than 6 characters in length and consists
+     * only of lower case letters.
+     */
+    private static String getBaseName(Class<?> c) {
+        String simpleName = c.getSimpleName();
+        if (simpleName.length() < 6) {
+            for (int i = 0; i < simpleName.length(); i++) {
+                if (!Character.isLowerCase(simpleName.charAt(0))) {
+                    return simpleName;
+                }
+            }
+            // Looks like an obfuscated simple class name so use qualified class name
+            return c.getName();
+        }
+        return simpleName;
+    }
+
     /**
      * There are paths where construction of formatted class names are common and the code below is
      * surprisingly expensive, so compute it once and cache it.
@@ -1848,17 +1867,21 @@ public final class DebugContext implements AutoCloseable {
     private static final ClassValue<String> formattedClassName = new ClassValue<String>() {
         @Override
         protected String computeValue(Class<?> c) {
-            final String simpleName = c.getSimpleName();
+            String baseName = getBaseName(c);
+            if (Character.isLowerCase(baseName.charAt(0))) {
+                // Looks like an obfuscated simple class name so use qualified class name
+                baseName = c.getName();
+            }
             Class<?> enclosingClass = c.getEnclosingClass();
             if (enclosingClass != null) {
                 String prefix = "";
                 while (enclosingClass != null) {
-                    prefix = enclosingClass.getSimpleName() + "_" + prefix;
+                    prefix = getBaseName(enclosingClass) + "_" + prefix;
                     enclosingClass = enclosingClass.getEnclosingClass();
                 }
-                return prefix + simpleName;
+                return prefix + baseName;
             } else {
-                return simpleName;
+                return baseName;
             }
         }
     };

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/ScopeImpl.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -306,7 +306,11 @@ public final class ScopeImpl implements DebugContext.Scope {
                 assert owner.lastClosedScope instanceof DisabledScope : owner.lastClosedScope;
             }
         } catch (Throwable t) {
-            t.initCause(e);
+            if (t != e && t.getCause() == null) {
+                // This mitigates the chance of `e` being swallowed/lost in
+                // the case there's an error in the above handling of `e`.
+                t.initCause(e);
+            }
             throw t;
         }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotBackendFactory.java

@@ -141,7 +141,7 @@ public class AArch64HotSpotBackendFactory extends HotSpotBackendFactory {
                 stampProvider = createStampProvider();
             }
             try (InitTimer rt = timer("create GC provider")) {
-                gc = createGCProvider(config);
+                gc = createGCProvider(config, metaAccess);
             }
 
             Providers p = new Providers(metaAccess, codeCache, constantReflection, constantFieldProvider, foreignCalls, lowerer, null, stampProvider, gc);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/NumberOfTrailingZeroings003.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package org.graalvm.compiler.hotspot.amd64.test;
+
+import org.graalvm.compiler.core.test.GraalCompilerTest;
+import org.junit.Test;
+
+public class NumberOfTrailingZeroings003 extends GraalCompilerTest {
+
+    public static boolean numberOfTrailingZeros0() {
+        return Integer.numberOfTrailingZeros(0) == 32;
+    }
+
+    @Test
+    public void testNumberOfTrailingZeros0() {
+        test("numberOfTrailingZeros0");
+    }
+
+    public static boolean numberOfTrailingZeros1() {
+        return Integer.numberOfTrailingZeros(1) == 0;
+    }
+
+    @Test
+    public void testNumberOfTrailingZeros1() {
+        test("numberOfTrailingZeros1");
+    }
+
+    public static boolean numberOfTrailingZerosM1() {
+        return Integer.numberOfTrailingZeros(-1) == 0;
+    }
+
+    @Test
+    public void testNumberOfTrailingZerosM1() {
+        test("numberOfTrailingZerosM1");
+    }
+
+    public static boolean numberOfTrailingZerosMin() {
+        return Integer.numberOfTrailingZeros(Integer.MIN_VALUE) == 31;
+    }
+
+    @Test
+    public void testNumberOfTrailingZerosMin() {
+        test("numberOfTrailingZerosMin");
+    }
+
+    public static boolean numberOfTrailingZerosMax() {
+        return Integer.numberOfTrailingZeros(Integer.MAX_VALUE) == 0;
+    }
+
+    @Test
+    public void testNumberOfTrailingZerosMax() {
+        test("numberOfTrailingZerosMax");
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotAddressLowering.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -118,6 +118,7 @@ public class AMD64HotSpotAddressLowering extends AMD64CompressAddressLowering {
         }
 
         addr.setScale(scale);
+        addr.setUncompressionScale(scale);
         addr.setIndex(compression.getValue());
         return true;
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackendFactory.java

@@ -133,7 +133,7 @@ public class AMD64HotSpotBackendFactory extends HotSpotBackendFactory {
                 stampProvider = createStampProvider();
             }
             try (InitTimer rt = timer("create GC provider")) {
-                gc = createGCProvider(config);
+                gc = createGCProvider(config, metaAccess);
             }
 
             Providers p = new Providers(metaAccess, codeCache, constantReflection, constantFieldProvider, foreignCalls, lowerer, null, stampProvider, gc);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotBackendFactory.java

@@ -101,7 +101,7 @@ public class SPARCHotSpotBackendFactory extends HotSpotBackendFactory {
         HotSpotForeignCallsProvider foreignCalls = new SPARCHotSpotForeignCallsProvider(jvmciRuntime, runtime, metaAccess, codeCache, wordTypes, nativeABICallerSaveRegisters);
         LoweringProvider lowerer = createLowerer(runtime, metaAccess, foreignCalls, registers, constantReflection, target);
         HotSpotStampProvider stampProvider = createStampProvider();
-        HotSpotGCProvider gc = createGCProvider(config);
+        HotSpotGCProvider gc = createGCProvider(config, metaAccess);
         Providers p = new Providers(metaAccess, codeCache, constantReflection, constantFieldProvider, foreignCalls, lowerer, null, stampProvider, gc);
         HotSpotSnippetReflectionProvider snippetReflection = createSnippetReflection(runtime, constantReflection, wordTypes);
         BytecodeProvider bytecodeProvider = createBytecodeProvider(metaAccess, snippetReflection);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CompilationWrapperTest.java

@@ -249,7 +249,7 @@ public class CompilationWrapperTest extends GraalCompilerTest {
             for (Probe probe : probes) {
                 String error = probe.test();
                 if (error != null) {
-                    Assert.fail(String.format("Did not find expected occurences of '%s' in output of command: %s%n%s", probe.substring, error, proc));
+                    Assert.fail(String.format("Did not find expected occurrences of '%s' in output of command: %s%n%s", probe.substring, error, proc));
                 }
             }
             String line = diagnosticProbe.lastMatchingLine;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CompileTheWorld.java

@@ -1092,7 +1092,8 @@ public final class CompileTheWorld {
                                    "'PartialEscapeAnalysis=false PrintCompilation=true'. " +
                                    "Unless explicitly enabled with 'Inline=true' here, inlining is disabled.",
                   "MultiThreaded", "Run using multiple threads for compilation.",
-                        "Threads", "Number of threads to use for multithreaded execution. Defaults to Runtime.getRuntime().availableProcessors().");
+                        "Threads", "Number of threads to use for multithreaded execution. Defaults to Runtime.getRuntime().availableProcessors().",
+        "InvalidateInstalledCode", "Invalidate the generated code so the code cache doesn't fill up.");
         // @formatter:on
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/MethodSubstitutionEffectTest.java

@@ -0,0 +1,391 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package org.graalvm.compiler.hotspot.test;
+
+import static org.graalvm.compiler.core.common.CompilationIdentifier.INVALID_COMPILATION_ID;
+import static org.graalvm.compiler.debug.DebugOptions.DumpOnError;
+
+import java.util.ArrayList;
+
+import org.graalvm.compiler.api.directives.GraalDirectives;
+import org.graalvm.compiler.api.replacements.ClassSubstitution;
+import org.graalvm.compiler.api.replacements.MethodSubstitution;
+import org.graalvm.compiler.api.test.Graal;
+import org.graalvm.compiler.core.test.GraalCompilerTest;
+import org.graalvm.compiler.debug.GraalError;
+import org.graalvm.compiler.debug.TTY;
+import org.graalvm.compiler.java.BytecodeParserOptions;
+import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions;
+import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins;
+import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins.Registration;
+import org.graalvm.compiler.options.OptionValues;
+import org.graalvm.compiler.phases.tiers.Suites;
+import org.graalvm.compiler.replacements.ReplacementsImpl;
+import org.graalvm.compiler.replacements.classfile.ClassfileBytecodeProvider;
+import org.graalvm.compiler.runtime.RuntimeProvider;
+import org.junit.Assert;
+import org.junit.Test;
+
+import jdk.vm.ci.meta.MetaAccessProvider;
+import jdk.vm.ci.meta.ResolvedJavaMethod;
+
+public class MethodSubstitutionEffectTest extends GraalCompilerTest {
+    public static int ValueFountain;
+
+    static class Substitutee {
+
+        public static int singleEffect(@SuppressWarnings("unused") int a) {
+            return 0;
+        }
+
+        public static int sequentialEffectInvalid(@SuppressWarnings("unused") int a) {
+            return 0;
+        }
+
+        public static void sequentialEffectInvalidVoid(@SuppressWarnings("unused") int a) {
+        }
+
+        public static int splitEffect(@SuppressWarnings("unused") int a) {
+            return 0;
+        }
+
+        public static void splitEffectVoid(@SuppressWarnings("unused") int a) {
+        }
+
+        public static int multiSplitEffectNoMerge(@SuppressWarnings("unused") int a) {
+            return 0;
+        }
+
+        public static int multiSplitEffectNoMergeInvalid(@SuppressWarnings("unused") int a) {
+            return 0;
+        }
+
+        public static int splitEffectWrong(@SuppressWarnings("unused") int a) {
+            return 0;
+        }
+
+        public static int splitParitalIntrinsicExit(@SuppressWarnings("unused") int a) {
+            return 0;
+        }
+    }
+
+    @ClassSubstitution(Substitutee.class)
+    public static class Substitutor {
+
+        @MethodSubstitution
+        public static int singleEffect(int a) {
+            return GraalDirectives.sideEffect(a);
+        }
+
+        @MethodSubstitution
+        public static int sequentialEffectInvalid(int a) {
+            GraalDirectives.sideEffect(a);
+            return GraalDirectives.sideEffect(a);
+        }
+
+        @MethodSubstitution
+        public static void sequentialEffectInvalidVoid(int a) {
+            GraalDirectives.sideEffect(a);
+            GraalDirectives.sideEffect(a);
+        }
+
+        @MethodSubstitution
+        public static int splitEffect(int a) {
+            int i;
+            if (a > 0) {
+                GraalDirectives.sideEffect(a);
+                i = a;
+            } else {
+                GraalDirectives.sideEffect(42);
+                i = 42;
+            }
+            return i;
+        }
+
+        @MethodSubstitution
+        public static void splitEffectVoid(int a) {
+            if (a > 0) {
+                GraalDirectives.sideEffect(a);
+            } else {
+                GraalDirectives.sideEffect(42);
+            }
+        }
+
+        @MethodSubstitution
+        public static int multiSplitEffectNoMerge(int a) {
+            switch (a) {
+                case 1:
+                    GraalDirectives.sideEffect(a);
+                    return 3;
+                case 2:
+                    GraalDirectives.sideEffect(a);
+                    return 2;
+                case 3:
+                    GraalDirectives.sideEffect(a);
+                    return 1;
+                default:
+                    GraalDirectives.sideEffect(a);
+                    return 0;
+            }
+        }
+
+        @MethodSubstitution
+        public static int multiSplitEffectNoMergeInvalid(int a) {
+            switch (a) {
+                case 1:
+                    GraalDirectives.sideEffect(a);
+                    return 3;
+                case 2:
+                    GraalDirectives.sideEffect(a);
+                    return 2;
+                case 3:
+                    GraalDirectives.sideEffect(a);
+                    return 1;
+                default:
+                    GraalDirectives.sideEffect(a);
+                    GraalDirectives.sideEffect(a);
+                    return 0;
+            }
+        }
+
+        @MethodSubstitution
+        public static int splitEffectWrong(int a) {
+            int i;
+            if (a > 0) {
+                GraalDirectives.sideEffect(a);
+                GraalDirectives.sideEffect(a);
+                i = a;
+            } else {
+                i = 42;
+                GraalDirectives.controlFlowAnchor();
+            }
+            return i;
+        }
+
+        @MethodSubstitution
+        public static int splitParitalIntrinsicExit(int a) {
+            int i;
+            if (a > 0) {
+                i = GraalDirectives.sideEffect(a);
+            } else {
+                i = splitParitalIntrinsicExit(a);
+            }
+            return i;
+        }
+    }
+
+    @Override
+    protected void registerInvocationPlugins(InvocationPlugins invocationPlugins) {
+        ClassfileBytecodeProvider bytecodeProvider = getSystemClassLoaderBytecodeProvider();
+        Registration r = new Registration(invocationPlugins, Substitutee.class, getReplacements(), bytecodeProvider);
+        r.registerMethodSubstitution(Substitutor.class, "singleEffect", int.class);
+        r.registerMethodSubstitution(Substitutor.class, "sequentialEffectInvalid", int.class);
+        r.registerMethodSubstitution(Substitutor.class, "sequentialEffectInvalidVoid", int.class);
+        r.registerMethodSubstitution(Substitutor.class, "splitEffect", int.class);
+        r.registerMethodSubstitution(Substitutor.class, "splitEffectVoid", int.class);
+        r.registerMethodSubstitution(Substitutor.class, "multiSplitEffectNoMerge", int.class);
+        r.registerMethodSubstitution(Substitutor.class, "multiSplitEffectNoMergeInvalid", int.class);
+        r.registerMethodSubstitution(Substitutor.class, "splitEffectWrong", int.class);
+        r.registerMethodSubstitution(Substitutor.class, "splitParitalIntrinsicExit", int.class);
+        super.registerInvocationPlugins(invocationPlugins);
+    }
+
+    private ClassfileBytecodeProvider getSystemClassLoaderBytecodeProvider() {
+        ReplacementsImpl d = (ReplacementsImpl) getReplacements();
+        MetaAccessProvider metaAccess = d.getProviders().getMetaAccess();
+        ClassfileBytecodeProvider bytecodeProvider = new ClassfileBytecodeProvider(metaAccess, d.snippetReflection, ClassLoader.getSystemClassLoader());
+        return bytecodeProvider;
+    }
+
+    static void snippet01() {
+        Substitutee.singleEffect(42);
+        if (ValueFountain == 42) {
+            GraalDirectives.deoptimize();
+        }
+    }
+
+    static void snippet02() {
+        Substitutee.sequentialEffectInvalid(42);
+        if (ValueFountain == 42) {
+            GraalDirectives.deoptimize();
+        }
+    }
+
+    static void snippet03() {
+        Substitutee.sequentialEffectInvalidVoid(42);
+        if (ValueFountain == 42) {
+            GraalDirectives.deoptimize();
+        }
+    }
+
+    static void snippet04() {
+        Substitutee.splitEffect(ValueFountain);
+        if (ValueFountain == 42) {
+            GraalDirectives.deoptimize();
+        }
+    }
+
+    static void snippet05() {
+        Substitutee.splitEffectVoid(ValueFountain);
+        if (ValueFountain == 42) {
+            GraalDirectives.deoptimize();
+        }
+    }
+
+    static void snippet06() {
+        Substitutee.splitEffectWrong(ValueFountain);
+        if (ValueFountain == 42) {
+            GraalDirectives.deoptimize();
+        }
+    }
+
+    static void snippet07() {
+        if (Substitutee.splitParitalIntrinsicExit(ValueFountain) == 42) {
+            GraalDirectives.deoptimize();
+        }
+    }
+
+    static void snippet08() {
+        Substitutee.multiSplitEffectNoMerge(ValueFountain);
+    }
+
+    StructuredGraph getGraph(String snippet) {
+        OptionValues options = new OptionValues(getInitialOptions(), DumpOnError, false);
+        /*
+         * We do not want to inline partial intrinsic exits in this test to test the state of the
+         * self recursive call.
+         */
+        options = new OptionValues(getInitialOptions(), BytecodeParserOptions.InlinePartialIntrinsicExitDuringParsing, false);
+        StructuredGraph g = parseEager(getResolvedJavaMethod(snippet), AllowAssumptions.NO, options);
+        Suites s = Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend().getSuites().getDefaultSuites(getInitialOptions());
+        s.getHighTier().apply(g, getDefaultHighTierContext());
+        s.getMidTier().apply(g, getDefaultMidTierContext());
+        return g;
+    }
+
+    @Test
+    public void test1() {
+        getGraph("snippet01");
+    }
+
+    @Test
+    @SuppressWarnings("try")
+    public void test2() {
+        try (AutoCloseable c = new TTY.Filter()) {
+            getGraph("snippet02");
+            Assert.fail("Compilation should not reach this point, must throw an exception before");
+        } catch (Throwable t) {
+            if (t.getCause() instanceof GraalError && t.getMessage().contains("unexpected node between return StateSplit and last instruction")) {
+                return;
+            }
+            throw new AssertionError(t);
+        }
+    }
+
+    @Test
+    @SuppressWarnings("try")
+    public void test3() {
+        try (AutoCloseable c = new TTY.Filter()) {
+            getGraph("snippet03");
+            Assert.fail("Compilation should not reach this point, must throw an exception before");
+        } catch (Throwable t) {
+            if (t.getCause() instanceof GraalError && t.getMessage().contains(" produced invalid framestate")) {
+                return;
+            }
+            throw new AssertionError(t);
+        }
+    }
+
+    @Test
+    public void test4() {
+        getGraph("snippet04");
+    }
+
+    @Test
+    public void test5() {
+        getGraph("snippet05");
+    }
+
+    @Test
+    @SuppressWarnings("try")
+    public void test6() {
+        try (AutoCloseable c = new TTY.Filter()) {
+            getGraph("snippet06");
+            Assert.fail("Compilation should not reach this point, must throw an exception before");
+        } catch (Throwable t) {
+            if (t.getCause() instanceof GraalError && t.getMessage().contains(" produced invalid framestate")) {
+                return;
+            }
+            throw new AssertionError(t);
+        }
+    }
+
+    @Test
+    public void test7() {
+        getGraph("snippet07");
+    }
+
+    @Test
+    public void test8() {
+        getGraph("snippet08");
+    }
+
+    @Test
+    @SuppressWarnings("try")
+    public void testRootCompiles() {
+        ArrayList<ResolvedJavaMethod> intrinisicsWithoutErrors = new ArrayList<>();
+        ArrayList<ResolvedJavaMethod> intrinisicsErrors = new ArrayList<>();
+
+        intrinisicsWithoutErrors.add(getResolvedJavaMethod(Substitutee.class, "singleEffect"));
+        intrinisicsWithoutErrors.add(getResolvedJavaMethod(Substitutee.class, "splitEffect"));
+        intrinisicsWithoutErrors.add(getResolvedJavaMethod(Substitutee.class, "splitEffectVoid"));
+        intrinisicsWithoutErrors.add(getResolvedJavaMethod(Substitutee.class, "multiSplitEffectNoMerge"));
+        intrinisicsWithoutErrors.add(getResolvedJavaMethod(Substitutee.class, "splitParitalIntrinsicExit"));
+
+        intrinisicsErrors.add(getResolvedJavaMethod(Substitutee.class, "sequentialEffectInvalid"));
+        intrinisicsErrors.add(getResolvedJavaMethod(Substitutee.class, "sequentialEffectInvalidVoid"));
+        intrinisicsErrors.add(getResolvedJavaMethod(Substitutee.class, "splitEffectWrong"));
+        intrinisicsErrors.add(getResolvedJavaMethod(Substitutee.class, "multiSplitEffectNoMergeInvalid"));
+
+        for (ResolvedJavaMethod method : intrinisicsWithoutErrors) {
+            StructuredGraph graph = getProviders().getReplacements().getIntrinsicGraph(method, INVALID_COMPILATION_ID, getDebugContext(), null);
+            getCode(method, graph);
+        }
+        for (ResolvedJavaMethod method : intrinisicsErrors) {
+            try (AutoCloseable c = new TTY.Filter()) {
+                StructuredGraph graph = getProviders().getReplacements().getIntrinsicGraph(method, INVALID_COMPILATION_ID, getDebugContext(), null);
+                getCode(method, graph);
+                Assert.fail("Compilation should not reach this point, must throw an exception before");
+            } catch (Throwable t) {
+                if ((t.getCause() instanceof GraalError || t instanceof GraalError) && t.getMessage().contains("invalid state")) {
+                    continue;
+                }
+                throw new AssertionError(t);
+            }
+        }
+
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/MethodSubstitutionForeignCallTest.java

@@ -0,0 +1,396 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package org.graalvm.compiler.hotspot.test;
+
+import static org.graalvm.compiler.debug.DebugOptions.DumpOnError;
+
+import org.graalvm.compiler.api.directives.GraalDirectives;
+import org.graalvm.compiler.api.replacements.ClassSubstitution;
+import org.graalvm.compiler.api.replacements.MethodSubstitution;
+import org.graalvm.compiler.api.test.Graal;
+import org.graalvm.compiler.core.common.LIRKind;
+import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
+import org.graalvm.compiler.core.common.spi.ForeignCallLinkage;
+import org.graalvm.compiler.core.common.spi.ForeignCallsProvider;
+import org.graalvm.compiler.core.test.GraalCompilerTest;
+import org.graalvm.compiler.debug.GraalError;
+import org.graalvm.compiler.debug.TTY;
+import org.graalvm.compiler.hotspot.nodes.CurrentJavaThreadNode;
+import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions;
+import org.graalvm.compiler.nodes.ValueNode;
+import org.graalvm.compiler.nodes.extended.ForeignCallNode;
+import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext;
+import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin;
+import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins;
+import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins.Registration;
+import org.graalvm.compiler.options.OptionValues;
+import org.graalvm.compiler.phases.tiers.Suites;
+import org.graalvm.compiler.replacements.ReplacementsImpl;
+import org.graalvm.compiler.replacements.classfile.ClassfileBytecodeProvider;
+import org.graalvm.compiler.runtime.RuntimeProvider;
+import jdk.internal.vm.compiler.word.LocationIdentity;
+import org.junit.Assert;
+import org.junit.Test;
+
+import jdk.vm.ci.meta.JavaKind;
+import jdk.vm.ci.meta.MetaAccessProvider;
+import jdk.vm.ci.meta.ResolvedJavaMethod;
+
+public class MethodSubstitutionForeignCallTest extends GraalCompilerTest {
+    public static final ForeignCallDescriptor TEST_CALL = new ForeignCallDescriptor("test", int.class, int.class);
+
+    public static class A {
+        static void invalidConsecutiveForeignCall1(@SuppressWarnings("unused") int phi) {
+
+        }
+
+        static void invalidConsecutiveForeignCall2(@SuppressWarnings("unused") int phi) {
+
+        }
+
+        static void validConsecutiveForeignCallReexecutable(@SuppressWarnings("unused") int phi) {
+
+        }
+
+        static void splitForeignCallInvalid(@SuppressWarnings("unused") int phi) {
+
+        }
+    }
+
+    @ClassSubstitution(A.class)
+    public static class ASubstitutions {
+
+        /*
+         * Invalid: two consecutive states, deopt could float in between.
+         */
+        @MethodSubstitution
+        static void invalidConsecutiveForeignCall1(int phi) {
+            testDeopt(phi);
+            // invalid two consecutive calls
+            testDeopt(phi);
+        }
+
+        /*
+         * Invalid: two consecutive states, deopt could float in between. Same applies for
+         * non-deopting framestates if they are not re-executable. If they are, we are good.
+         */
+        @MethodSubstitution
+        static void invalidConsecutiveForeignCall2(int phi) {
+            testNonDeopting(phi);
+            testNonDeopting(phi);
+        }
+
+        /*
+         * Valid, the foreign calls are re-executable and non-deopting (thus completely side-effect
+         * free), they do not need a state.
+         */
+        @MethodSubstitution
+        static void validConsecutiveForeignCallReexecutable(int phi) {
+            testPureReexectuable(phi);
+            testPureReexectuable(phi);
+        }
+
+        /**
+         * Invalid: Splitting effect in a method substitution is allowed as long as it is just one
+         * effect per call. This is not the case here.
+         */
+        @MethodSubstitution
+        static void splitForeignCallInvalid(int phi) {
+            if (SideEffect == 0) {
+                testDeopt(phi);
+            } else {
+                CurrentJavaThreadNode.get().writeByte(0, (byte) 0);
+                testDeopt(phi);
+            }
+        }
+    }
+
+    @Override
+    protected void registerInvocationPlugins(InvocationPlugins invocationPlugins) {
+
+        invocationPlugins.register(new InvocationPlugin() {
+
+            @Override
+            public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode arg) {
+                ForeignCallsProvider foreignCalls = new ForeignCallsProvider() {
+
+                    @Override
+                    public LIRKind getValueKind(JavaKind javaKind) {
+                        return LIRKind.fromJavaKind(getTarget().arch, javaKind);
+                    }
+
+                    @Override
+                    public ForeignCallLinkage lookupForeignCall(ForeignCallDescriptor descriptor) {
+                        throw GraalError.shouldNotReachHere("Test code must not need this method");
+                    }
+
+                    @Override
+                    public boolean isReexecutable(ForeignCallDescriptor descriptor) {
+                        return false;
+                    }
+
+                    @Override
+                    public boolean isGuaranteedSafepoint(ForeignCallDescriptor descriptor) {
+                        return true;
+                    }
+
+                    @Override
+                    public boolean isAvailable(ForeignCallDescriptor descriptor) {
+                        return true;
+                    }
+
+                    @Override
+                    public LocationIdentity[] getKilledLocations(ForeignCallDescriptor descriptor) {
+                        return new LocationIdentity[]{LocationIdentity.any()};
+                    }
+
+                    @Override
+                    public boolean canDeoptimize(ForeignCallDescriptor descriptor) {
+                        return true;
+                    }
+                };
+                ForeignCallNode node = new ForeignCallNode(foreignCalls, TEST_CALL, arg);
+                node.setBci(b.bci());
+                b.addPush(JavaKind.Int, node);
+                return true;
+            }
+        }, MethodSubstitutionForeignCallTest.class, "testDeopt", int.class);
+        invocationPlugins.register(new InvocationPlugin() {
+
+            @Override
+            public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode arg) {
+                ForeignCallsProvider foreignCalls = new ForeignCallsProvider() {
+
+                    @Override
+                    public LIRKind getValueKind(JavaKind javaKind) {
+                        return LIRKind.fromJavaKind(getTarget().arch, javaKind);
+                    }
+
+                    @Override
+                    public ForeignCallLinkage lookupForeignCall(ForeignCallDescriptor descriptor) {
+                        throw GraalError.shouldNotReachHere("Test code must not need this method");
+                    }
+
+                    @Override
+                    public boolean isReexecutable(ForeignCallDescriptor descriptor) {
+                        return false;
+                    }
+
+                    @Override
+                    public boolean isGuaranteedSafepoint(ForeignCallDescriptor descriptor) {
+                        return false;
+                    }
+
+                    @Override
+                    public boolean isAvailable(ForeignCallDescriptor descriptor) {
+                        return true;
+                    }
+
+                    @Override
+                    public LocationIdentity[] getKilledLocations(ForeignCallDescriptor descriptor) {
+                        return new LocationIdentity[]{LocationIdentity.any()};
+                    }
+
+                    @Override
+                    public boolean canDeoptimize(ForeignCallDescriptor descriptor) {
+                        return false;
+                    }
+                };
+                ForeignCallNode node = new ForeignCallNode(foreignCalls, TEST_CALL, arg);
+                node.setBci(b.bci());
+                b.addPush(JavaKind.Int, node);
+                return true;
+            }
+        }, MethodSubstitutionForeignCallTest.class, "testNonDeopting", int.class);
+        invocationPlugins.register(new InvocationPlugin() {
+
+            @Override
+            public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode arg) {
+                ForeignCallsProvider foreignCalls = new ForeignCallsProvider() {
+
+                    @Override
+                    public LIRKind getValueKind(JavaKind javaKind) {
+                        return LIRKind.fromJavaKind(getTarget().arch, javaKind);
+                    }
+
+                    @Override
+                    public ForeignCallLinkage lookupForeignCall(ForeignCallDescriptor descriptor) {
+                        throw GraalError.shouldNotReachHere("Test code must not need this method");
+                    }
+
+                    @Override
+                    public boolean isReexecutable(ForeignCallDescriptor descriptor) {
+                        return true;
+                    }
+
+                    @Override
+                    public boolean isGuaranteedSafepoint(ForeignCallDescriptor descriptor) {
+                        return false;
+                    }
+
+                    @Override
+                    public boolean isAvailable(ForeignCallDescriptor descriptor) {
+                        return true;
+                    }
+
+                    @Override
+                    public LocationIdentity[] getKilledLocations(ForeignCallDescriptor descriptor) {
+                        return new LocationIdentity[]{LocationIdentity.any()};
+                    }
+
+                    @Override
+                    public boolean canDeoptimize(ForeignCallDescriptor descriptor) {
+                        return false;
+                    }
+                };
+                ForeignCallNode node = new ForeignCallNode(foreignCalls, TEST_CALL, arg);
+                node.setBci(b.bci());
+                b.addPush(JavaKind.Int, node);
+                return true;
+            }
+        }, MethodSubstitutionForeignCallTest.class, "testPureReexectuable", int.class);
+        ClassfileBytecodeProvider bytecodeProvider = getSystemClassLoaderBytecodeProvider();
+        Registration r = new Registration(invocationPlugins, A.class, getReplacements(), bytecodeProvider);
+        r.registerMethodSubstitution(ASubstitutions.class, "invalidConsecutiveForeignCall1", int.class);
+        r.registerMethodSubstitution(ASubstitutions.class, "invalidConsecutiveForeignCall2", int.class);
+        r.registerMethodSubstitution(ASubstitutions.class, "validConsecutiveForeignCallReexecutable", int.class);
+        r.registerMethodSubstitution(ASubstitutions.class, "splitForeignCallInvalid", int.class);
+        super.registerInvocationPlugins(invocationPlugins);
+    }
+
+    private ClassfileBytecodeProvider getSystemClassLoaderBytecodeProvider() {
+        ReplacementsImpl d = (ReplacementsImpl) getReplacements();
+        MetaAccessProvider metaAccess = d.getProviders().getMetaAccess();
+        ClassfileBytecodeProvider bytecodeProvider = new ClassfileBytecodeProvider(metaAccess, d.snippetReflection, ClassLoader.getSystemClassLoader());
+        return bytecodeProvider;
+    }
+
+    public static int SideEffect;
+
+    public static int testDeopt(int value) {
+        SideEffect = value;
+        return value;
+    }
+
+    public static int testNonDeopting(int value) {
+        return value;
+    }
+
+    public static int testPureReexectuable(int value) {
+        return value;
+    }
+
+    public static void testSnippetInvalidSequential() {
+        A.invalidConsecutiveForeignCall1(SideEffect);
+        if (SideEffect == 1) {
+            GraalDirectives.deoptimize();
+        }
+    }
+
+    public static void testNonDeoptingInvalid() {
+        A.invalidConsecutiveForeignCall2(SideEffect);
+        if (SideEffect == 1) {
+            GraalDirectives.deoptimize();
+        }
+    }
+
+    public static void testNonDeoptingSplit() {
+        A.splitForeignCallInvalid(SideEffect);
+        if (SideEffect == 1) {
+            GraalDirectives.deoptimize();
+        }
+    }
+
+    public static void testNonDeoptingReexectuable() {
+        A.validConsecutiveForeignCallReexecutable(SideEffect);
+        if (SideEffect == 1) {
+            GraalDirectives.deoptimize();
+        }
+    }
+
+    @Test
+    @SuppressWarnings("try")
+    public void test1() {
+        try (AutoCloseable c = new TTY.Filter()) {
+            OptionValues options = new OptionValues(getInitialOptions(), DumpOnError, false);
+            StructuredGraph g = parseEager(getResolvedJavaMethod("testSnippetInvalidSequential"), AllowAssumptions.NO, options);
+            Suites s = Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend().getSuites().getDefaultSuites(getInitialOptions());
+            s.getHighTier().apply(g, getDefaultHighTierContext());
+            s.getMidTier().apply(g, getDefaultMidTierContext());
+            Assert.fail("Compilation should not reach this point, must throw an exception before");
+        } catch (Throwable t) {
+            if ((t.getCause() instanceof GraalError || t instanceof GraalError) && t.getMessage().contains("invalid framestate")) {
+                return;
+            }
+            throw new AssertionError(t);
+        }
+    }
+
+    @Test
+    @SuppressWarnings("try")
+    public void test2() {
+        try (AutoCloseable c = new TTY.Filter()) {
+            OptionValues options = new OptionValues(getInitialOptions(), DumpOnError, false);
+            StructuredGraph g = parseEager(getResolvedJavaMethod("testSnippetInvalidSequential"), AllowAssumptions.NO, options);
+            Suites s = Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend().getSuites().getDefaultSuites(getInitialOptions());
+            s.getHighTier().apply(g, getDefaultHighTierContext());
+            s.getMidTier().apply(g, getDefaultMidTierContext());
+            Assert.fail("Compilation should not reach this point, must throw an exception before");
+        } catch (Throwable t) {
+            if ((t.getCause() instanceof GraalError || t instanceof GraalError) && t.getMessage().contains("invalid framestate")) {
+                return;
+            }
+            throw new AssertionError(t);
+        }
+    }
+
+    @Test
+    @SuppressWarnings("try")
+    public void test3() {
+        try (AutoCloseable c = new TTY.Filter()) {
+            OptionValues options = new OptionValues(getInitialOptions(), DumpOnError, false);
+            StructuredGraph g = parseEager(getResolvedJavaMethod("testNonDeoptingSplit"), AllowAssumptions.NO, options);
+            Suites s = Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend().getSuites().getDefaultSuites(getInitialOptions());
+            s.getHighTier().apply(g, getDefaultHighTierContext());
+            s.getMidTier().apply(g, getDefaultMidTierContext());
+            Assert.fail("Compilation should not reach this point, must throw an exception before");
+        } catch (Throwable t) {
+            if ((t.getCause() instanceof GraalError || t instanceof GraalError) && t.getMessage().contains("invalid framestate")) {
+                return;
+            }
+            throw new AssertionError(t);
+        }
+    }
+
+    @Test
+    public void test4() {
+        StructuredGraph g = parseEager(getResolvedJavaMethod("testNonDeoptingReexectuable"), AllowAssumptions.NO);
+        Suites s = Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend().getSuites().getDefaultSuites(getInitialOptions());
+        s.getHighTier().apply(g, getDefaultHighTierContext());
+        s.getMidTier().apply(g, getDefaultMidTierContext());
+    }
+
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/WriteBarrierAdditionTest.java

@@ -24,83 +24,117 @@
 
 package org.graalvm.compiler.hotspot.test;
 
+import static org.graalvm.compiler.core.common.GraalOptions.FullUnroll;
+import static org.graalvm.compiler.core.common.GraalOptions.LoopPeeling;
+import static org.graalvm.compiler.core.common.GraalOptions.PartialEscapeAnalysis;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.referentOffset;
 
+import java.lang.ref.Reference;
 import java.lang.ref.WeakReference;
+import java.util.EnumSet;
+import java.util.ListIterator;
+import java.util.Objects;
 
 import org.graalvm.compiler.api.test.Graal;
-import org.graalvm.compiler.debug.DebugContext;
 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
 import org.graalvm.compiler.hotspot.HotSpotBackend;
+import org.graalvm.compiler.hotspot.HotSpotGraalRuntime.HotSpotGC;
+import org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil;
+import org.graalvm.compiler.nodeinfo.NodeSize;
 import org.graalvm.compiler.nodes.StructuredGraph;
-import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions;
 import org.graalvm.compiler.nodes.gc.G1PostWriteBarrier;
 import org.graalvm.compiler.nodes.gc.G1PreWriteBarrier;
 import org.graalvm.compiler.nodes.gc.G1ReferentFieldReadBarrier;
 import org.graalvm.compiler.nodes.gc.SerialWriteBarrier;
-import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType;
+import org.graalvm.compiler.nodes.memory.HeapAccess;
 import org.graalvm.compiler.nodes.memory.ReadNode;
 import org.graalvm.compiler.nodes.memory.WriteNode;
 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
-import org.graalvm.compiler.nodes.spi.LoweringTool;
-import org.graalvm.compiler.phases.OptimisticOptimizations;
-import org.graalvm.compiler.phases.common.GuardLoweringPhase;
-import org.graalvm.compiler.phases.common.LoweringPhase;
+import org.graalvm.compiler.options.OptionValues;
+import org.graalvm.compiler.phases.BasePhase;
+import org.graalvm.compiler.phases.Phase;
 import org.graalvm.compiler.phases.common.WriteBarrierAdditionPhase;
-import org.graalvm.compiler.phases.common.inlining.InliningPhase;
-import org.graalvm.compiler.phases.common.inlining.policy.InlineEverythingPolicy;
-import org.graalvm.compiler.phases.tiers.HighTierContext;
 import org.graalvm.compiler.phases.tiers.MidTierContext;
+import org.graalvm.compiler.phases.tiers.Suites;
 import org.graalvm.compiler.runtime.RuntimeProvider;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 
-import jdk.vm.ci.hotspot.HotSpotInstalledCode;
 import jdk.vm.ci.meta.JavaConstant;
-import jdk.vm.ci.meta.ResolvedJavaMethod;
-import sun.misc.Unsafe;
+import jdk.vm.ci.meta.MetaAccessProvider;
 
 /**
- * The following unit tests assert the presence of write barriers for both Serial and G1 GCs.
- * Normally, the tests check for compile time inserted barriers. However, there are the cases of
- * unsafe loads of the java.lang.ref.Reference.referent field where runtime checks have to be
- * performed also. For those cases, the unit tests check the presence of the compile-time inserted
- * barriers. Concerning the runtime checks, the results of variable inputs (object types and
- * offsets) passed as input parameters can be checked against printed output from the G1 write
- * barrier snippets. The runtime checks have been validated offline.
+ * The following unit tests assert the presence of write barriers for G1 and for the other GCs that
+ * use a simple card mark barrier, like Serial, CMS, ParallelGC and Pthe arNew/ParOld GCs. Normally,
+ * the tests check for compile time inserted barriers. However, there are the cases of unsafe loads
+ * of the java.lang.ref.Reference.referent field where runtime checks have to be performed also. For
+ * those cases, the unit tests check the presence of the compile-time inserted barriers. Concerning
+ * the runtime checks, the results of variable inputs (object types and offsets) passed as input
+ * parameters can be checked against printed output from the G1 write barrier snippets. The runtime
+ * checks have been validated offline.
  */
 public class WriteBarrierAdditionTest extends HotSpotGraalCompilerTest {
 
+    /**
+     * The set of GCs known at the time of writing of this test. The number of expected barrier
+     * might need to be adjusted for new GCs implementations.
+     */
+    private static EnumSet<HotSpotGC> knownSupport = EnumSet.of(HotSpotGC.G1, HotSpotGC.CMS, HotSpotGC.Parallel, HotSpotGC.Serial);
+
     private final GraalHotSpotVMConfig config = runtime().getVMConfig();
 
     public static class Container {
 
         public Container a;
         public Container b;
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            Container container = (Container) o;
+            return Objects.equals(a, container.a) && Objects.equals(b, container.b);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(a, b);
+        }
     }
 
+    private int expectedBarriers;
+
     /**
-     * Expected 2 barriers for the Serial GC and 4 for G1 (2 pre + 2 post).
+     * Expected 2 barriers for the card mark GCs and 4 for G1 (2 pre + 2 post).
      */
     @Test
-    public void test1() throws Exception {
-        testHelper("test1Snippet", (config.useG1GC) ? 4 : 2);
+    public void testAllocation() throws Exception {
+        this.expectedBarriers = (config.useG1GC) ? 4 : 2;
+        testWithoutPEA("testAllocationSnippet");
     }
 
-    public static void test1Snippet() {
+    public static Container testAllocationSnippet() {
         Container main = new Container();
         Container temp1 = new Container();
         Container temp2 = new Container();
         main.a = temp1;
         main.b = temp2;
+        return main;
     }
 
     /**
-     * Expected 4 barriers for the Serial GC and 8 for G1 (4 pre + 4 post).
+     * Expected 4 barriers for the card mark GCs and 8 for G1 (4 pre + 4 post).
      */
     @Test
-    public void test2() throws Exception {
-        testHelper("test2Snippet", config.useG1GC ? 8 : 4);
+    public void testLoopAllocation1() throws Exception {
+        this.expectedBarriers = config.useG1GC ? 8 : 4;
+        testWithoutPEA("test2Snippet", false);
+        testWithoutPEA("test2Snippet", true);
     }
 
     public static void test2Snippet(boolean test) {
@@ -119,11 +153,12 @@ public class WriteBarrierAdditionTest extends HotSpotGraalCompilerTest {
     }
 
     /**
-     * Expected 4 barriers for the Serial GC and 8 for G1 (4 pre + 4 post).
+     * Expected 4 barriers for the card mark GCs and 8 for G1 (4 pre + 4 post).
      */
     @Test
-    public void test3() throws Exception {
-        testHelper("test3Snippet", config.useG1GC ? 8 : 4);
+    public void testLoopAllocation2() throws Exception {
+        this.expectedBarriers = config.useG1GC ? 8 : 4;
+        testWithoutPEA("test3Snippet");
     }
 
     public static void test3Snippet() {
@@ -140,84 +175,109 @@ public class WriteBarrierAdditionTest extends HotSpotGraalCompilerTest {
     }
 
     /**
-     * Expected 2 barriers for the Serial GC and 5 for G1 (3 pre + 2 post) The (2 or 4) barriers are
-     * emitted while initializing the fields of the WeakReference instance. The extra pre barrier of
-     * G1 concerns the read of the referent field.
+     * Expected 2 barriers for the card mark GCs and 5 for G1 (3 pre + 2 post) The (2 or 4) barriers
+     * are emitted while initializing the fields of the WeakReference instance. The extra pre
+     * barrier of G1 concerns the read of the referent field.
      */
     @Test
-    public void test4() throws Exception {
-        testHelper("test4Snippet", config.useG1GC ? 5 : 2);
+    public void testReferenceGet() throws Exception {
+        this.expectedBarriers = config.useG1GC ? 1 : 0;
+        test("testReferenceGetSnippet");
     }
 
-    public static Object test4Snippet() {
-        WeakReference<Object> weakRef = new WeakReference<>(new Object());
-        return weakRef.get();
+    public static Object testReferenceGetSnippet() {
+        return weakReference.get();
     }
 
-    static WeakReference<Object> wr = new WeakReference<>(new Object());
-    static Container con = new Container();
+    static class DummyReference {
+        Object referent;
+    }
+
+    private static MetaAccessProvider getStaticMetaAccess() {
+        return ((HotSpotBackend) Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend()).getRuntime().getHostProviders().getMetaAccess();
+    }
+
+    private static final WeakReference<?> weakReference = new WeakReference<>(new Object());
+    private static final Object weakReferenceAsObject = new WeakReference<>(new Object());
+    private static final long referenceReferentFieldOffset = HotSpotReplacementsUtil.getFieldOffset(getStaticMetaAccess().lookupJavaType(Reference.class), "referent");
+    private static final long referenceQueueFieldOffset = HotSpotReplacementsUtil.getFieldOffset(getStaticMetaAccess().lookupJavaType(Reference.class), "queue");
+
+    private static final DummyReference dummyReference = new DummyReference();
+    private static final long dummyReferenceReferentFieldOffset = HotSpotReplacementsUtil.getFieldOffset(getStaticMetaAccess().lookupJavaType(DummyReference.class), "referent");
 
     /**
-     * Expected 0 barrier for the Serial GC and 1 for G1. In this test, we load the correct offset
-     * of the WeakReference object so naturally we assert the presence of the pre barrier.
+     * The type is known to be WeakReference and the offset is a constant, so the
+     * {@link org.graalvm.compiler.nodes.extended.RawLoadNode} is converted back into a normal
+     * LoadFieldNode and the lowering of the field node inserts the proper barrier.
      */
     @Test
-    public void test5() throws Exception {
-        testHelper("test5Snippet", config.useG1GC ? 1 : 0);
+    public void testReferenceReferent1() throws Exception {
+        this.expectedBarriers = config.useG1GC ? 1 : 0;
+        test("testReferenceReferentSnippet");
     }
 
-    private static final boolean useCompressedOops = ((HotSpotBackend) Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend()).getRuntime().getVMConfig().useCompressedOops;
-
-    public Object test5Snippet() {
-        return UNSAFE.getObject(wr, useCompressedOops ? 12L : 16L);
+    public Object testReferenceReferentSnippet() {
+        return UNSAFE.getObject(weakReference, referenceReferentFieldOffset);
     }
 
     /**
-     * The following test concerns the runtime checks of the unsafe loads. In this test, we unsafely
-     * load the java.lang.ref.Reference.referent field so the pre barier has to be executed.
+     * The type is known to be WeakReference and the offset is non-constant, so the lowering of the
+     * {@link org.graalvm.compiler.nodes.extended.RawLoadNode} is guarded by a check that the offset
+     * is the same as {@link #referenceReferentFieldOffset} which does a barrier if requires it.
      */
     @Test
-    public void test6() throws Exception {
-        test2("testUnsafeLoad", UNSAFE, wr, Long.valueOf(referentOffset(getMetaAccess())), null);
+    public void testReferenceReferent2() throws Exception {
+        this.expectedBarriers = config.useG1GC ? 1 : 0;
+        test("testReferenceReferent2Snippet", referenceReferentFieldOffset);
+    }
+
+    public Object testReferenceReferent2Snippet(long offset) {
+        return UNSAFE.getObject(weakReference, offset);
     }
 
     /**
-     * The following test concerns the runtime checks of the unsafe loads. In this test, we unsafely
-     * load a matching offset of a wrong object so the pre barier must not be executed.
+     * The type is known to be WeakReference and the offset is constant but not the referent field,
+     * so no barrier is required.
      */
     @Test
-    public void test7() throws Exception {
-        test2("testUnsafeLoad", UNSAFE, con, Long.valueOf(referentOffset(getMetaAccess())), null);
+    public void testReferenceReferent3() throws Exception {
+        this.expectedBarriers = 0;
+        test("testReferenceReferent3Snippet");
+    }
+
+    public Object testReferenceReferent3Snippet() {
+        return UNSAFE.getObject(weakReference, referenceQueueFieldOffset);
     }
 
     /**
-     * The following test concerns the runtime checks of the unsafe loads. In this test, we unsafely
-     * load a non-matching offset field of the java.lang.ref.Reference object so the pre barier must
-     * not be executed.
+     * The type is a super class of WeakReference and the offset is non-constant, so the lowering of
+     * the {@link org.graalvm.compiler.nodes.extended.RawLoadNode} is guarded by a check that the
+     * offset is the same as {@link #referenceReferentFieldOffset} and the base object is a
+     * subclasses of {@link java.lang.ref.Reference} and does a barrier if requires it.
      */
     @Test
-    public void test8() throws Exception {
-        test2("testUnsafeLoad", UNSAFE, wr, Long.valueOf(config.useCompressedOops ? 20 : 32), null);
+    public void testReferenceReferent4() throws Exception {
+        this.expectedBarriers = config.useG1GC ? 1 : 0;
+        test("testReferenceReferent4Snippet");
+    }
+
+    public Object testReferenceReferent4Snippet() {
+        return UNSAFE.getObject(weakReferenceAsObject, referenceReferentFieldOffset);
     }
 
     /**
-     * The following test concerns the runtime checks of the unsafe loads. In this test, we unsafely
-     * load a matching offset+disp field of the java.lang.ref.Reference object so the pre barier
-     * must be executed.
+     * The type is not related to Reference at all so no barrier check is required. This should be
+     * statically detectable.
      */
     @Test
-    public void test10() throws Exception {
-        test2("testUnsafeLoad", UNSAFE, wr, Long.valueOf(config.useCompressedOops ? 6 : 8), Integer.valueOf(config.useCompressedOops ? 6 : 8));
+    public void testReferenceReferent5() throws Exception {
+        this.expectedBarriers = 0;
+        Assert.assertEquals("expected fields to have the same offset", referenceReferentFieldOffset, dummyReferenceReferentFieldOffset);
+        test("testReferenceReferent5Snippet");
     }
 
-    /**
-     * The following test concerns the runtime checks of the unsafe loads. In this test, we unsafely
-     * load a non-matching offset+disp field of the java.lang.ref.Reference object so the pre barier
-     * must not be executed.
-     */
-    @Test
-    public void test9() throws Exception {
-        test2("testUnsafeLoad", UNSAFE, wr, Long.valueOf(config.useCompressedOops ? 10 : 16), Integer.valueOf(config.useCompressedOops ? 10 : 16));
+    public Object testReferenceReferent5Snippet() {
+        return UNSAFE.getObject(dummyReference, referenceReferentFieldOffset);
     }
 
     static Object[] src = new Object[1];
@@ -232,88 +292,93 @@ public class WriteBarrierAdditionTest extends HotSpotGraalCompilerTest {
         }
     }
 
-    public static void testArrayCopy(Object a, Object b, Object c) throws Exception {
+    public static void testArrayCopySnippet(Object a, Object b, Object c) throws Exception {
         System.arraycopy(a, 0, b, 0, (int) c);
     }
 
     @Test
-    public void test11() throws Exception {
-        test2("testArrayCopy", src, dst, dst.length);
+    public void testArrayCopy() throws Exception {
+        this.expectedBarriers = 0;
+        test("testArrayCopySnippet", src, dst, dst.length);
     }
 
-    public static Object testUnsafeLoad(Unsafe theUnsafe, Object a, Object b, Object c) throws Exception {
-        final int offset = (c == null ? 0 : ((Integer) c).intValue());
-        final long displacement = (b == null ? 0 : ((Long) b).longValue());
-        return theUnsafe.getObject(a, offset + displacement);
-    }
-
-    private HotSpotInstalledCode getInstalledCode(String name, boolean withUnsafePrefix) throws Exception {
-        final ResolvedJavaMethod javaMethod = withUnsafePrefix ? getResolvedJavaMethod(WriteBarrierAdditionTest.class, name, Unsafe.class, Object.class, Object.class, Object.class)
-                        : getResolvedJavaMethod(WriteBarrierAdditionTest.class, name, Object.class, Object.class, Object.class);
-        final HotSpotInstalledCode installedCode = (HotSpotInstalledCode) getCode(javaMethod);
-        return installedCode;
-    }
-
-    @SuppressWarnings("try")
-    private void testHelper(final String snippetName, final int expectedBarriers) throws Exception, SecurityException {
-        ResolvedJavaMethod snippet = getResolvedJavaMethod(snippetName);
-        DebugContext debug = getDebugContext();
-        try (DebugContext.Scope s = debug.scope("WriteBarrierAdditionTest", snippet)) {
-            StructuredGraph graph = parseEager(snippet, AllowAssumptions.NO, debug);
-            HighTierContext highContext = getDefaultHighTierContext();
-            MidTierContext midContext = new MidTierContext(getProviders(), getTargetProvider(), OptimisticOptimizations.ALL, graph.getProfilingInfo());
-            new InliningPhase(new InlineEverythingPolicy(), createCanonicalizerPhase()).apply(graph, highContext);
-            this.createCanonicalizerPhase().apply(graph, highContext);
-            new LoweringPhase(this.createCanonicalizerPhase(), LoweringTool.StandardLoweringStage.HIGH_TIER).apply(graph, highContext);
-            new GuardLoweringPhase().apply(graph, midContext);
-            new LoweringPhase(this.createCanonicalizerPhase(), LoweringTool.StandardLoweringStage.MID_TIER).apply(graph, midContext);
-            new WriteBarrierAdditionPhase().apply(graph, midContext);
-            debug.dump(DebugContext.BASIC_LEVEL, graph, "After Write Barrier Addition");
-
-            int barriers = 0;
+    private void verifyBarriers(StructuredGraph graph) {
+        Assert.assertTrue("Unknown collector selected", knownSupport.contains(runtime().getGarbageCollector()));
+        Assert.assertNotEquals("test must set expected barrier count", expectedBarriers, -1);
+        int barriers = 0;
+        if (config.useG1GC) {
+            barriers = graph.getNodes().filter(G1ReferentFieldReadBarrier.class).count() + graph.getNodes().filter(G1PreWriteBarrier.class).count() +
+                            graph.getNodes().filter(G1PostWriteBarrier.class).count();
+        } else {
+            barriers = graph.getNodes().filter(SerialWriteBarrier.class).count();
+        }
+        if (expectedBarriers != barriers) {
+            Assert.assertEquals(expectedBarriers, barriers);
+        }
+        for (WriteNode write : graph.getNodes().filter(WriteNode.class)) {
             if (config.useG1GC) {
-                barriers = graph.getNodes().filter(G1ReferentFieldReadBarrier.class).count() + graph.getNodes().filter(G1PreWriteBarrier.class).count() +
-                                graph.getNodes().filter(G1PostWriteBarrier.class).count();
+                if (write.getBarrierType() != HeapAccess.BarrierType.NONE) {
+                    Assert.assertEquals(1, write.successors().count());
+                    Assert.assertTrue(write.next() instanceof G1PostWriteBarrier);
+                    Assert.assertTrue(write.predecessor() instanceof G1PreWriteBarrier || write.getLocationIdentity().isImmutable());
+                }
             } else {
-                barriers = graph.getNodes().filter(SerialWriteBarrier.class).count();
-            }
-            if (expectedBarriers != barriers) {
-                Assert.assertEquals(getScheduledGraphString(graph), expectedBarriers, barriers);
-            }
-            for (WriteNode write : graph.getNodes().filter(WriteNode.class)) {
-                if (config.useG1GC) {
-                    if (write.getBarrierType() != BarrierType.NONE) {
-                        Assert.assertEquals(1, write.successors().count());
-                        Assert.assertTrue(write.next() instanceof G1PostWriteBarrier);
-                        Assert.assertTrue(write.predecessor() instanceof G1PreWriteBarrier);
-                    }
-                } else {
-                    if (write.getBarrierType() != BarrierType.NONE) {
-                        Assert.assertEquals(1, write.successors().count());
-                        Assert.assertTrue(write.next() instanceof SerialWriteBarrier);
-                    }
+                if (write.getBarrierType() != HeapAccess.BarrierType.NONE) {
+                    Assert.assertEquals(1, write.successors().count());
+                    Assert.assertTrue(write.next() instanceof SerialWriteBarrier);
                 }
             }
+        }
 
-            for (ReadNode read : graph.getNodes().filter(ReadNode.class)) {
-                if (read.getBarrierType() != BarrierType.NONE) {
-                    Assert.assertTrue(read.getAddress() instanceof OffsetAddressNode);
+        for (ReadNode read : graph.getNodes().filter(ReadNode.class)) {
+            if (read.getBarrierType() != HeapAccess.BarrierType.NONE) {
+                if (read.getAddress() instanceof OffsetAddressNode) {
                     JavaConstant constDisp = ((OffsetAddressNode) read.getAddress()).getOffset().asJavaConstant();
-                    Assert.assertNotNull(constDisp);
-                    Assert.assertEquals(referentOffset(getMetaAccess()), constDisp.asLong());
-                    Assert.assertEquals(BarrierType.WEAK_FIELD, read.getBarrierType());
-                    if (config.useG1GC) {
-                        Assert.assertTrue(read.next() instanceof G1ReferentFieldReadBarrier);
+                    if (constDisp != null) {
+                        Assert.assertEquals(referentOffset(getMetaAccess()), constDisp.asLong());
                     }
                 }
+                Assert.assertTrue(HeapAccess.BarrierType.WEAK_FIELD == read.getBarrierType() || HeapAccess.BarrierType.MAYBE_WEAK_FIELD == read.getBarrierType());
+                if (config.useG1GC) {
+                    Assert.assertTrue(read.next() instanceof G1ReferentFieldReadBarrier);
+                }
             }
-        } catch (Throwable e) {
-            throw debug.handle(e);
         }
     }
 
-    private void test2(final String snippet, Object... args) throws Exception {
-        HotSpotInstalledCode code = getInstalledCode(snippet, args[0] instanceof Unsafe);
-        code.executeVarargs(args);
+    protected Result testWithoutPEA(String name, Object... args) {
+        return test(new OptionValues(getInitialOptions(), PartialEscapeAnalysis, false, FullUnroll, false, LoopPeeling, false), name, args);
+    }
+
+    @Before
+    public void before() {
+        expectedBarriers = -1;
+    }
+
+    /*
+     * Check the state of the barriers immediately after insertion.
+     */
+    @Override
+    protected Suites createSuites(OptionValues opts) {
+        Suites ret = getBackend().getSuites().getDefaultSuites(opts).copy();
+        ListIterator<BasePhase<? super MidTierContext>> iter = ret.getMidTier().findPhase(WriteBarrierAdditionPhase.class, true);
+        iter.add(new Phase() {
+
+            @Override
+            protected void run(StructuredGraph graph) {
+                verifyBarriers(graph);
+            }
+
+            @Override
+            public float codeSizeIncrease() {
+                return NodeSize.IGNORE_SIZE_CONTRACT_FACTOR;
+            }
+
+            @Override
+            protected CharSequence getName() {
+                return "VerifyBarriersPhase";
+            }
+        });
+        return ret;
     }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/GraalHotSpotVMConfig.java

@@ -165,6 +165,7 @@ public class GraalHotSpotVMConfig extends GraalHotSpotVMConfigBase {
     }
 
     public final boolean useG1GC = getFlag("UseG1GC", Boolean.class);
+    public final boolean useCMSGC = getFlag("UseConcMarkSweepGC", Boolean.class, false);
 
     public final int allocatePrefetchStyle = getFlag("AllocatePrefetchStyle", Integer.class);
     public final int allocatePrefetchInstr = getFlag("AllocatePrefetchInstr", Integer.class);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotBackendFactory.java

@@ -39,6 +39,7 @@ import jdk.vm.ci.code.TargetDescription;
 import jdk.vm.ci.hotspot.HotSpotConstantReflectionProvider;
 import jdk.vm.ci.hotspot.HotSpotJVMCIRuntime;
 import jdk.vm.ci.hotspot.HotSpotMetaAccessProvider;
+import jdk.vm.ci.meta.MetaAccessProvider;
 
 public abstract class HotSpotBackendFactory {
 
@@ -54,8 +55,8 @@ public abstract class HotSpotBackendFactory {
         return new HotSpotStampProvider();
     }
 
-    protected HotSpotGCProvider createGCProvider(GraalHotSpotVMConfig config) {
-        return new HotSpotGCProvider(config);
+    protected HotSpotGCProvider createGCProvider(GraalHotSpotVMConfig config, MetaAccessProvider metaAccess) {
+        return new HotSpotGCProvider(config, metaAccess);
     }
 
     protected HotSpotReplacementsImpl createReplacements(TargetDescription target, Providers p, HotSpotSnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalRuntime.java

@@ -231,7 +231,7 @@ public final class HotSpotGraalRuntime implements HotSpotGraalRuntimeProvider {
     public enum HotSpotGC {
         // Supported GCs
         Serial(true, "UseSerialGC"),
-        Parallel(true, "UseParallelGC", "UseParallelOldGC"),
+        Parallel(true, "UseParallelGC", "UseParallelOldGC", "UseParNewGC"),
         CMS(true, "UseConcMarkSweepGC"),
         G1(true, "UseG1GC"),
 
@@ -355,6 +355,7 @@ public final class HotSpotGraalRuntime implements HotSpotGraalRuntimeProvider {
         return null;
     }
 
+    @Override
     public HotSpotGC getGarbageCollector() {
         return garbageCollector;
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalRuntimeProvider.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -58,6 +58,8 @@ public interface HotSpotGraalRuntimeProvider extends GraalRuntime, RuntimeProvid
         return getClass().getSimpleName();
     }
 
+    HotSpotGraalRuntime.HotSpotGC getGarbageCollector();
+
     @Override
     HotSpotBackend getHostBackend();
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotReplacementsImpl.java

@@ -27,8 +27,8 @@ package org.graalvm.compiler.hotspot;
 import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE;
 import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE;
 import static org.graalvm.compiler.core.common.GraalOptions.UseEncodedGraphs;
-import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.INLINE_AFTER_PARSING;
 import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.ROOT_COMPILATION;
+import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.INLINE_AFTER_PARSING;
 
 import java.util.Set;
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/SymbolicSnippetEncoder.java

@@ -30,7 +30,7 @@ import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE;
 import static org.graalvm.compiler.core.common.GraalOptions.UseEncodedGraphs;
 import static org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin.InlineInfo.createIntrinsicInlineInfo;
 import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.INLINE_AFTER_PARSING;
-import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.ROOT_COMPILATION;
+import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.ROOT_COMPILATION_ENCODING;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -306,13 +306,17 @@ public class SymbolicSnippetEncoder {
 
         StructuredGraph getMethodSubstitutionGraph(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original, ReplacementsImpl replacements, IntrinsicContext.CompilationContext context,
                         StructuredGraph.AllowAssumptions allowAssumptions, Cancellable cancellable, OptionValues options) {
-            Integer startOffset = snippetStartOffsets.get(plugin.toString() + context);
+            IntrinsicContext.CompilationContext contextToUse = context;
+            if (context == IntrinsicContext.CompilationContext.ROOT_COMPILATION) {
+                contextToUse = IntrinsicContext.CompilationContext.ROOT_COMPILATION_ENCODING;
+            }
+            Integer startOffset = snippetStartOffsets.get(plugin.toString() + contextToUse);
             if (startOffset == null) {
-                throw GraalError.shouldNotReachHere("plugin graph not found: " + plugin + " with " + context);
+                throw GraalError.shouldNotReachHere("plugin graph not found: " + plugin + " with " + contextToUse);
             }
 
             ResolvedJavaType accessingClass = replacements.getProviders().getMetaAccess().lookupJavaType(plugin.getDeclaringClass());
-            return decodeGraph(original, accessingClass, startOffset, replacements, context, allowAssumptions, cancellable, options);
+            return decodeGraph(original, accessingClass, startOffset, replacements, contextToUse, allowAssumptions, cancellable, options);
         }
 
         @SuppressWarnings("try")
@@ -406,8 +410,13 @@ public class SymbolicSnippetEncoder {
         //
         // -J-Dgraal.Dump=SymbolicSnippetEncoder_:2 -J-Dgraal.PrintGraph=File
         // -J-Dgraal.DebugStubsAndSnippets=true
+        IntrinsicContext.CompilationContext contextToUse = context;
+        if (context == IntrinsicContext.CompilationContext.ROOT_COMPILATION) {
+            contextToUse = IntrinsicContext.CompilationContext.ROOT_COMPILATION_ENCODING;
+        }
         try (DebugContext debug = openDebugContext("SymbolicSnippetEncoder_", method, options)) {
-            StructuredGraph graph = snippetReplacements.makeGraph(debug, snippetReplacements.getDefaultReplacementBytecodeProvider(), method, args, original, trackNodeSourcePosition, null, context);
+            StructuredGraph graph = snippetReplacements.makeGraph(debug, snippetReplacements.getDefaultReplacementBytecodeProvider(), method, args, original, trackNodeSourcePosition, null,
+                            contextToUse);
 
             // Check if all methods which should be inlined are really inlined.
             for (MethodCallTargetNode callTarget : graph.getNodes(MethodCallTargetNode.TYPE)) {
@@ -508,7 +517,7 @@ public class SymbolicSnippetEncoder {
                 ResolvedJavaMethod original = plugin.getOriginalMethod(originalReplacements.getProviders().getMetaAccess());
                 registerMethodSubstitution(plugin, original, INLINE_AFTER_PARSING, options);
                 if (!original.isNative()) {
-                    registerMethodSubstitution(plugin, original, ROOT_COMPILATION, options);
+                    registerMethodSubstitution(plugin, original, ROOT_COMPILATION_ENCODING, options);
                 }
             }
             preparedPlugins = plugins.size();

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java

@@ -74,6 +74,7 @@ import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
 import org.graalvm.compiler.hotspot.nodes.type.MethodPointerStamp;
 import org.graalvm.compiler.hotspot.replacements.AssertionSnippets;
 import org.graalvm.compiler.hotspot.replacements.ClassGetHubNode;
+import org.graalvm.compiler.hotspot.replacements.FastNotifyNode;
 import org.graalvm.compiler.hotspot.replacements.HashCodeSnippets;
 import org.graalvm.compiler.hotspot.replacements.HotSpotG1WriteBarrierSnippets;
 import org.graalvm.compiler.hotspot.replacements.HotSpotSerialWriteBarrierSnippets;
@@ -85,8 +86,11 @@ import org.graalvm.compiler.hotspot.replacements.LoadExceptionObjectSnippets;
 import org.graalvm.compiler.hotspot.replacements.MonitorSnippets;
 import org.graalvm.compiler.hotspot.replacements.NewObjectSnippets;
 import org.graalvm.compiler.hotspot.replacements.ObjectCloneSnippets;
+import org.graalvm.compiler.hotspot.replacements.ObjectSnippets;
 import org.graalvm.compiler.hotspot.replacements.StringToBytesSnippets;
+import org.graalvm.compiler.hotspot.replacements.UnsafeCopyMemoryNode;
 import org.graalvm.compiler.hotspot.replacements.UnsafeLoadSnippets;
+import org.graalvm.compiler.hotspot.replacements.UnsafeSnippets;
 import org.graalvm.compiler.hotspot.replacements.aot.ResolveConstantSnippets;
 import org.graalvm.compiler.hotspot.replacements.arraycopy.HotSpotArraycopySnippets;
 import org.graalvm.compiler.hotspot.replacements.profiling.ProfileSnippets;
@@ -107,6 +111,7 @@ import org.graalvm.compiler.nodes.ParameterNode;
 import org.graalvm.compiler.nodes.SafepointNode;
 import org.graalvm.compiler.nodes.StartNode;
 import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.nodes.StructuredGraph.GuardsStage;
 import org.graalvm.compiler.nodes.UnwindNode;
 import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.calc.AddNode;
@@ -120,14 +125,12 @@ import org.graalvm.compiler.nodes.extended.BytecodeExceptionNode;
 import org.graalvm.compiler.nodes.extended.BytecodeExceptionNode.BytecodeExceptionKind;
 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
 import org.graalvm.compiler.nodes.extended.GetClassNode;
-import org.graalvm.compiler.nodes.extended.GuardedUnsafeLoadNode;
 import org.graalvm.compiler.nodes.extended.LoadHubNode;
 import org.graalvm.compiler.nodes.extended.LoadMethodNode;
 import org.graalvm.compiler.nodes.extended.OSRLocalNode;
 import org.graalvm.compiler.nodes.extended.OSRLockNode;
 import org.graalvm.compiler.nodes.extended.OSRMonitorEnterNode;
 import org.graalvm.compiler.nodes.extended.OSRStartNode;
-import org.graalvm.compiler.nodes.extended.RawLoadNode;
 import org.graalvm.compiler.nodes.extended.StoreHubNode;
 import org.graalvm.compiler.nodes.gc.G1ArrayRangePostWriteBarrier;
 import org.graalvm.compiler.nodes.gc.G1ArrayRangePreWriteBarrier;
@@ -202,7 +205,8 @@ public abstract class DefaultHotSpotLoweringProvider extends DefaultJavaLowering
     protected HashCodeSnippets.Templates hashCodeSnippets;
     protected ResolveConstantSnippets.Templates resolveConstantSnippets;
     protected ProfileSnippets.Templates profileSnippets;
-
+    protected ObjectSnippets.Templates objectSnippets;
+    protected UnsafeSnippets.Templates unsafeSnippets;
     protected ObjectCloneSnippets.Templates objectCloneSnippets;
     protected ForeignCallSnippets.Templates foreignCallSnippets;
 
@@ -212,6 +216,7 @@ public abstract class DefaultHotSpotLoweringProvider extends DefaultJavaLowering
         this.runtime = runtime;
         this.registers = registers;
         this.constantReflection = constantReflection;
+
     }
 
     @Override
@@ -233,6 +238,8 @@ public abstract class DefaultHotSpotLoweringProvider extends DefaultJavaLowering
         resolveConstantSnippets = new ResolveConstantSnippets.Templates(options, factories, providers, target);
         objectCloneSnippets = new ObjectCloneSnippets.Templates(options, factories, providers, target);
         foreignCallSnippets = new ForeignCallSnippets.Templates(options, factories, providers, target);
+        objectSnippets = new ObjectSnippets.Templates(options, factories, providers, target);
+        unsafeSnippets = new UnsafeSnippets.Templates(options, factories, providers, target);
         if (JavaVersionUtil.JAVA_SPEC <= 8) {
             // AOT only introduced in JDK 9
             profileSnippets = null;
@@ -408,10 +415,19 @@ public abstract class DefaultHotSpotLoweringProvider extends DefaultJavaLowering
                 profileSnippets.lower((ProfileNode) n, tool);
             } else if (n instanceof KlassBeingInitializedCheckNode) {
                 newObjectSnippets.lower((KlassBeingInitializedCheckNode) n, registers, tool);
+            } else if (n instanceof FastNotifyNode) {
+                if (graph.getGuardsStage() == GuardsStage.AFTER_FSA) {
+                    objectSnippets.lower(n, tool);
+                }
+            } else if (n instanceof UnsafeCopyMemoryNode) {
+                if (graph.getGuardsStage() == GuardsStage.AFTER_FSA) {
+                    unsafeSnippets.lower((UnsafeCopyMemoryNode) n, tool);
+                }
             } else {
                 super.lower(n, tool);
             }
         }
+
     }
 
     private static void lowerComputeObjectAddressNode(ComputeObjectAddressNode n) {
@@ -560,16 +576,6 @@ public abstract class DefaultHotSpotLoweringProvider extends DefaultJavaLowering
         return graph.unique(new FloatingReadNode(address, OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION, null, KlassPointerStamp.klassNonNull(), AbstractBeginNode.prevBegin(anchor)));
     }
 
-    @Override
-    protected void lowerUnsafeLoadNode(RawLoadNode load, LoweringTool tool) {
-        StructuredGraph graph = load.graph();
-        if (!(load instanceof GuardedUnsafeLoadNode) && !graph.getGuardsStage().allowsFloatingGuards() && addReadBarrier(load)) {
-            unsafeLoadSnippets.lower(load, tool);
-        } else {
-            super.lowerUnsafeLoadNode(load, tool);
-        }
-    }
-
     private void lowerLoadMethodNode(LoadMethodNode loadMethodNode) {
         StructuredGraph graph = loadMethodNode.graph();
         HotSpotResolvedJavaMethod method = (HotSpotResolvedJavaMethod) loadMethodNode.getMethod();
@@ -720,17 +726,6 @@ public abstract class DefaultHotSpotLoweringProvider extends DefaultJavaLowering
         graph.replaceFixedWithFixed(node, foreignCallNode);
     }
 
-    private boolean addReadBarrier(RawLoadNode load) {
-        if (runtime.getVMConfig().useG1GC && load.graph().getGuardsStage() == StructuredGraph.GuardsStage.FIXED_DEOPTS && load.object().getStackKind() == JavaKind.Object &&
-                        load.accessKind() == JavaKind.Object && !StampTool.isPointerAlwaysNull(load.object())) {
-            ResolvedJavaType type = StampTool.typeOrNull(load.object());
-            if (type != null && !type.isArray()) {
-                return true;
-            }
-        }
-        return false;
-    }
-
     private ReadNode createReadVirtualMethod(StructuredGraph graph, ValueNode hub, HotSpotResolvedJavaMethod method, ResolvedJavaType receiverType) {
         return createReadVirtualMethod(graph, hub, method.vtableEntryOffset(receiverType));
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGCProvider.java

@@ -35,11 +35,13 @@ import org.graalvm.compiler.nodes.java.AbstractNewObjectNode;
 import org.graalvm.compiler.nodes.memory.FixedAccessNode;
 import org.graalvm.compiler.nodes.spi.GCProvider;
 
+import jdk.vm.ci.meta.MetaAccessProvider;
+
 public class HotSpotGCProvider implements GCProvider {
     private final BarrierSet barrierSet;
 
-    public HotSpotGCProvider(GraalHotSpotVMConfig config) {
-        this.barrierSet = createBarrierSet(config);
+    public HotSpotGCProvider(GraalHotSpotVMConfig config, MetaAccessProvider metaAccess) {
+        this.barrierSet = createBarrierSet(config, metaAccess);
     }
 
     @Override
@@ -47,10 +49,10 @@ public class HotSpotGCProvider implements GCProvider {
         return barrierSet;
     }
 
-    private BarrierSet createBarrierSet(GraalHotSpotVMConfig config) {
+    private BarrierSet createBarrierSet(GraalHotSpotVMConfig config, MetaAccessProvider metaAccess) {
         boolean useDeferredInitBarriers = config.useDeferredInitBarriers;
         if (config.useG1GC) {
-            return new G1BarrierSet() {
+            return new G1BarrierSet(metaAccess) {
                 @Override
                 protected boolean writeRequiresPostBarrier(FixedAccessNode initializingWrite, ValueNode writtenValue) {
                     if (!super.writeRequiresPostBarrier(initializingWrite, writtenValue)) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java

@@ -59,11 +59,11 @@ import org.graalvm.compiler.hotspot.replacements.CipherBlockChainingSubstitution
 import org.graalvm.compiler.hotspot.replacements.ClassGetHubNode;
 import org.graalvm.compiler.hotspot.replacements.CounterModeSubstitutions;
 import org.graalvm.compiler.hotspot.replacements.DigestBaseSubstitutions;
+import org.graalvm.compiler.hotspot.replacements.FastNotifyNode;
 import org.graalvm.compiler.hotspot.replacements.HotSpotArraySubstitutions;
 import org.graalvm.compiler.hotspot.replacements.HotSpotClassSubstitutions;
 import org.graalvm.compiler.hotspot.replacements.IdentityHashCodeNode;
 import org.graalvm.compiler.hotspot.replacements.ObjectCloneNode;
-import org.graalvm.compiler.hotspot.replacements.ObjectSubstitutions;
 import org.graalvm.compiler.hotspot.replacements.ReflectionGetCallerClassNode;
 import org.graalvm.compiler.hotspot.replacements.ReflectionSubstitutions;
 import org.graalvm.compiler.hotspot.replacements.SHA2Substitutions;
@@ -194,7 +194,7 @@ public class HotSpotGraphBuilderPlugins {
                 registerCounterModePlugins(invocationPlugins, config, replacements);
                 registerBase64Plugins(invocationPlugins, config, metaAccess, foreignCalls);
                 registerUnsafePlugins(invocationPlugins, config, replacements);
-                StandardGraphBuilderPlugins.registerInvocationPlugins(metaAccess, snippetReflection, invocationPlugins, replacements, true, false);
+                StandardGraphBuilderPlugins.registerInvocationPlugins(metaAccess, snippetReflection, invocationPlugins, replacements, true, false, true);
                 registerArrayPlugins(invocationPlugins, replacements);
                 registerStringPlugins(invocationPlugins, replacements);
                 registerArraysSupportPlugins(invocationPlugins, config, replacements);
@@ -230,12 +230,48 @@ public class HotSpotGraphBuilderPlugins {
                 }
             });
         }
-        r.registerMethodSubstitution(ObjectSubstitutions.class, "hashCode", Receiver.class);
+        r.register1("hashCode", Receiver.class, new InvocationPlugin() {
+            @Override
+            public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) {
+                ValueNode object = receiver.get();
+                b.addPush(JavaKind.Int, new IdentityHashCodeNode(object));
+                return true;
+            }
+
+            @Override
+            public boolean inlineOnly() {
+                return true;
+            }
+        });
         if (config.inlineNotify()) {
-            r.registerMethodSubstitution(ObjectSubstitutions.class, "notify", Receiver.class);
+            r.register1("notify", Receiver.class, new InvocationPlugin() {
+                @Override
+                public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) {
+                    ValueNode object = receiver.get();
+                    b.add(new FastNotifyNode(object, false, b.bci()));
+                    return true;
+                }
+
+                @Override
+                public boolean inlineOnly() {
+                    return true;
+                }
+            });
         }
         if (config.inlineNotifyAll()) {
-            r.registerMethodSubstitution(ObjectSubstitutions.class, "notifyAll", Receiver.class);
+            r.register1("notifyAll", Receiver.class, new InvocationPlugin() {
+                @Override
+                public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) {
+                    ValueNode object = receiver.get();
+                    b.add(new FastNotifyNode(object, true, b.bci()));
+                    return true;
+                }
+
+                @Override
+                public boolean inlineOnly() {
+                    return true;
+                }
+            });
         }
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotUnsafeSubstitutions.java

@@ -24,18 +24,10 @@
 
 package org.graalvm.compiler.hotspot.meta;
 
-import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.doingUnsafeAccessOffset;
-
 import org.graalvm.compiler.api.replacements.ClassSubstitution;
 import org.graalvm.compiler.api.replacements.MethodSubstitution;
-import org.graalvm.compiler.hotspot.HotSpotBackend;
-import org.graalvm.compiler.hotspot.nodes.CurrentJavaThreadNode;
-import org.graalvm.compiler.nodes.ComputeObjectAddressNode;
+import org.graalvm.compiler.hotspot.replacements.UnsafeCopyMemoryNode;
 import org.graalvm.compiler.serviceprovider.JavaVersionUtil;
-import org.graalvm.compiler.word.Word;
-import jdk.internal.vm.compiler.word.LocationIdentity;
-import jdk.internal.vm.compiler.word.WordFactory;
 
 @ClassSubstitution(className = {"jdk.internal.misc.Unsafe", "sun.misc.Unsafe"})
 public class HotSpotUnsafeSubstitutions {
@@ -45,27 +37,12 @@ public class HotSpotUnsafeSubstitutions {
     @SuppressWarnings("unused")
     @MethodSubstitution(isStatic = false)
     static void copyMemory(Object receiver, Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes) {
-        Word srcAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(srcBase, srcOffset));
-        Word dstAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(destBase, destOffset));
-        Word size = WordFactory.signed(bytes);
-
-        HotSpotBackend.unsafeArraycopy(srcAddr, dstAddr, size);
+        UnsafeCopyMemoryNode.copyMemory(false, receiver, srcBase, srcOffset, destBase, destOffset, bytes);
     }
 
     @SuppressWarnings("unused")
     @MethodSubstitution(value = "copyMemory", isStatic = false)
     static void copyMemoryGuarded(Object receiver, Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes) {
-        Word srcAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(srcBase, srcOffset));
-        Word dstAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(destBase, destOffset));
-        Word size = WordFactory.signed(bytes);
-        Word javaThread = CurrentJavaThreadNode.get();
-        int offset = doingUnsafeAccessOffset(INJECTED_VMCONFIG);
-        LocationIdentity any = LocationIdentity.any();
-
-        /* Set doingUnsafeAccess to guard and handle unsafe memory access failures */
-        javaThread.writeByte(offset, (byte) 1, any);
-        HotSpotBackend.unsafeArraycopy(srcAddr, dstAddr, size);
-        /* Reset doingUnsafeAccess */
-        javaThread.writeByte(offset, (byte) 0, any);
+        UnsafeCopyMemoryNode.copyMemory(true, receiver, srcBase, srcOffset, destBase, destOffset, bytes);
     }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/FastNotifyNode.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package org.graalvm.compiler.hotspot.replacements;
+
+import static org.graalvm.compiler.nodeinfo.InputType.State;
+import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_2;
+import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_0;
+
+import org.graalvm.compiler.core.common.type.StampFactory;
+import org.graalvm.compiler.graph.NodeClass;
+import org.graalvm.compiler.nodeinfo.NodeInfo;
+import org.graalvm.compiler.nodes.DeoptimizingNode;
+import org.graalvm.compiler.nodes.FrameState;
+import org.graalvm.compiler.nodes.ValueNode;
+import org.graalvm.compiler.nodes.memory.AbstractMemoryCheckpoint;
+import org.graalvm.compiler.nodes.memory.MemoryCheckpoint;
+import org.graalvm.compiler.nodes.spi.Lowerable;
+import org.graalvm.compiler.nodes.spi.LoweringTool;
+import jdk.internal.vm.compiler.word.LocationIdentity;
+
+@NodeInfo(cycles = CYCLES_2, size = SIZE_0)
+public class FastNotifyNode extends AbstractMemoryCheckpoint implements Lowerable, MemoryCheckpoint.Single, DeoptimizingNode.DeoptDuring {
+
+    public static final NodeClass<FastNotifyNode> TYPE = NodeClass.create(FastNotifyNode.class);
+    private final boolean notifyAll;
+
+    private final int bci;
+
+    @Input ValueNode object;
+
+    @OptionalInput(State) FrameState stateDuring;
+
+    public FastNotifyNode(ValueNode object, boolean notifyAll, int bci) {
+        super(TYPE, StampFactory.forVoid());
+        this.object = object;
+        this.notifyAll = notifyAll;
+        this.bci = bci;
+    }
+
+    @Override
+    public void lower(LoweringTool tool) {
+        tool.getLowerer().lower(this, tool);
+    }
+
+    public boolean isNotifyAll() {
+        return notifyAll;
+    }
+
+    @Override
+    public LocationIdentity getKilledLocationIdentity() {
+        return LocationIdentity.any();
+    }
+
+    @Override
+    public FrameState stateDuring() {
+        return stateDuring;
+    }
+
+    @Override
+    public void setStateDuring(FrameState stateDuring) {
+        updateUsages(this.stateDuring, stateDuring);
+        this.stateDuring = stateDuring;
+    }
+
+    @Override
+    public void computeStateDuring(FrameState currentStateAfter) {
+        FrameState newStateDuring = currentStateAfter.duplicateModifiedDuringCall(bci, asNode().getStackKind());
+        setStateDuring(newStateDuring);
+    }
+
+    @Override
+    public boolean canDeoptimize() {
+        return true;
+    }
+
+    public int getBci() {
+        return bci;
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotG1WriteBarrierSnippets.java

@@ -56,6 +56,7 @@ import jdk.internal.vm.compiler.word.WordFactory;
 import jdk.vm.ci.code.Register;
 import jdk.vm.ci.code.TargetDescription;
 import jdk.vm.ci.meta.JavaKind;
+import jdk.vm.ci.meta.ResolvedJavaType;
 
 public final class HotSpotG1WriteBarrierSnippets extends G1WriteBarrierSnippets {
     public static final ForeignCallDescriptor G1WBPRECALL = new ForeignCallDescriptor("write_barrier_pre", void.class, Object.class);
@@ -175,6 +176,16 @@ public final class HotSpotG1WriteBarrierSnippets extends G1WriteBarrierSnippets
         return Log.LOG_PRINTF;
     }
 
+    @Override
+    protected ResolvedJavaType referenceType() {
+        return HotSpotReplacementsUtil.referenceType(INJECTED_METAACCESS);
+    }
+
+    @Override
+    protected long referentOffset() {
+        return HotSpotReplacementsUtil.referentOffset(INJECTED_METAACCESS);
+    }
+
     public static class Templates extends AbstractTemplates {
         private final SnippetInfo g1PreWriteBarrier;
         private final SnippetInfo g1ReferentReadBarrier;
@@ -191,7 +202,9 @@ public final class HotSpotG1WriteBarrierSnippets extends G1WriteBarrierSnippets
             HotSpotG1WriteBarrierSnippets receiver = new HotSpotG1WriteBarrierSnippets(config, providers.getRegisters());
             g1PreWriteBarrier = snippet(G1WriteBarrierSnippets.class, "g1PreWriteBarrier", null, receiver, GC_INDEX_LOCATION, GC_LOG_LOCATION, SATB_QUEUE_MARKING_LOCATION, SATB_QUEUE_INDEX_LOCATION,
                             SATB_QUEUE_BUFFER_LOCATION);
-            g1ReferentReadBarrier = g1PreWriteBarrier;
+            g1ReferentReadBarrier = snippet(G1WriteBarrierSnippets.class, "g1ReferentReadBarrier", null, receiver, GC_INDEX_LOCATION, GC_LOG_LOCATION, SATB_QUEUE_MARKING_LOCATION,
+                            SATB_QUEUE_INDEX_LOCATION,
+                            SATB_QUEUE_BUFFER_LOCATION);
             g1PostWriteBarrier = snippet(G1WriteBarrierSnippets.class, "g1PostWriteBarrier", null, receiver, GC_CARD_LOCATION, GC_INDEX_LOCATION, GC_LOG_LOCATION, CARD_QUEUE_INDEX_LOCATION,
                             CARD_QUEUE_BUFFER_LOCATION);
             g1ArrayRangePreWriteBarrier = snippet(G1WriteBarrierSnippets.class, "g1ArrayRangePreWriteBarrier", null, receiver, GC_INDEX_LOCATION, GC_LOG_LOCATION, SATB_QUEUE_MARKING_LOCATION,

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotReplacementsUtil.java

@@ -155,7 +155,7 @@ public class HotSpotReplacementsUtil {
     }
 
     @Fold
-    static int getFieldOffset(ResolvedJavaType type, String fieldName) {
+    public static int getFieldOffset(ResolvedJavaType type, String fieldName) {
         for (ResolvedJavaField field : type.getInstanceFields(true)) {
             if (field.getName().equals(fieldName)) {
                 return field.getOffset();
@@ -889,6 +889,11 @@ public class HotSpotReplacementsUtil {
         return getFieldOffset(metaAccessProvider.lookupJavaType(Reference.class), "referent");
     }
 
+    @Fold
+    public static ResolvedJavaType referenceType(@InjectedParameter MetaAccessProvider metaAccessProvider) {
+        return metaAccessProvider.lookupJavaType(Reference.class);
+    }
+
     public static final LocationIdentity OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION = new HotSpotOptimizingLocationIdentity("ObjArrayKlass::_element_klass") {
         @Override
         public ValueNode canonicalizeRead(ValueNode read, AddressNode location, ValueNode object, CanonicalizerTool tool) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/ObjectSnippets.java

@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package org.graalvm.compiler.hotspot.replacements;
+
+import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
+
+import jdk.internal.vm.compiler.collections.UnmodifiableEconomicMap;
+import org.graalvm.compiler.api.replacements.Snippet;
+import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
+import org.graalvm.compiler.debug.DebugHandlersFactory;
+import org.graalvm.compiler.debug.GraalError;
+import org.graalvm.compiler.graph.Node;
+import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
+import org.graalvm.compiler.graph.Node.NodeIntrinsic;
+import org.graalvm.compiler.hotspot.meta.HotSpotHostForeignCallsProvider;
+import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
+import org.graalvm.compiler.nodes.FrameState;
+import org.graalvm.compiler.nodes.InvokeNode;
+import org.graalvm.compiler.nodes.InvokeWithExceptionNode;
+import org.graalvm.compiler.nodes.PiNode;
+import org.graalvm.compiler.nodes.SnippetAnchorNode;
+import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.nodes.extended.ForeignCallNode;
+import org.graalvm.compiler.nodes.spi.LoweringTool;
+import org.graalvm.compiler.options.OptionValues;
+import org.graalvm.compiler.replacements.SnippetTemplate;
+import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
+import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
+import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
+import org.graalvm.compiler.replacements.Snippets;
+
+import jdk.vm.ci.code.TargetDescription;
+import jdk.vm.ci.meta.ResolvedJavaMethod;
+
+public class ObjectSnippets implements Snippets {
+
+    @NodeIntrinsic(ForeignCallNode.class)
+    public static native boolean fastNotifyStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object o);
+
+    @Snippet
+    public static void fastNotify(Object thisObj) {
+        if (fastNotifyStub(HotSpotHostForeignCallsProvider.NOTIFY, thisObj)) {
+            return;
+        } else {
+            PiNode.piCastNonNull(thisObj, SnippetAnchorNode.anchor()).notify();
+        }
+    }
+
+    @Snippet
+    public static void fastNotifyAll(Object thisObj) {
+        if (fastNotifyStub(HotSpotHostForeignCallsProvider.NOTIFY_ALL, thisObj)) {
+            return;
+        } else {
+            PiNode.piCastNonNull(thisObj, SnippetAnchorNode.anchor()).notifyAll();
+        }
+    }
+
+    public static class Templates extends AbstractTemplates {
+        private final SnippetInfo notifySnippet = snippet(ObjectSnippets.class, "fastNotify", originalNotifyCall(false), null);
+        private final SnippetInfo notifyAllSnippet = snippet(ObjectSnippets.class, "fastNotifyAll", originalNotifyCall(true), null);
+
+        public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, HotSpotProviders providers, TargetDescription target) {
+            super(options, factories, providers, providers.getSnippetReflection(), target);
+        }
+
+        private ResolvedJavaMethod originalNotifyCall(boolean notifyAll) throws GraalError {
+            if (notifyAll) {
+                return findMethod(providers.getMetaAccess(), Object.class, "notifyAll");
+            } else {
+                return findMethod(providers.getMetaAccess(), Object.class, "notify");
+            }
+        }
+
+        public void lower(Node n, LoweringTool tool) {
+            if (n instanceof FastNotifyNode) {
+                FastNotifyNode fn = (FastNotifyNode) n;
+                StructuredGraph graph = (StructuredGraph) n.graph();
+                FrameState stateDuringCall = fn.stateDuring();
+                assert stateDuringCall != null : "Must have valid state for snippet recursive notify call";
+                Arguments args = new Arguments(fn.isNotifyAll() ? notifyAllSnippet : notifySnippet, graph.getGuardsStage(), tool.getLoweringStage());
+                args.add("thisObj", fn.object);
+                SnippetTemplate template = template(fn, args);
+                graph.getDebug().log("Lowering fast notify in %s: node=%s, template=%s, arguments=%s", graph, fn, template, args);
+                UnmodifiableEconomicMap<Node, Node> replacements = template.instantiate(providers.getMetaAccess(), fn, DEFAULT_REPLACER, args);
+                for (Node originalNode : replacements.getKeys()) {
+                    if (originalNode instanceof InvokeNode) {
+                        InvokeNode invoke = (InvokeNode) replacements.get(originalNode);
+                        assert invoke.asNode().graph() == graph;
+                        // Here we need to fix the bci of the invoke
+                        invoke.replaceBci(fn.getBci());
+                        invoke.setStateDuring(null);
+                        invoke.setStateAfter(null);
+                        invoke.setStateDuring(stateDuringCall);
+                    } else if (originalNode instanceof InvokeWithExceptionNode) {
+                        throw new GraalError("unexpected invoke with exception %s in snippet", originalNode);
+                    }
+                }
+            } else {
+                GraalError.shouldNotReachHere("Unknown object snippet lowered node");
+            }
+        }
+
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/ObjectSubstitutions.java

@@ -1,64 +0,0 @@
-/*
- * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-
-package org.graalvm.compiler.hotspot.replacements;
-
-import org.graalvm.compiler.api.replacements.ClassSubstitution;
-import org.graalvm.compiler.api.replacements.MethodSubstitution;
-import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
-import org.graalvm.compiler.hotspot.meta.HotSpotHostForeignCallsProvider;
-import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
-import org.graalvm.compiler.graph.Node.NodeIntrinsic;
-import org.graalvm.compiler.nodes.extended.ForeignCallNode;
-
-// JaCoCo Exclude
-
-/**
- * Substitutions for {@link java.lang.Object} methods.
- */
-@ClassSubstitution(Object.class)
-public class ObjectSubstitutions {
-
-    @MethodSubstitution(isStatic = false)
-    public static int hashCode(final Object thisObj) {
-        return IdentityHashCodeNode.identityHashCode(thisObj);
-    }
-
-    @MethodSubstitution(isStatic = false)
-    public static void notify(final Object thisObj) {
-        if (!fastNotifyStub(HotSpotHostForeignCallsProvider.NOTIFY, thisObj)) {
-            notify(thisObj);
-        }
-    }
-
-    @MethodSubstitution(isStatic = false)
-    public static void notifyAll(final Object thisObj) {
-        if (!fastNotifyStub(HotSpotHostForeignCallsProvider.NOTIFY_ALL, thisObj)) {
-            notifyAll(thisObj);
-        }
-    }
-
-    @NodeIntrinsic(ForeignCallNode.class)
-    public static native boolean fastNotifyStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object o);
-}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/UnsafeCopyMemoryNode.java

@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package org.graalvm.compiler.hotspot.replacements;
+
+import static org.graalvm.compiler.nodeinfo.InputType.Memory;
+import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64;
+import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_16;
+
+import org.graalvm.compiler.core.common.type.StampFactory;
+import org.graalvm.compiler.graph.Node;
+import org.graalvm.compiler.graph.NodeClass;
+import org.graalvm.compiler.nodeinfo.NodeInfo;
+import org.graalvm.compiler.nodes.AbstractStateSplit;
+import org.graalvm.compiler.nodes.ValueNode;
+import org.graalvm.compiler.nodes.ValueNodeUtil;
+import org.graalvm.compiler.nodes.memory.MemoryAccess;
+import org.graalvm.compiler.nodes.memory.MemoryCheckpoint;
+import org.graalvm.compiler.nodes.memory.MemoryNode;
+import org.graalvm.compiler.nodes.spi.Lowerable;
+import org.graalvm.compiler.nodes.spi.LoweringTool;
+import jdk.internal.vm.compiler.word.LocationIdentity;
+
+@NodeInfo(cycles = CYCLES_64, size = SIZE_16, allowedUsageTypes = {Memory})
+public class UnsafeCopyMemoryNode extends AbstractStateSplit implements Lowerable, MemoryCheckpoint.Single, MemoryAccess {
+
+    public static final NodeClass<UnsafeCopyMemoryNode> TYPE = NodeClass.create(UnsafeCopyMemoryNode.class);
+
+    @Input ValueNode receiver;
+    @Input ValueNode srcBase;
+    @Input ValueNode srcOffset;
+    @Input ValueNode destBase;
+    @Input ValueNode desOffset;
+    @Input ValueNode bytes;
+
+    @OptionalInput(Memory) Node lastLocationAccess;
+
+    private final boolean guarded;
+
+    public UnsafeCopyMemoryNode(boolean guarded, ValueNode receiver, ValueNode srcBase, ValueNode srcOffset, ValueNode destBase, ValueNode desOffset,
+                    ValueNode bytes) {
+        super(TYPE, StampFactory.forVoid());
+        this.guarded = guarded;
+        this.receiver = receiver;
+        this.srcBase = srcBase;
+        this.srcOffset = srcOffset;
+        this.destBase = destBase;
+        this.desOffset = desOffset;
+        this.bytes = bytes;
+    }
+
+    public boolean isGuarded() {
+        return guarded;
+    }
+
+    @Override
+    public LocationIdentity getKilledLocationIdentity() {
+        return LocationIdentity.any();
+    }
+
+    @Override
+    public void lower(LoweringTool tool) {
+        tool.getLowerer().lower(this, tool);
+    }
+
+    @Override
+    public LocationIdentity getLocationIdentity() {
+        return getKilledLocationIdentity();
+    }
+
+    @Override
+    public MemoryNode getLastLocationAccess() {
+        return (MemoryNode) lastLocationAccess;
+    }
+
+    @Override
+    public void setLastLocationAccess(MemoryNode lla) {
+        Node newLla = ValueNodeUtil.asNode(lla);
+        updateUsages(lastLocationAccess, newLla);
+        lastLocationAccess = newLla;
+    }
+
+    @NodeIntrinsic
+    public static native void copyMemory(@ConstantNodeParameter boolean guarded, Object receiver, Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes);
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/UnsafeSnippets.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package org.graalvm.compiler.hotspot.replacements;
+
+import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.doingUnsafeAccessOffset;
+import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
+
+import org.graalvm.compiler.api.replacements.Snippet;
+import org.graalvm.compiler.debug.DebugHandlersFactory;
+import org.graalvm.compiler.hotspot.HotSpotBackend;
+import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
+import org.graalvm.compiler.hotspot.nodes.CurrentJavaThreadNode;
+import org.graalvm.compiler.nodes.ComputeObjectAddressNode;
+import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.nodes.spi.LoweringTool;
+import org.graalvm.compiler.options.OptionValues;
+import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
+import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
+import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
+import org.graalvm.compiler.replacements.SnippetTemplate;
+import org.graalvm.compiler.replacements.Snippets;
+import org.graalvm.compiler.serviceprovider.JavaVersionUtil;
+import org.graalvm.compiler.word.Word;
+import jdk.internal.vm.compiler.word.LocationIdentity;
+import jdk.internal.vm.compiler.word.WordFactory;
+
+import jdk.vm.ci.code.TargetDescription;
+
+public class UnsafeSnippets implements Snippets {
+
+    public static final String copyMemoryName = JavaVersionUtil.JAVA_SPEC <= 8 ? "copyMemory" : "copyMemory0";
+
+    @SuppressWarnings("unused")
+    @Snippet
+    static void copyMemory(Object receiver, Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes) {
+        Word srcAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(srcBase, srcOffset));
+        Word dstAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(destBase, destOffset));
+        Word size = WordFactory.signed(bytes);
+
+        HotSpotBackend.unsafeArraycopy(srcAddr, dstAddr, size);
+    }
+
+    @SuppressWarnings("unused")
+    @Snippet
+    static void copyMemoryGuarded(Object receiver, Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes) {
+        Word srcAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(srcBase, srcOffset));
+        Word dstAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(destBase, destOffset));
+        Word size = WordFactory.signed(bytes);
+        Word javaThread = CurrentJavaThreadNode.get();
+        int offset = doingUnsafeAccessOffset(INJECTED_VMCONFIG);
+        LocationIdentity any = LocationIdentity.any();
+
+        /* Set doingUnsafeAccess to guard and handle unsafe memory access failures */
+        javaThread.writeByte(offset, (byte) 1, any);
+        HotSpotBackend.unsafeArraycopy(srcAddr, dstAddr, size);
+        /* Reset doingUnsafeAccess */
+        javaThread.writeByte(offset, (byte) 0, any);
+    }
+
+    public static class Templates extends AbstractTemplates {
+        private final SnippetInfo copyMemory = snippet(UnsafeSnippets.class, "copyMemory");
+        private final SnippetInfo copyMemoryGuarded = snippet(UnsafeSnippets.class, "copyMemoryGuarded");
+
+        public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, HotSpotProviders providers, TargetDescription target) {
+            super(options, factories, providers, providers.getSnippetReflection(), target);
+        }
+
+        public void lower(UnsafeCopyMemoryNode copyMemoryNode, LoweringTool tool) {
+            StructuredGraph graph = copyMemoryNode.graph();
+            Arguments args = new Arguments(copyMemoryNode.isGuarded() ? copyMemoryGuarded : copyMemory, graph.getGuardsStage(), tool.getLoweringStage());
+            args.add("receiver", copyMemoryNode.receiver);
+            args.add("srcBase", copyMemoryNode.srcBase);
+            args.add("srcOffset", copyMemoryNode.srcOffset);
+            args.add("destBase", copyMemoryNode.destBase);
+            args.add("destOffset", copyMemoryNode.desOffset);
+            args.add("bytes", copyMemoryNode.bytes);
+            SnippetTemplate template = template(copyMemoryNode, args);
+            template.instantiate(providers.getMetaAccess(), copyMemoryNode, DEFAULT_REPLACER, args);
+        }
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BciBlockMapping.java

@@ -1079,7 +1079,7 @@ public final class BciBlockMapping {
                 BciBlock successor = block.getSuccessor(i);
                 JsrScope nextScope = scope;
                 if (successor == block.getJsrSuccessor()) {
-                    nextScope = scope.push(block.getJsrReturnBci());
+                    nextScope = scope.push(block.getJsrReturnBci(), successor);
                 }
                 if (successor == block.getRetSuccessor()) {
                     nextScope = scope.pop();
@@ -1109,12 +1109,25 @@ public final class BciBlockMapping {
             }
         }
         for (BciBlock successor : block.getSuccessors()) {
-            if (!jsrVisited.contains(successor)) {
+            if (!jsrVisited.contains(successor) && shouldFollowEdge(successor, scope)) {
                 createJsrAlternatives(blockMap, successor);
             }
         }
     }
 
+    private static boolean shouldFollowEdge(BciBlock successor, JsrScope scope) {
+        if (successor instanceof ExceptionDispatchBlock && scope.getJsrEntryBlock() != null) {
+            ExceptionDispatchBlock exceptionDispatchBlock = (ExceptionDispatchBlock) successor;
+            int bci = scope.getJsrEntryBlock().startBci;
+            if (exceptionDispatchBlock.handler.getStartBCI() < bci && bci < exceptionDispatchBlock.handler.getEndBCI()) {
+                // Handler covers start of JSR block and the bci before that => don't follow edge.
+                return false;
+            }
+        }
+
+        return true;
+    }
+
     private ExceptionDispatchBlock handleExceptions(BciBlock[] blockMap, int bci) {
         ExceptionDispatchBlock lastHandler = null;
         int dispatchBlocks = 0;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BytecodeParser.java

@@ -32,7 +32,6 @@ import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateRecompile;
 import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateReprofile;
 import static jdk.vm.ci.meta.DeoptimizationAction.None;
 import static jdk.vm.ci.meta.DeoptimizationReason.ClassCastException;
-import static jdk.vm.ci.meta.DeoptimizationReason.JavaSubroutineMismatch;
 import static jdk.vm.ci.meta.DeoptimizationReason.NullCheckException;
 import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;
 import static jdk.vm.ci.meta.DeoptimizationReason.UnreachedCode;
@@ -684,6 +683,114 @@ public class BytecodeParser implements GraphBuilderContext {
                 // Restore the original return value
                 parser.frameState.push(returnKind, returnValue);
             }
+            boolean inlinedIntrinsic = parser.getInvokeReturnType() != null;
+            if (inlinedIntrinsic) {
+                for (Node n : parser.graph.getNewNodes(mark)) {
+                    if (n instanceof FrameState) {
+                        GraalError.guarantee(((FrameState) n).bci != BytecodeFrame.INVALID_FRAMESTATE_BCI,
+                                        "Inlined call to intrinsic (callee %s) produced invalid framestate %s. " +
+                                                        "Such framestates must never be used as deoptimizing targets, thus they cannot be part of a high-tier graph, " +
+                                                        "and must only be used after framestate assignment. A common error is invalid usage of foreign call nodes in method " +
+                                                        "substitutions, which can be avoided by ensuring such calls are either replaced with nodes that are snippet " +
+                                                        "lowered after framestate assignment (see FastNotifyNode.java for example) or by ensuring all foreign use the state after of the " +
+                                                        "original call instruction.",
+                                        callee, n);
+                    }
+                }
+            } else {
+
+                /*
+                 * Special case root compiled method substitutions
+                 *
+                 * Root compiled intrinsics with self recursive calls (partial intrinsic exit) must
+                 * never produce more than one state except the start framestate since we do not
+                 * compile calls to the original method (or inline them) but deopt
+                 *
+                 * See ByteCodeParser::inline and search for compilationRoot
+                 */
+                assert intrinsic == null || intrinsic.isIntrinsicEncoding() || verifyIntrinsicRootCompileEffects();
+            }
+        }
+
+        private boolean verifyIntrinsicRootCompileEffects() {
+            int invalidBCIsInRootCompiledIntrinsic = 0;
+            for (Node n : parser.graph.getNewNodes(mark)) {
+                if (n instanceof FrameState) {
+                    if (((FrameState) n).bci == BytecodeFrame.INVALID_FRAMESTATE_BCI) {
+                        invalidBCIsInRootCompiledIntrinsic++;
+                    }
+                }
+            }
+            if (invalidBCIsInRootCompiledIntrinsic > 1) {
+                int invalidBCIsToFind = invalidBCIsInRootCompiledIntrinsic;
+                List<ReturnNode> returns = parser.getGraph().getNodes(ReturnNode.TYPE).snapshot();
+                if (returns.size() > 1) {
+                    outer: for (ReturnNode ret : returns) {
+                        for (FixedNode f : GraphUtil.predecessorIterable(ret)) {
+                            if (f instanceof StateSplit) {
+                                StateSplit split = (StateSplit) f;
+                                if (split.hasSideEffect()) {
+                                    assert ((StateSplit) f).stateAfter() != null;
+                                    if (split.stateAfter().bci == BytecodeFrame.INVALID_FRAMESTATE_BCI) {
+                                        invalidBCIsToFind--;
+                                        continue outer;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    GraalError.guarantee(invalidBCIsToFind == 0, "Root compiled intrinsic with invalid states has more than one return. " +
+                                    "This is allowed, however one path down a sink has more than one state, this is prohibited. " +
+                                    "Intrinsic %s", parser.method);
+                    return true;
+                }
+                ReturnNode ret = returns.get(0);
+                MergeNode merge = null;
+                int mergeCount = parser.graph.getNodes(MergeNode.TYPE).count();
+                if (mergeCount != 1) {
+                    throw new GraalError("Root compiled intrinsic with invalid states %s:Must have exactly one merge node. %d found", parser.method, mergeCount);
+                }
+                if (ret.predecessor() instanceof MergeNode) {
+                    merge = (MergeNode) ret.predecessor();
+                }
+                if (merge == null) {
+                    throw new GraalError("Root compiled intrinsic with invalid state: Unexpected node between return and merge.");
+                }
+                //@formatter:off
+                GraalError.guarantee(invalidBCIsInRootCompiledIntrinsic <= merge.phiPredecessorCount() + 1 /* merge itself */,
+                                "Root compiled intrinsic with invalid states %s must at maximum produce (0,1 or if the last instruction is a merge |merge.predCount|" +
+                                                " invalid BCI state, however %d where found.",
+                                parser.method, invalidBCIsInRootCompiledIntrinsic);
+                //@formatter:on
+                if (merge.stateAfter() != null && merge.stateAfter().bci == BytecodeFrame.INVALID_FRAMESTATE_BCI) {
+                    invalidBCIsToFind--;
+                }
+                for (EndNode pred : merge.cfgPredecessors()) {
+                    Node lastPred = pred.predecessor();
+                    for (FixedNode f : GraphUtil.predecessorIterable((FixedNode) lastPred)) {
+                        if (f instanceof StateSplit) {
+                            StateSplit split = (StateSplit) f;
+                            if (split.hasSideEffect()) {
+                                assert ((StateSplit) f).stateAfter() != null;
+                                if (split.stateAfter().bci == BytecodeFrame.INVALID_FRAMESTATE_BCI) {
+                                    invalidBCIsToFind--;
+                                }
+                            }
+                        }
+                    }
+                }
+                if (invalidBCIsToFind != 0) {
+                    throw new GraalError(
+                                    "Invalid BCI state missmatch: This root compiled method substitution %s " +
+                                                    "uses invalid side-effecting nodes resulting in invalid deoptimization information. " +
+                                                    "Method substitutions must never have more than one state (the after state) for deoptimization." +
+                                                    " Multiple states are only allowed if they are dominated by a control-flow split, there is only" +
+                                                    " a single effect per branch and a post dominating merge with the same invalid_bci state " +
+                                                    "(that must only be different in its return value).",
+                                    parser.method);
+                }
+            }
+            return true;
         }
 
         private void updateSplitFrameState(StateSplit split, JavaKind returnKind, ValueNode returnValue) {
@@ -1820,7 +1927,6 @@ public class BytecodeParser implements GraphBuilderContext {
         } finally {
             currentInvoke = null;
         }
-
         int invokeBci = bci();
         JavaTypeProfile profile = getProfileForInvoke(invokeKind);
         ExceptionEdgeAction edgeAction = getActionForInvokeExceptionEdge(inlineInfo);
@@ -2738,8 +2844,9 @@ public class BytecodeParser implements GraphBuilderContext {
         int retAddress = scope.nextReturnAddress();
         ConstantNode returnBciNode = getJsrConstant(retAddress);
         LogicNode guard = IntegerEqualsNode.create(getConstantReflection(), getMetaAccess(), options, null, local, returnBciNode, NodeView.DEFAULT);
-        guard = graph.addOrUniqueWithInputs(guard);
-        append(new FixedGuardNode(guard, JavaSubroutineMismatch, InvalidateReprofile));
+        if (!guard.isTautology()) {
+            throw new JsrNotSupportedBailout("cannot statically decide jsr return address " + local);
+        }
         if (!successor.getJsrScope().equals(scope.pop())) {
             throw new JsrNotSupportedBailout("unstructured control flow (ret leaves more than one scope)");
         }
@@ -3436,7 +3543,6 @@ public class BytecodeParser implements GraphBuilderContext {
             probability = getProfileProbability(canonicalizedCondition.mustNegate());
         }
 
-        probability = clampProbability(probability);
         genIf(condition, trueSuccessor, falseSuccessor, probability);
     }
 
@@ -3458,10 +3564,10 @@ public class BytecodeParser implements GraphBuilderContext {
             // the probability coming from profile is about the original condition
             probability = 1 - probability;
         }
-        return probability;
+        return clampProbability(probability);
     }
 
-    private static double extractInjectedProbability(IntegerEqualsNode condition) {
+    private double extractInjectedProbability(IntegerEqualsNode condition) {
         // Propagate injected branch probability if any.
         IntegerEqualsNode equalsNode = condition;
         BranchProbabilityNode probabilityNode = null;
@@ -3475,7 +3581,7 @@ public class BytecodeParser implements GraphBuilderContext {
         }
 
         if (probabilityNode != null && probabilityNode.getProbability().isConstant() && other != null && other.isConstant()) {
-            double probabilityValue = probabilityNode.getProbability().asJavaConstant().asDouble();
+            double probabilityValue = clampProbability(probabilityNode.getProbability().asJavaConstant().asDouble());
             return other.asJavaConstant().asInt() == 0 ? 1.0 - probabilityValue : probabilityValue;
         }
         return -1;
@@ -4166,11 +4272,15 @@ public class BytecodeParser implements GraphBuilderContext {
 
     private JavaMethod lookupMethod(int cpi, int opcode) {
         maybeEagerlyResolve(cpi, opcode);
-        JavaMethod result = constantPool.lookupMethod(cpi, opcode);
+        JavaMethod result = lookupMethodInPool(cpi, opcode);
         assert !graphBuilderConfig.unresolvedIsError() || result instanceof ResolvedJavaMethod : unresolvedMethodAssertionMessage(result);
         return result;
     }
 
+    protected JavaMethod lookupMethodInPool(int cpi, int opcode) {
+        return constantPool.lookupMethod(cpi, opcode);
+    }
+
     protected JavaField lookupField(int cpi, int opcode) {
         maybeEagerlyResolve(cpi, opcode);
         JavaField result = constantPool.lookupField(cpi, method, opcode);
@@ -4319,6 +4429,7 @@ public class BytecodeParser implements GraphBuilderContext {
         }
     }
 
+    @SuppressWarnings("try")
     protected void genInstanceOf(ResolvedJavaType resolvedType, ValueNode objectIn) {
         ValueNode object = objectIn;
         TypeReference checkedType = TypeReference.createTrusted(graph.getAssumptions(), resolvedType);
@@ -4353,18 +4464,20 @@ public class BytecodeParser implements GraphBuilderContext {
         int value = getStream().readUByte(next);
         if (next <= currentBlock.endBci && (value == Bytecodes.IFEQ || value == Bytecodes.IFNE)) {
             getStream().next();
-            BciBlock firstSucc = currentBlock.getSuccessor(0);
-            BciBlock secondSucc = currentBlock.getSuccessor(1);
-            if (firstSucc != secondSucc) {
-                boolean negate = value != Bytecodes.IFNE;
-                if (negate) {
-                    BciBlock tmp = firstSucc;
-                    firstSucc = secondSucc;
-                    secondSucc = tmp;
+            try (DebugCloseable context = openNodeContext()) {
+                BciBlock firstSucc = currentBlock.getSuccessor(0);
+                BciBlock secondSucc = currentBlock.getSuccessor(1);
+                if (firstSucc != secondSucc) {
+                    boolean negate = value != Bytecodes.IFNE;
+                    if (negate) {
+                        BciBlock tmp = firstSucc;
+                        firstSucc = secondSucc;
+                        secondSucc = tmp;
+                    }
+                    genIf(instanceOfNode, firstSucc, secondSucc, getProfileProbability(negate));
+                } else {
+                    appendGoto(firstSucc);
                 }
-                genIf(instanceOfNode, firstSucc, secondSucc, getProfileProbability(negate));
-            } else {
-                appendGoto(firstSucc);
             }
         } else {
             // Most frequent for value is IRETURN, followed by ISTORE.

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/JsrScope.java

@@ -25,6 +25,7 @@
 package org.graalvm.compiler.java;
 
 import org.graalvm.compiler.bytecode.Bytecodes;
+import org.graalvm.compiler.java.BciBlockMapping.BciBlock;
 
 /**
  * Represents a subroutine entered via {@link Bytecodes#JSR} and exited via {@link Bytecodes#RET}.
@@ -40,34 +41,46 @@ public final class JsrScope {
 
     private final JsrScope parent;
 
-    private JsrScope(int returnBci, JsrScope parent) {
+    private final BciBlock jsrEntryBlock;
+
+    private JsrScope(int returnBci, BciBlock jsrEntryBlock, JsrScope parent) {
         this.returnAddress = (char) returnBci;
         this.parent = parent;
+        this.jsrEntryBlock = jsrEntryBlock;
     }
 
     private JsrScope() {
         this.returnAddress = 0;
         this.parent = null;
+        this.jsrEntryBlock = null;
     }
 
     public int nextReturnAddress() {
         return returnAddress;
     }
 
+    public BciBlock getJsrEntryBlock() {
+        return jsrEntryBlock;
+    }
+
     /**
      * Enters a new subroutine from the current scope represented by this object.
      *
      * @param returnBci the bytecode address returned to when leaving the new scope
      * @return an object representing the newly entered scope
      */
-    public JsrScope push(int returnBci) {
+    public JsrScope push(int returnBci, BciBlock newJsrEntryBlock) {
         if (returnBci == 0) {
             throw new IllegalArgumentException("A bytecode subroutine cannot have a return address of 0");
         }
         if (returnBci < 1 || returnBci > 0xFFFF) {
             throw new IllegalArgumentException("Bytecode subroutine return address cannot be encoded as a char: " + returnBci);
         }
-        return new JsrScope(returnBci, this);
+        return new JsrScope(returnBci, newJsrEntryBlock, this);
+    }
+
+    public JsrScope push(int returnBci) {
+        return push(returnBci, null);
     }
 
     /**
@@ -85,13 +98,13 @@ public final class JsrScope {
      *         {@code int[]} with {@code value.chars().toArray()}.
      */
     public String getAncestry() {
-        StringBuilder sb = new StringBuilder();
+        String result = "";
         for (JsrScope s = this; s != null; s = s.parent) {
             if (!s.isEmpty()) {
-                sb.append(s.returnAddress);
+                result = s.returnAddress + result;
             }
         }
-        return sb.reverse().toString();
+        return result;
     }
 
     /**

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathPowOp.java

@@ -113,7 +113,7 @@ public final class AMD64MathPowOp extends AMD64MathIntrinsicBinaryOp {
 
     public AMD64MathPowOp() {
         super(TYPE, /* GPR */ rax, rcx, rdx, r8, r9, r10, r11,
-                        /* XMM */ xmm2, xmm3, xmm4, xmm5, xmm6, xmm7);
+                        /* XMM */ xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7);
     }
 
     private ArrayDataPointerConstant highsigmask = pointerConstant(16, new int[]{

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/vector/AVXBlendOp.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package org.graalvm.compiler.lir.amd64.vector;
+
+import static jdk.vm.ci.code.ValueUtil.asRegister;
+import static jdk.vm.ci.code.ValueUtil.isRegister;
+import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG;
+import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK;
+
+import org.graalvm.compiler.asm.amd64.AMD64Address;
+import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler;
+import org.graalvm.compiler.asm.amd64.AVXKind;
+import org.graalvm.compiler.asm.amd64.AMD64Assembler.VexRVMROp;
+import org.graalvm.compiler.lir.LIRInstructionClass;
+import org.graalvm.compiler.lir.Opcode;
+import org.graalvm.compiler.lir.amd64.AMD64LIRInstruction;
+import org.graalvm.compiler.lir.asm.CompilationResultBuilder;
+
+import jdk.vm.ci.meta.AllocatableValue;
+
+public class AVXBlendOp extends AMD64LIRInstruction {
+    public static final LIRInstructionClass<AVXBlendOp> TYPE = LIRInstructionClass.create(AVXBlendOp.class);
+
+    @Opcode private final VexRVMROp opcode;
+    private final AVXKind.AVXSize size;
+
+    @Def({REG}) protected AllocatableValue result;
+    @Use({REG}) protected AllocatableValue x;
+    @Use({REG, STACK}) protected AllocatableValue y;
+    @Use({REG}) protected AllocatableValue mask;
+
+    public AVXBlendOp(VexRVMROp opcode, AVXKind.AVXSize size, AllocatableValue result, AllocatableValue x, AllocatableValue y, AllocatableValue mask) {
+        super(TYPE);
+        this.opcode = opcode;
+        this.size = size;
+        this.result = result;
+        this.x = x;
+        this.y = y;
+        this.mask = mask;
+    }
+
+    @Override
+    public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
+        if (isRegister(y)) {
+            opcode.emit(masm, size, asRegister(result), asRegister(mask), asRegister(x), asRegister(y));
+        } else {
+            opcode.emit(masm, size, asRegister(result), asRegister(mask), asRegister(x), (AMD64Address) crb.asAddress(y));
+        }
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/LIRGeneratorTool.java

@@ -279,7 +279,10 @@ public interface LIRGeneratorTool extends DiagnosticLIRGeneratorTool, ValueKindF
         throw GraalError.unimplemented("String.compareTo substitution is not implemented on this architecture");
     }
 
-    Variable emitArrayEquals(JavaKind kind, Value array1, Value array2, Value length, boolean directPointers);
+    @SuppressWarnings("unused")
+    default Variable emitArrayEquals(JavaKind kind, Value array1, Value array2, Value length, boolean directPointers) {
+        throw GraalError.unimplemented("Array.equals substitution is not implemented on this architecture");
+    }
 
     @SuppressWarnings("unused")
     default Variable emitArrayEquals(JavaKind kind1, JavaKind kind2, Value array1, Value array2, Value length, boolean directPointers) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes.test/src/org/graalvm/compiler/nodes/test/StampToolTest.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package org.graalvm.compiler.nodes.test;
+
+import static org.graalvm.compiler.nodes.type.StampTool.stampForTrailingZeros;
+import static org.graalvm.compiler.test.GraalTest.assertTrue;
+
+import org.graalvm.compiler.core.common.type.IntegerStamp;
+import org.graalvm.compiler.core.common.type.Stamp;
+import org.graalvm.compiler.core.common.type.StampFactory;
+import org.graalvm.compiler.graph.test.GraphTest;
+import org.junit.Test;
+
+public class StampToolTest extends GraphTest {
+
+    @Test
+    public void testStampForTrailingZeros() {
+        assertIntegerStampEquals(stampForTrailingZeros(forInt(0)), 32);
+        assertIntegerStampEquals(stampForTrailingZeros(forInt(1)), 0);
+        assertIntegerStampEquals(stampForTrailingZeros(forInt(-1)), 0);
+        assertIntegerStampEquals(stampForTrailingZeros(forInt(Integer.MIN_VALUE)), 31);
+        assertIntegerStampEquals(stampForTrailingZeros(forInt(Integer.MAX_VALUE)), 0);
+    }
+
+    private static IntegerStamp forInt(int value) {
+        return StampFactory.forInteger(32, value, value);
+    }
+
+    private static void assertIntegerStampEquals(Stamp stamp, int value) {
+        assertTrue(stamp instanceof IntegerStamp);
+        IntegerStamp iStamp = (IntegerStamp) stamp;
+        assertTrue(iStamp.lowerBound() == value);
+        assertTrue(iStamp.upperBound() == value);
+    }
+
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GraphDecoder.java

@@ -139,7 +139,7 @@ public class GraphDecoder {
                 maxFixedNodeOrderId = 0;
             }
 
-            if (loopExplosion != LoopExplosionKind.NONE) {
+            if (loopExplosion.useExplosion()) {
                 loopExplosionMerges = EconomicSet.create(Equivalence.IDENTITY);
             } else {
                 loopExplosionMerges = null;
@@ -196,12 +196,11 @@ public class GraphDecoder {
         protected LoopScope(MethodScope methodScope) {
             this.methodScope = methodScope;
             this.outer = null;
-            this.nextIterations = methodScope.loopExplosion == LoopExplosionKind.FULL_EXPLODE_UNTIL_RETURN ? new ArrayDeque<>(2) : null;
+            this.nextIterations = methodScope.loopExplosion.duplicateLoopExits() ? new ArrayDeque<>(2) : null;
             this.loopDepth = 0;
             this.loopIteration = 0;
             this.iterationStates = null;
             this.loopBeginOrderId = -1;
-
             int nodeCount = methodScope.encodedGraph.nodeStartOffsets.length;
             this.nodesToProcess = new BitSet(methodScope.maxFixedNodeOrderId);
             this.createdNodes = new Node[nodeCount];
@@ -433,7 +432,7 @@ public class GraphDecoder {
             /*
              * Finished with an inlined method. Perform end-of-method cleanup tasks.
              */
-            if (methodScope.loopExplosion == LoopExplosionKind.MERGE_EXPLODE) {
+            if (methodScope.loopExplosion.mergeLoops()) {
                 LoopDetector loopDetector = new LoopDetector(graph, methodScope);
                 loopDetector.run();
             }
@@ -472,8 +471,7 @@ public class GraphDecoder {
         }
 
         if ((node instanceof MergeNode ||
-                        (node instanceof LoopBeginNode && (methodScope.loopExplosion == LoopExplosionKind.FULL_UNROLL || methodScope.loopExplosion == LoopExplosionKind.FULL_EXPLODE ||
-                                        methodScope.loopExplosion == LoopExplosionKind.FULL_EXPLODE_UNTIL_RETURN))) &&
+                        (node instanceof LoopBeginNode && (methodScope.loopExplosion.unrollLoops() && !methodScope.loopExplosion.mergeLoops()))) &&
                         ((AbstractMergeNode) node).forwardEndCount() == 1) {
             AbstractMergeNode merge = (AbstractMergeNode) node;
             EndNode singleEnd = merge.forwardEndAt(0);
@@ -492,7 +490,7 @@ public class GraphDecoder {
         LoopScope successorAddScope = loopScope;
         boolean updatePredecessors = true;
         if (node instanceof LoopExitNode) {
-            if (methodScope.loopExplosion == LoopExplosionKind.FULL_EXPLODE_UNTIL_RETURN || (methodScope.loopExplosion == LoopExplosionKind.MERGE_EXPLODE && loopScope.loopDepth > 1)) {
+            if (methodScope.loopExplosion.duplicateLoopExits() || (methodScope.loopExplosion.mergeLoops() && loopScope.loopDepth > 1)) {
                 /*
                  * We do not want to merge loop exits of inner loops. Instead, we want to keep
                  * exploding the outer loop separately for every loop exit and then merge the outer
@@ -519,7 +517,7 @@ public class GraphDecoder {
             } else {
                 successorAddScope = loopScope.outer;
             }
-            updatePredecessors = methodScope.loopExplosion == LoopExplosionKind.NONE;
+            updatePredecessors = methodScope.loopExplosion.isNoExplosion();
         }
 
         methodScope.reader.setByteIndex(methodScope.encodedGraph.nodeStartOffsets[nodeOrderId]);
@@ -531,12 +529,12 @@ public class GraphDecoder {
 
         LoopScope resultScope = loopScope;
         if (node instanceof LoopBeginNode) {
-            if (methodScope.loopExplosion != LoopExplosionKind.NONE) {
+            if (methodScope.loopExplosion.useExplosion()) {
                 handleLoopExplosionBegin(methodScope, loopScope, (LoopBeginNode) node);
             }
 
         } else if (node instanceof LoopExitNode) {
-            if (methodScope.loopExplosion != LoopExplosionKind.NONE) {
+            if (methodScope.loopExplosion.useExplosion()) {
                 handleLoopExplosionProxyNodes(methodScope, loopScope, successorAddScope, (LoopExitNode) node, nodeOrderId);
             } else {
                 handleProxyNodes(methodScope, loopScope, (LoopExitNode) node);
@@ -549,7 +547,7 @@ public class GraphDecoder {
             LoopScope phiInputScope = loopScope;
             LoopScope phiNodeScope = loopScope;
 
-            if (methodScope.loopExplosion != LoopExplosionKind.NONE && node instanceof LoopEndNode) {
+            if (methodScope.loopExplosion.useExplosion() && node instanceof LoopEndNode) {
                 node = handleLoopExplosionEnd(methodScope, loopScope, (LoopEndNode) node);
                 phiNodeScope = loopScope.nextIterations.getLast();
             }
@@ -562,14 +560,14 @@ public class GraphDecoder {
                 if (merge instanceof LoopBeginNode) {
                     assert phiNodeScope == phiInputScope && phiNodeScope == loopScope;
                     resultScope = new LoopScope(methodScope, loopScope, loopScope.loopDepth + 1, 0, mergeOrderId,
-                                    methodScope.loopExplosion != LoopExplosionKind.NONE ? Arrays.copyOf(loopScope.createdNodes, loopScope.createdNodes.length) : null,
-                                    methodScope.loopExplosion != LoopExplosionKind.NONE ? Arrays.copyOf(loopScope.createdNodes, loopScope.createdNodes.length) : loopScope.createdNodes, //
-                                    methodScope.loopExplosion != LoopExplosionKind.NONE ? new ArrayDeque<>(2) : null, //
-                                    methodScope.loopExplosion == LoopExplosionKind.MERGE_EXPLODE ? EconomicMap.create(Equivalence.DEFAULT) : null);
+                                    methodScope.loopExplosion.useExplosion() ? Arrays.copyOf(loopScope.createdNodes, loopScope.createdNodes.length) : null,
+                                    methodScope.loopExplosion.useExplosion() ? Arrays.copyOf(loopScope.createdNodes, loopScope.createdNodes.length) : loopScope.createdNodes, //
+                                    methodScope.loopExplosion.useExplosion() ? new ArrayDeque<>(2) : null, //
+                                    methodScope.loopExplosion.mergeLoops() ? EconomicMap.create(Equivalence.DEFAULT) : null);
                     phiInputScope = resultScope;
                     phiNodeScope = resultScope;
 
-                    if (methodScope.loopExplosion != LoopExplosionKind.NONE) {
+                    if (methodScope.loopExplosion.useExplosion()) {
                         registerNode(loopScope, mergeOrderId, null, true, true);
                     }
                     loopScope.nodesToProcess.clear(mergeOrderId);
@@ -582,7 +580,6 @@ public class GraphDecoder {
         } else if (node instanceof Invoke) {
             InvokeData invokeData = readInvokeData(methodScope, nodeOrderId, (Invoke) node);
             resultScope = handleInvoke(methodScope, loopScope, invokeData);
-
         } else if (node instanceof ReturnNode || node instanceof UnwindNode) {
             methodScope.returnAndUnwindNodes.add((ControlSinkNode) node);
         } else {
@@ -658,7 +655,7 @@ public class GraphDecoder {
         FixedNode successor = loopBegin.next();
         FrameState frameState = loopBegin.stateAfter();
 
-        if (methodScope.loopExplosion == LoopExplosionKind.MERGE_EXPLODE) {
+        if (methodScope.loopExplosion.mergeLoops()) {
             LoopExplosionState queryState = new LoopExplosionState(frameState, null);
             LoopExplosionState existingState = loopScope.iterationStates.get(queryState);
             if (existingState != null) {
@@ -674,7 +671,7 @@ public class GraphDecoder {
         MergeNode merge = graph.add(new MergeNode());
         methodScope.loopExplosionMerges.add(merge);
 
-        if (methodScope.loopExplosion == LoopExplosionKind.MERGE_EXPLODE) {
+        if (methodScope.loopExplosion.mergeLoops()) {
             if (loopScope.iterationStates.size() == 0 && loopScope.loopDepth == 1) {
                 if (methodScope.loopExplosionHead != null) {
                     throw new PermanentBailoutException("Graal implementation restriction: Method with %s loop explosion must not have more than one top-level loop", LoopExplosionKind.MERGE_EXPLODE);
@@ -725,7 +722,7 @@ public class GraphDecoder {
             merge.addForwardEnd(predecessor);
         }
 
-        if (methodScope.loopExplosion == LoopExplosionKind.MERGE_EXPLODE) {
+        if (methodScope.loopExplosion.mergeLoops()) {
             LoopExplosionState explosionState = new LoopExplosionState(frameState, merge);
             loopScope.iterationStates.put(explosionState, explosionState);
         }
@@ -746,8 +743,8 @@ public class GraphDecoder {
         loopEnd.replaceAtPredecessor(replacementNode);
         loopEnd.safeDelete();
 
-        assert methodScope.loopExplosion != LoopExplosionKind.NONE;
-        if (methodScope.loopExplosion != LoopExplosionKind.FULL_UNROLL || loopScope.nextIterations.isEmpty()) {
+        assert methodScope.loopExplosion.useExplosion();
+        if (methodScope.loopExplosion.duplicateLoopEnds() || loopScope.nextIterations.isEmpty()) {
             int nextIterationNumber = loopScope.nextIterations.isEmpty() ? loopScope.loopIteration + 1 : loopScope.nextIterations.getLast().loopIteration + 1;
             LoopScope nextIterationScope = new LoopScope(methodScope, loopScope.outer, loopScope.loopDepth, nextIterationNumber, loopScope.loopBeginOrderId,
                             Arrays.copyOf(loopScope.initialCreatedNodes, loopScope.initialCreatedNodes.length),
@@ -800,7 +797,7 @@ public class GraphDecoder {
         loopExit.replaceAtPredecessor(begin);
 
         MergeNode loopExitPlaceholder = null;
-        if (methodScope.loopExplosion == LoopExplosionKind.MERGE_EXPLODE && loopScope.loopDepth == 1) {
+        if (methodScope.loopExplosion.mergeLoops() && loopScope.loopDepth == 1) {
             /*
              * This exit might end up as a loop exit of a loop detected after partial evaluation. We
              * need to be able to create a FrameState and the necessary proxy nodes in this case.
@@ -955,7 +952,7 @@ public class GraphDecoder {
          * not processed yet when processing the loop body, we need to create all phi functions
          * upfront.
          */
-        boolean lazyPhi = allowLazyPhis() && (!(merge instanceof LoopBeginNode) || methodScope.loopExplosion != LoopExplosionKind.NONE);
+        boolean lazyPhi = allowLazyPhis() && (!(merge instanceof LoopBeginNode) || methodScope.loopExplosion.useExplosion());
         int numPhis = methodScope.reader.getUVInt();
         for (int i = 0; i < numPhis; i++) {
             int phiInputOrderId = readOrderId(methodScope);
@@ -974,7 +971,7 @@ public class GraphDecoder {
                  * the stale value because it will never be needed to be merged (we are exploding
                  * until we hit a return).
                  */
-                assert methodScope.loopExplosion == LoopExplosionKind.FULL_EXPLODE_UNTIL_RETURN && phiNodeScope.loopIteration > 0;
+                assert methodScope.loopExplosion.duplicateLoopExits() && phiNodeScope.loopIteration > 0;
                 existing = null;
             }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GuardedValueNode.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -59,6 +59,13 @@ public final class GuardedValueNode extends FloatingGuardedNode implements LIRLo
         this.object = object;
     }
 
+    public static ValueNode create(ValueNode object, GuardingNode guard) {
+        if (guard == null) {
+            return object;
+        }
+        return new GuardedValueNode(object, guard);
+    }
+
     public ValueNode object() {
         return object;
     }
@@ -85,7 +92,7 @@ public final class GuardedValueNode extends FloatingGuardedNode implements LIRLo
 
     @Override
     public Node canonical(CanonicalizerTool tool) {
-        if (getGuard() == null) {
+        if (guard == null) {
             if (stamp(NodeView.DEFAULT).equals(object().stamp(NodeView.DEFAULT))) {
                 return object();
             } else {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/FloatEqualsNode.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -44,6 +44,7 @@ import org.graalvm.compiler.nodes.util.GraphUtil;
 import org.graalvm.compiler.options.OptionValues;
 
 import jdk.vm.ci.meta.ConstantReflectionProvider;
+import jdk.vm.ci.meta.JavaKind;
 import jdk.vm.ci.meta.MetaAccessProvider;
 import jdk.vm.ci.meta.TriState;
 
@@ -54,7 +55,8 @@ public final class FloatEqualsNode extends CompareNode implements BinaryCommutat
 
     public FloatEqualsNode(ValueNode x, ValueNode y) {
         super(TYPE, CanonicalCondition.EQ, false, x, y);
-        assert x.stamp(NodeView.DEFAULT) instanceof FloatStamp && y.stamp(NodeView.DEFAULT) instanceof FloatStamp : x.stamp(NodeView.DEFAULT) + " " + y.stamp(NodeView.DEFAULT);
+        assert !x.getStackKind().isNumericInteger() && x.getStackKind() != JavaKind.Object;
+        assert !y.getStackKind().isNumericInteger() && y.getStackKind() != JavaKind.Object;
         assert x.stamp(NodeView.DEFAULT).isCompatible(y.stamp(NodeView.DEFAULT));
     }
 
@@ -78,15 +80,19 @@ public final class FloatEqualsNode extends CompareNode implements BinaryCommutat
 
     @Override
     public boolean isIdentityComparison() {
-        FloatStamp xStamp = (FloatStamp) x.stamp(NodeView.DEFAULT);
-        FloatStamp yStamp = (FloatStamp) y.stamp(NodeView.DEFAULT);
-        /*
-         * If both stamps have at most one 0.0 and it's the same 0.0 then this is an identity
-         * comparison. FloatStamp isn't careful about tracking the presence of -0.0 so assume that
-         * anything that includes 0.0 might include -0.0. So if either one is non-zero then it's an
-         * identity comparison.
-         */
-        return (!xStamp.contains(0.0) || !yStamp.contains(0.0));
+        Stamp xStamp = x.stamp(NodeView.DEFAULT);
+        Stamp yStamp = y.stamp(NodeView.DEFAULT);
+        if (xStamp instanceof FloatStamp && yStamp instanceof FloatStamp) {
+            /*
+             * If both stamps have at most one 0.0 and it's the same 0.0 then this is an identity
+             * comparison. FloatStamp isn't careful about tracking the presence of -0.0 so assume
+             * that anything that includes 0.0 might include -0.0. So if either one is non-zero then
+             * it's an identity comparison.
+             */
+            return (!((FloatStamp) xStamp).contains(0.0) || !((FloatStamp) yStamp).contains(0.0));
+        } else {
+            return false;
+        }
     }
 
     @Override

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IsNullNode.java

@@ -97,7 +97,7 @@ public final class IsNullNode extends UnaryOpLogicNode implements LIRLowerable,
     @Override
     public boolean verify() {
         assertTrue(getValue() != null, "is null input must not be null");
-        assertTrue(getValue().stamp(NodeView.DEFAULT) instanceof AbstractPointerStamp, "input must be a pointer not %s", getValue().stamp(NodeView.DEFAULT));
+        assertTrue(getValue().stamp(NodeView.DEFAULT).isPointerStamp(), "input must be a pointer not %s", getValue().stamp(NodeView.DEFAULT));
         return super.verify();
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/PointerEqualsNode.java

@@ -69,8 +69,8 @@ public class PointerEqualsNode extends CompareNode implements BinaryCommutative<
 
     protected PointerEqualsNode(NodeClass<? extends PointerEqualsNode> c, ValueNode x, ValueNode y) {
         super(c, CanonicalCondition.EQ, false, x, y);
-        assert x.stamp(NodeView.DEFAULT) instanceof AbstractPointerStamp;
-        assert y.stamp(NodeView.DEFAULT) instanceof AbstractPointerStamp;
+        assert x.stamp(NodeView.DEFAULT).isPointerStamp();
+        assert y.stamp(NodeView.DEFAULT).isPointerStamp();
     }
 
     @Override
@@ -137,9 +137,9 @@ public class PointerEqualsNode extends CompareNode implements BinaryCommutative<
             return LogicConstantNode.tautology();
         } else if (forX.stamp(view).alwaysDistinct(forY.stamp(view))) {
             return LogicConstantNode.contradiction();
-        } else if (((AbstractPointerStamp) forX.stamp(view)).alwaysNull()) {
+        } else if (forX.stamp(view) instanceof AbstractPointerStamp && ((AbstractPointerStamp) forX.stamp(view)).alwaysNull()) {
             return nullSynonym(forY, forX);
-        } else if (((AbstractPointerStamp) forY.stamp(view)).alwaysNull()) {
+        } else if (forY.stamp(view) instanceof AbstractPointerStamp && ((AbstractPointerStamp) forY.stamp(view)).alwaysNull()) {
             return nullSynonym(forX, forY);
         } else {
             return null;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/debug/SideEffectNode.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package org.graalvm.compiler.nodes.debug;
+
+import org.graalvm.compiler.core.common.type.StampFactory;
+import org.graalvm.compiler.graph.NodeClass;
+import org.graalvm.compiler.nodeinfo.InputType;
+import org.graalvm.compiler.nodeinfo.NodeCycles;
+import org.graalvm.compiler.nodeinfo.NodeInfo;
+import org.graalvm.compiler.nodeinfo.NodeSize;
+import org.graalvm.compiler.nodes.NodeView;
+import org.graalvm.compiler.nodes.ValueNode;
+import org.graalvm.compiler.nodes.memory.AbstractMemoryCheckpoint;
+import org.graalvm.compiler.nodes.memory.MemoryCheckpoint;
+import org.graalvm.compiler.nodes.spi.LIRLowerable;
+import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool;
+import jdk.internal.vm.compiler.word.LocationIdentity;
+
+/**
+ * Debug node that can be used when an arbitrary side-effect and when a
+ * {@link LocationIdentity#ANY_LOCATION} kill is needed.
+ */
+@NodeInfo(cycles = NodeCycles.CYCLES_IGNORED, size = NodeSize.SIZE_IGNORED, allowedUsageTypes = {InputType.Memory})
+public class SideEffectNode extends AbstractMemoryCheckpoint implements LIRLowerable, MemoryCheckpoint.Single {
+    public static final NodeClass<SideEffectNode> TYPE = NodeClass.create(SideEffectNode.class);
+
+    @OptionalInput ValueNode value;
+
+    public SideEffectNode() {
+        super(TYPE, StampFactory.forVoid());
+    }
+
+    public SideEffectNode(ValueNode value) {
+        super(TYPE, value.stamp(NodeView.DEFAULT));
+        this.value = value;
+    }
+
+    @Override
+    public boolean hasSideEffect() {
+        return true;
+    }
+
+    @Override
+    public void generate(NodeLIRBuilderTool generator) {
+        generator.setResult(this, generator.operand(value));
+    }
+
+    @Override
+    public LocationIdentity getKilledLocationIdentity() {
+        return LocationIdentity.any();
+    }
+
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/gc/BarrierSet.java

@@ -25,8 +25,12 @@
 
 package org.graalvm.compiler.nodes.gc;
 
+import org.graalvm.compiler.nodes.extended.RawLoadNode;
 import org.graalvm.compiler.nodes.memory.FixedAccessNode;
+import org.graalvm.compiler.nodes.memory.HeapAccess;
 
 public interface BarrierSet {
     void addBarriers(FixedAccessNode n);
+
+    HeapAccess.BarrierType readBarrierType(RawLoadNode load);
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/gc/CardTableBarrierSet.java

@@ -31,6 +31,7 @@ import org.graalvm.compiler.nodes.NodeView;
 import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.extended.ArrayRangeWrite;
+import org.graalvm.compiler.nodes.extended.RawLoadNode;
 import org.graalvm.compiler.nodes.java.AbstractCompareAndSwapNode;
 import org.graalvm.compiler.nodes.java.LoweredAtomicReadAndWriteNode;
 import org.graalvm.compiler.nodes.memory.FixedAccessNode;
@@ -47,6 +48,11 @@ public class CardTableBarrierSet implements BarrierSet {
     public CardTableBarrierSet() {
     }
 
+    @Override
+    public BarrierType readBarrierType(RawLoadNode load) {
+        return BarrierType.NONE;
+    }
+
     @Override
     public void addBarriers(FixedAccessNode n) {
         if (n instanceof ReadNode) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/gc/G1BarrierSet.java

@@ -25,12 +25,15 @@
 
 package org.graalvm.compiler.nodes.gc;
 
+import java.lang.ref.Reference;
+
 import org.graalvm.compiler.core.common.type.AbstractObjectStamp;
 import org.graalvm.compiler.debug.GraalError;
 import org.graalvm.compiler.nodes.NodeView;
 import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.extended.ArrayRangeWrite;
+import org.graalvm.compiler.nodes.extended.RawLoadNode;
 import org.graalvm.compiler.nodes.java.AbstractCompareAndSwapNode;
 import org.graalvm.compiler.nodes.java.LoweredAtomicReadAndWriteNode;
 import org.graalvm.compiler.nodes.memory.FixedAccessNode;
@@ -41,8 +44,56 @@ import org.graalvm.compiler.nodes.memory.WriteNode;
 import org.graalvm.compiler.nodes.memory.address.AddressNode;
 import org.graalvm.compiler.nodes.type.StampTool;
 
+import jdk.vm.ci.meta.JavaKind;
+import jdk.vm.ci.meta.MetaAccessProvider;
+import jdk.vm.ci.meta.ResolvedJavaField;
+import jdk.vm.ci.meta.ResolvedJavaType;
+
 public class G1BarrierSet implements BarrierSet {
-    public G1BarrierSet() {
+
+    private final long referentFieldOffset;
+    private final ResolvedJavaType referenceType;
+
+    public G1BarrierSet(MetaAccessProvider metaAccess) {
+        this.referenceType = metaAccess.lookupJavaType(Reference.class);
+        int offset = -1;
+        for (ResolvedJavaField field : referenceType.getInstanceFields(true)) {
+            if (field.getName().equals("referent")) {
+                offset = field.getOffset();
+            }
+        }
+        if (offset == 1) {
+            throw new GraalError("Can't find Reference.referent field");
+        }
+        this.referentFieldOffset = offset;
+    }
+
+    @Override
+    public BarrierType readBarrierType(RawLoadNode load) {
+        if (load.object().getStackKind() == JavaKind.Object &&
+                        load.accessKind() == JavaKind.Object &&
+                        !StampTool.isPointerAlwaysNull(load.object())) {
+            if (load.offset().isJavaConstant() && referentFieldOffset != load.offset().asJavaConstant().asLong()) {
+                // Reading at a constant offset which is different than the referent field.
+                return BarrierType.NONE;
+            }
+            ResolvedJavaType type = StampTool.typeOrNull(load.object());
+            if (type != null && referenceType.isAssignableFrom(type)) {
+                // It's definitely a field of a Reference type
+                if (load.offset().isJavaConstant() && referentFieldOffset == load.offset().asJavaConstant().asLong()) {
+                    // Exactly Reference.referent
+                    return BarrierType.WEAK_FIELD;
+                }
+                // An unknown offset into Reference
+                return BarrierType.MAYBE_WEAK_FIELD;
+            }
+            if (type == null || type.isAssignableFrom(referenceType)) {
+                // The object is a supertype of Reference with an unknown offset or a constant
+                // offset which is the same as Reference.referent.
+                return BarrierType.MAYBE_WEAK_FIELD;
+            }
+        }
+        return BarrierType.NONE;
     }
 
     @Override
@@ -66,9 +117,9 @@ public class G1BarrierSet implements BarrierSet {
     }
 
     private static void addReadNodeBarriers(ReadNode node) {
-        if (node.getBarrierType() == HeapAccess.BarrierType.WEAK_FIELD) {
+        if (node.getBarrierType() == HeapAccess.BarrierType.WEAK_FIELD || node.getBarrierType() == BarrierType.MAYBE_WEAK_FIELD) {
             StructuredGraph graph = node.graph();
-            G1ReferentFieldReadBarrier barrier = graph.add(new G1ReferentFieldReadBarrier(node.getAddress(), node, false));
+            G1ReferentFieldReadBarrier barrier = graph.add(new G1ReferentFieldReadBarrier(node.getAddress(), node, node.getBarrierType() == BarrierType.MAYBE_WEAK_FIELD));
             graph.addAfterFixed(node, barrier);
         }
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/gc/G1ReferentFieldReadBarrier.java

@@ -42,18 +42,18 @@ import org.graalvm.compiler.nodes.memory.address.AddressNode;
 public final class G1ReferentFieldReadBarrier extends ObjectWriteBarrier {
     public static final NodeClass<G1ReferentFieldReadBarrier> TYPE = NodeClass.create(G1ReferentFieldReadBarrier.class);
 
-    private final boolean doLoad;
+    private final boolean dynamicCheck;
 
-    public G1ReferentFieldReadBarrier(AddressNode address, ValueNode expectedObject, boolean doLoad) {
+    public G1ReferentFieldReadBarrier(AddressNode address, ValueNode expectedObject, boolean dynamicCheck) {
         super(TYPE, address, expectedObject, true);
-        this.doLoad = doLoad;
+        this.dynamicCheck = dynamicCheck;
     }
 
     public ValueNode getExpectedObject() {
         return getValue();
     }
 
-    public boolean doLoad() {
-        return doLoad;
+    public boolean isDynamicCheck() {
+        return dynamicCheck;
     }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/IntrinsicContext.java

@@ -28,6 +28,7 @@ import static jdk.vm.ci.code.BytecodeFrame.AFTER_BCI;
 import static jdk.vm.ci.code.BytecodeFrame.AFTER_EXCEPTION_BCI;
 import static jdk.vm.ci.code.BytecodeFrame.BEFORE_BCI;
 import static jdk.vm.ci.code.BytecodeFrame.INVALID_FRAMESTATE_BCI;
+import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.ROOT_COMPILATION_ENCODING;
 import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.INLINE_AFTER_PARSING;
 import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.ROOT_COMPILATION;
 
@@ -136,7 +137,11 @@ public class IntrinsicContext {
     }
 
     public boolean isCompilationRoot() {
-        return compilationContext.equals(ROOT_COMPILATION);
+        return compilationContext.equals(ROOT_COMPILATION) || compilationContext.equals(ROOT_COMPILATION_ENCODING);
+    }
+
+    public boolean isIntrinsicEncoding() {
+        return compilationContext.equals(ROOT_COMPILATION_ENCODING);
     }
 
     public NodeSourcePosition getNodeSourcePosition() {
@@ -166,7 +171,12 @@ public class IntrinsicContext {
         /**
          * An intrinsic is the root of compilation.
          */
-        ROOT_COMPILATION
+        ROOT_COMPILATION,
+
+        /**
+         * An intrinsic is the root of a compilation done for graph encoding.
+         */
+        ROOT_COMPILATION_ENCODING
     }
 
     /**

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/LoopExplosionPlugin.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,35 +32,83 @@ public interface LoopExplosionPlugin extends GraphBuilderPlugin {
         /**
          * No loop explosion.
          */
-        NONE,
+        NONE(false, false, false, false),
         /**
          * Fully unroll all loops. The loops must have a known finite number of iterations. If a
          * loop has multiple loop ends, they are merged so that the subsequent loop iteration is
          * processed only once. For example, a loop with 4 iterations and 2 loop ends leads to
          * 1+1+1+1 = 4 copies of the loop body.
          */
-        FULL_UNROLL,
+        FULL_UNROLL(true, false, false, false),
+        /**
+         * Like {@link #FULL_UNROLL}, but in addition loop unrolling duplicates loop exits in every
+         * iteration instead of merging them. Code after a loop exit is duplicated for every loop
+         * exit and every loop iteration. For example, a loop with 4 iterations and 2 loop exits
+         * (exit1 and exit2, where exit1 is an early return inside a loop) leads to 4 copies of the
+         * loop body and 4 copies of exit1 and 1 copy if exit2. After each exit all code until a
+         * return is duplicated per iteration. Beware of break statements inside loops since they
+         * cause additional loop exits leading to code duplication along exit2.
+         */
+        FULL_UNROLL_UNTIL_RETURN(true, false, true, false),
         /**
          * Fully explode all loops. The loops must have a known finite number of iterations. If a
          * loop has multiple loop ends, they are not merged so that subsequent loop iterations are
          * processed multiple times. For example, a loop with 4 iterations and 2 loop ends leads to
          * 1+2+4+8 = 15 copies of the loop body.
          */
-        FULL_EXPLODE,
+        FULL_EXPLODE(true, true, false, false),
         /**
          * Like {@link #FULL_EXPLODE}, but in addition explosion does not stop at loop exits. Code
          * after the loop is duplicated for every loop exit of every loop iteration. For example, a
          * loop with 4 iterations and 2 loop exits leads to 4 * 2 = 8 copies of the code after the
          * loop.
          */
-        FULL_EXPLODE_UNTIL_RETURN,
+        FULL_EXPLODE_UNTIL_RETURN(true, true, true, false),
         /**
          * like {@link #FULL_EXPLODE}, but copies of the loop body that have the exact same state
          * (all local variables have the same value) are merged. This reduces the number of copies
          * necessary, but can introduce loops again. This kind is useful for bytecode interpreter
          * loops.
          */
-        MERGE_EXPLODE
+        MERGE_EXPLODE(true, true, false, true);
+
+        private final boolean unrollLoops;
+        private final boolean duplicateLoopEnds;
+        private final boolean duplicateLoopExits;
+        private final boolean mergeLoops;
+
+        LoopExplosionKind(boolean unrollLoops, boolean duplicateLoopEnds, boolean duplicateLoopExits, boolean mergeLoops) {
+            this.unrollLoops = unrollLoops;
+            assert !duplicateLoopEnds || unrollLoops;
+            this.duplicateLoopEnds = duplicateLoopEnds;
+            assert !duplicateLoopExits || unrollLoops;
+            this.duplicateLoopExits = duplicateLoopExits;
+            this.mergeLoops = mergeLoops;
+        }
+
+        public boolean unrollLoops() {
+            return unrollLoops;
+        }
+
+        public boolean duplicateLoopExits() {
+            return duplicateLoopExits;
+        }
+
+        public boolean duplicateLoopEnds() {
+            return duplicateLoopEnds;
+        }
+
+        public boolean mergeLoops() {
+            return mergeLoops;
+        }
+
+        public boolean useExplosion() {
+            return this != NONE;
+        }
+
+        public boolean isNoExplosion() {
+            return this == NONE;
+        }
     }
 
     LoopExplosionKind loopExplosionKind(ResolvedJavaMethod method);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/memory/HeapAccess.java

@@ -52,7 +52,12 @@ public interface HeapAccess {
         /**
          * Weak field access (e.g. Hotspot's Reference.referent field).
          */
-        WEAK_FIELD
+        WEAK_FIELD,
+        /**
+         * An access which requires a dynamic check for Weak field access (e.g. Hotspot's
+         * Reference.referent field).
+         */
+        MAYBE_WEAK_FIELD
     }
 
     /**

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/type/StampTool.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -121,9 +121,10 @@ public class StampTool {
     }
 
     public static Stamp stampForTrailingZeros(IntegerStamp valueStamp) {
-        long mask = CodeUtil.mask(valueStamp.getBits());
-        int min = Long.numberOfTrailingZeros(valueStamp.upMask() & mask);
-        int max = Long.numberOfTrailingZeros(valueStamp.downMask() & mask);
+        int bits = valueStamp.getBits();
+        long mask = CodeUtil.mask(bits);
+        int min = Math.min(Long.numberOfTrailingZeros(valueStamp.upMask() & mask), bits);
+        int max = Math.min(Long.numberOfTrailingZeros(valueStamp.downMask() & mask), bits);
         return StampFactory.forInteger(JavaKind.Int, min, max);
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/ConditionalEliminationPhase.java

@@ -25,6 +25,9 @@
 package org.graalvm.compiler.phases.common;
 
 import static org.graalvm.compiler.nodes.StaticDeoptimizingNode.mergeActions;
+import static org.graalvm.compiler.phases.common.ConditionalEliminationUtil.getOtherSafeStamp;
+import static org.graalvm.compiler.phases.common.ConditionalEliminationUtil.getSafeStamp;
+import static org.graalvm.compiler.phases.common.ConditionalEliminationUtil.rewireGuards;
 
 import java.util.ArrayDeque;
 import java.util.Deque;
@@ -69,20 +72,15 @@ import org.graalvm.compiler.nodes.LogicNode;
 import org.graalvm.compiler.nodes.LoopExitNode;
 import org.graalvm.compiler.nodes.MergeNode;
 import org.graalvm.compiler.nodes.NodeView;
-import org.graalvm.compiler.nodes.ParameterNode;
 import org.graalvm.compiler.nodes.PiNode;
 import org.graalvm.compiler.nodes.ProxyNode;
-import org.graalvm.compiler.nodes.ShortCircuitOrNode;
 import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.nodes.StructuredGraph.ScheduleResult;
 import org.graalvm.compiler.nodes.UnaryOpLogicNode;
 import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.ValuePhiNode;
 import org.graalvm.compiler.nodes.calc.AndNode;
-import org.graalvm.compiler.nodes.calc.BinaryArithmeticNode;
-import org.graalvm.compiler.nodes.calc.BinaryNode;
 import org.graalvm.compiler.nodes.calc.IntegerEqualsNode;
-import org.graalvm.compiler.nodes.calc.UnaryNode;
 import org.graalvm.compiler.nodes.cfg.Block;
 import org.graalvm.compiler.nodes.cfg.ControlFlowGraph;
 import org.graalvm.compiler.nodes.extended.GuardingNode;
@@ -96,18 +94,23 @@ import org.graalvm.compiler.nodes.spi.NodeWithState;
 import org.graalvm.compiler.nodes.spi.StampInverter;
 import org.graalvm.compiler.nodes.util.GraphUtil;
 import org.graalvm.compiler.phases.BasePhase;
+import org.graalvm.compiler.phases.common.ConditionalEliminationUtil.GuardFolding;
+import org.graalvm.compiler.phases.common.ConditionalEliminationUtil.GuardRewirer;
+import org.graalvm.compiler.phases.common.ConditionalEliminationUtil.GuardedCondition;
+import org.graalvm.compiler.phases.common.ConditionalEliminationUtil.InfoElement;
+import org.graalvm.compiler.phases.common.ConditionalEliminationUtil.InfoElementProvider;
+import org.graalvm.compiler.phases.common.ConditionalEliminationUtil.InputFilter;
+import org.graalvm.compiler.phases.common.ConditionalEliminationUtil.Marks;
 import org.graalvm.compiler.phases.schedule.SchedulePhase;
 import org.graalvm.compiler.phases.schedule.SchedulePhase.SchedulingStrategy;
 
 import jdk.vm.ci.meta.DeoptimizationAction;
-import jdk.vm.ci.meta.JavaConstant;
 import jdk.vm.ci.meta.SpeculationLog.Speculation;
 import jdk.vm.ci.meta.TriState;
 
 public class ConditionalEliminationPhase extends BasePhase<CoreProviders> {
 
     private static final CounterKey counterStampsRegistered = DebugContext.counter("StampsRegistered");
-    private static final CounterKey counterStampsFound = DebugContext.counter("StampsFound");
     private static final CounterKey counterIfsKilled = DebugContext.counter("CE_KilledIfs");
     private static final CounterKey counterPhiStampsImproved = DebugContext.counter("CE_ImprovedPhis");
     private final boolean fullSchedule;
@@ -257,40 +260,6 @@ public class ConditionalEliminationPhase extends BasePhase<CoreProviders> {
         }
     }
 
-    public static final class Marks {
-        final int infoElementOperations;
-        final int conditions;
-
-        public Marks(int infoElementOperations, int conditions) {
-            this.infoElementOperations = infoElementOperations;
-            this.conditions = conditions;
-        }
-    }
-
-    protected static final class GuardedCondition {
-        private final GuardingNode guard;
-        private final LogicNode condition;
-        private final boolean negated;
-
-        public GuardedCondition(GuardingNode guard, LogicNode condition, boolean negated) {
-            this.guard = guard;
-            this.condition = condition;
-            this.negated = negated;
-        }
-
-        public GuardingNode getGuard() {
-            return guard;
-        }
-
-        public LogicNode getCondition() {
-            return condition;
-        }
-
-        public boolean isNegated() {
-            return negated;
-        }
-    }
-
     public static class Instance implements ControlFlowGraph.RecursiveVisitor<Marks> {
         protected final NodeMap<InfoElement> map;
         protected final BlockMap<List<Node>> blockToNodes;
@@ -300,6 +269,8 @@ public class ConditionalEliminationPhase extends BasePhase<CoreProviders> {
         protected final StructuredGraph graph;
         protected final DebugContext debug;
         protected final EconomicMap<MergeNode, EconomicMap<ValuePhiNode, PhiInfoElement>> mergeMaps;
+        private final InfoElementProvider infoElementProvider;
+        private final GuardFolding guardFolding;
 
         protected final ArrayDeque<GuardedCondition> conditions;
 
@@ -320,10 +291,24 @@ public class ConditionalEliminationPhase extends BasePhase<CoreProviders> {
             tool = GraphUtil.getDefaultSimplifier(context.getMetaAccess(), context.getConstantReflection(), context.getConstantFieldProvider(), false, graph.getAssumptions(), graph.getOptions(),
                             context.getLowerer());
             mergeMaps = EconomicMap.create(Equivalence.IDENTITY);
+            infoElementProvider = new InfoElementProvider() {
+
+                @Override
+                public InfoElement infoElements(ValueNode value) {
+                    return getInfoElements(value);
+                }
+            };
+            guardFolding = new GuardFolding() {
+
+                @Override
+                public boolean foldGuard(DeoptimizingGuard thisGuard, ValueNode original, Stamp newStamp, GuardRewirer rewireGuardFunction) {
+                    return foldPendingTest(thisGuard, original, newStamp, rewireGuardFunction);
+                }
+            };
         }
 
         protected void processConditionAnchor(ConditionAnchorNode node) {
-            tryProveCondition(node.condition(), (guard, result, guardedValueStamp, newInput) -> {
+            tryProveGuardCondition(null, node.condition(), (guard, result, guardedValueStamp, newInput) -> {
                 if (result != node.isNegated()) {
                     node.replaceAtUsages(guard.asNode());
                     GraphUtil.unlinkFixedNode(node);
@@ -443,7 +428,7 @@ public class ConditionalEliminationPhase extends BasePhase<CoreProviders> {
         }
 
         protected void processIf(IfNode node) {
-            tryProveCondition(node.condition(), (guard, result, guardedValueStamp, newInput) -> {
+            tryProveGuardCondition(null, node.condition(), (guard, result, guardedValueStamp, newInput) -> {
                 node.setCondition(LogicConstantNode.forBoolean(result, node.graph()));
                 AbstractBeginNode survivingSuccessor = node.getSuccessor(result);
                 survivingSuccessor.replaceAtUsages(InputType.Guard, guard.asNode());
@@ -601,7 +586,7 @@ public class ConditionalEliminationPhase extends BasePhase<CoreProviders> {
                                     if (input == null) {
                                         input = valueAt;
                                     }
-                                    valueAt = graph.maybeAddOrUnique(PiNode.create(input, curBestStamp, (ValueNode) infoElement.guard));
+                                    valueAt = graph.maybeAddOrUnique(PiNode.create(input, curBestStamp, (ValueNode) infoElement.getGuard()));
                                 }
                                 newPhi.addInput(valueAt);
                             }
@@ -693,60 +678,6 @@ public class ConditionalEliminationPhase extends BasePhase<CoreProviders> {
             registerCondition(condition, negated, guard);
         }
 
-        Pair<InfoElement, Stamp> recursiveFoldStamp(Node node) {
-            if (node instanceof UnaryNode) {
-                UnaryNode unary = (UnaryNode) node;
-                ValueNode value = unary.getValue();
-                InfoElement infoElement = getInfoElements(value);
-                while (infoElement != null) {
-                    Stamp result = unary.foldStamp(infoElement.getStamp());
-                    if (result != null) {
-                        return Pair.create(infoElement, result);
-                    }
-                    infoElement = nextElement(infoElement);
-                }
-            } else if (node instanceof BinaryNode) {
-                BinaryNode binary = (BinaryNode) node;
-                ValueNode y = binary.getY();
-                ValueNode x = binary.getX();
-                if (y.isConstant()) {
-                    InfoElement infoElement = getInfoElements(x);
-                    while (infoElement != null) {
-                        Stamp result = binary.foldStamp(infoElement.stamp, y.stamp(NodeView.DEFAULT));
-                        if (result != null) {
-                            return Pair.create(infoElement, result);
-                        }
-                        infoElement = nextElement(infoElement);
-                    }
-                }
-            }
-            return null;
-        }
-
-        /**
-         * Get the stamp that may be used for the value for which we are registering the condition.
-         * We may directly use the stamp here without restriction, because any later lookup of the
-         * registered info elements is in the same chain of pi nodes.
-         */
-        private static Stamp getSafeStamp(ValueNode x) {
-            return x.stamp(NodeView.DEFAULT);
-        }
-
-        /**
-         * We can only use the stamp of a second value involved in the condition if we are sure that
-         * we are not implicitly creating a dependency on a pi node that is responsible for that
-         * stamp. For now, we are conservatively only using the stamps of constants. Under certain
-         * circumstances, we may also be able to use the stamp of the value after skipping pi nodes
-         * (e.g., the stamp of a parameter after inlining, or the stamp of a fixed node that can
-         * never be replaced with a pi node via canonicalization).
-         */
-        private static Stamp getOtherSafeStamp(ValueNode x) {
-            if (x.isConstant() || x.graph().isAfterFixedReadPhase()) {
-                return x.stamp(NodeView.DEFAULT);
-            }
-            return x.stamp(NodeView.DEFAULT).unrestricted();
-        }
-
         /**
          * Recursively try to fold stamps within this expression using information from
          * {@link #getInfoElements(ValueNode)}. It's only safe to use constants and one
@@ -757,7 +688,7 @@ public class ConditionalEliminationPhase extends BasePhase<CoreProviders> {
          *         expression
          */
         Pair<InfoElement, Stamp> recursiveFoldStampFromInfo(Node node) {
-            return recursiveFoldStamp(node);
+            return ConditionalEliminationUtil.recursiveFoldStamp(infoElementProvider, node);
         }
 
         /**
@@ -873,6 +804,10 @@ public class ConditionalEliminationPhase extends BasePhase<CoreProviders> {
             return false;
         }
 
+        protected boolean tryProveGuardCondition(DeoptimizingGuard thisGuard, LogicNode node, GuardRewirer rewireGuardFunction) {
+            return ConditionalEliminationUtil.tryProveGuardCondition(infoElementProvider, conditions, guardFolding, thisGuard, node, rewireGuardFunction);
+        }
+
         protected void registerCondition(LogicNode condition, boolean negated, GuardingNode guard) {
             if (condition.hasMoreThanOneUsage()) {
                 registerNewStamp(condition, negated ? StampFactory.contradiction() : StampFactory.tautology(), guard);
@@ -891,15 +826,6 @@ public class ConditionalEliminationPhase extends BasePhase<CoreProviders> {
             return infoElement;
         }
 
-        protected boolean rewireGuards(GuardingNode guard, boolean result, ValueNode proxifiedInput, Stamp guardedValueStamp, GuardRewirer rewireGuardFunction) {
-            counterStampsFound.increment(debug);
-            return rewireGuardFunction.rewire(guard, result, guardedValueStamp, proxifiedInput);
-        }
-
-        protected boolean tryProveCondition(LogicNode node, GuardRewirer rewireGuardFunction) {
-            return tryProveGuardCondition(null, node, rewireGuardFunction);
-        }
-
         private InfoElement nextElement(InfoElement current) {
             InfoElement parent = current.getParent();
             if (parent != null) {
@@ -914,166 +840,6 @@ public class ConditionalEliminationPhase extends BasePhase<CoreProviders> {
             return null;
         }
 
-        protected boolean tryProveGuardCondition(DeoptimizingGuard thisGuard, LogicNode node, GuardRewirer rewireGuardFunction) {
-            InfoElement infoElement = getInfoElements(node);
-            while (infoElement != null) {
-                Stamp stamp = infoElement.getStamp();
-                JavaConstant constant = (JavaConstant) stamp.asConstant();
-                if (constant != null) {
-                    // No proxified input and stamp required.
-                    return rewireGuards(infoElement.getGuard(), constant.asBoolean(), null, null, rewireGuardFunction);
-                }
-                infoElement = nextElement(infoElement);
-            }
-
-            for (GuardedCondition guardedCondition : this.conditions) {
-                TriState result = guardedCondition.getCondition().implies(guardedCondition.isNegated(), node);
-                if (result.isKnown()) {
-                    return rewireGuards(guardedCondition.guard, result.toBoolean(), null, null, rewireGuardFunction);
-                }
-            }
-
-            if (node instanceof UnaryOpLogicNode) {
-                UnaryOpLogicNode unaryLogicNode = (UnaryOpLogicNode) node;
-                ValueNode value = unaryLogicNode.getValue();
-                infoElement = getInfoElements(value);
-                while (infoElement != null) {
-                    Stamp stamp = infoElement.getStamp();
-                    TriState result = unaryLogicNode.tryFold(stamp);
-                    if (result.isKnown()) {
-                        return rewireGuards(infoElement.getGuard(), result.toBoolean(), infoElement.getProxifiedInput(), infoElement.getStamp(), rewireGuardFunction);
-                    }
-                    infoElement = nextElement(infoElement);
-                }
-                Pair<InfoElement, Stamp> foldResult = recursiveFoldStampFromInfo(value);
-                if (foldResult != null) {
-                    TriState result = unaryLogicNode.tryFold(foldResult.getRight());
-                    if (result.isKnown()) {
-                        return rewireGuards(foldResult.getLeft().getGuard(), result.toBoolean(), foldResult.getLeft().getProxifiedInput(), foldResult.getRight(), rewireGuardFunction);
-                    }
-                }
-                if (thisGuard != null) {
-                    Stamp newStamp = unaryLogicNode.getSucceedingStampForValue(thisGuard.isNegated());
-                    if (newStamp != null && foldPendingTest(thisGuard, value, newStamp, rewireGuardFunction)) {
-                        return true;
-                    }
-
-                }
-            } else if (node instanceof BinaryOpLogicNode) {
-                BinaryOpLogicNode binaryOpLogicNode = (BinaryOpLogicNode) node;
-                ValueNode x = binaryOpLogicNode.getX();
-                ValueNode y = binaryOpLogicNode.getY();
-                infoElement = getInfoElements(x);
-                while (infoElement != null) {
-                    TriState result = binaryOpLogicNode.tryFold(infoElement.getStamp(), y.stamp(NodeView.DEFAULT));
-                    if (result.isKnown()) {
-                        return rewireGuards(infoElement.getGuard(), result.toBoolean(), infoElement.getProxifiedInput(), infoElement.getStamp(), rewireGuardFunction);
-                    }
-                    infoElement = nextElement(infoElement);
-                }
-
-                if (y.isConstant()) {
-                    Pair<InfoElement, Stamp> foldResult = recursiveFoldStampFromInfo(x);
-                    if (foldResult != null) {
-                        TriState result = binaryOpLogicNode.tryFold(foldResult.getRight(), y.stamp(NodeView.DEFAULT));
-                        if (result.isKnown()) {
-                            return rewireGuards(foldResult.getLeft().getGuard(), result.toBoolean(), foldResult.getLeft().getProxifiedInput(), foldResult.getRight(), rewireGuardFunction);
-                        }
-                    }
-                } else {
-                    infoElement = getInfoElements(y);
-                    while (infoElement != null) {
-                        TriState result = binaryOpLogicNode.tryFold(x.stamp(NodeView.DEFAULT), infoElement.getStamp());
-                        if (result.isKnown()) {
-                            return rewireGuards(infoElement.getGuard(), result.toBoolean(), infoElement.getProxifiedInput(), infoElement.getStamp(), rewireGuardFunction);
-                        }
-                        infoElement = nextElement(infoElement);
-                    }
-                }
-
-                /*
-                 * For complex expressions involving constants, see if it's possible to fold the
-                 * tests by using stamps one level up in the expression. For instance, (x + n < y)
-                 * might fold if something is known about x and all other values are constants. The
-                 * reason for the constant restriction is that if more than 1 real value is involved
-                 * the code might need to adopt multiple guards to have proper dependences.
-                 */
-                if (x instanceof BinaryArithmeticNode<?> && y.isConstant()) {
-                    BinaryArithmeticNode<?> binary = (BinaryArithmeticNode<?>) x;
-                    if (binary.getY().isConstant()) {
-                        infoElement = getInfoElements(binary.getX());
-                        while (infoElement != null) {
-                            Stamp newStampX = binary.foldStamp(infoElement.getStamp(), binary.getY().stamp(NodeView.DEFAULT));
-                            TriState result = binaryOpLogicNode.tryFold(newStampX, y.stamp(NodeView.DEFAULT));
-                            if (result.isKnown()) {
-                                return rewireGuards(infoElement.getGuard(), result.toBoolean(), infoElement.getProxifiedInput(), newStampX, rewireGuardFunction);
-                            }
-                            infoElement = nextElement(infoElement);
-                        }
-                    }
-                }
-
-                if (thisGuard != null && binaryOpLogicNode instanceof IntegerEqualsNode && !thisGuard.isNegated()) {
-                    if (y.isConstant() && x instanceof AndNode) {
-                        AndNode and = (AndNode) x;
-                        if (and.getY() == y) {
-                            /*
-                             * This 'and' proves something about some of the bits in and.getX().
-                             * It's equivalent to or'ing in the mask value since those values are
-                             * known to be set.
-                             */
-                            BinaryOp<Or> op = ArithmeticOpTable.forStamp(x.stamp(NodeView.DEFAULT)).getOr();
-                            IntegerStamp newStampX = (IntegerStamp) op.foldStamp(getSafeStamp(and.getX()), getOtherSafeStamp(y));
-                            if (foldPendingTest(thisGuard, and.getX(), newStampX, rewireGuardFunction)) {
-                                return true;
-                            }
-                        }
-                    }
-                }
-
-                if (thisGuard != null) {
-                    if (!x.isConstant()) {
-                        Stamp newStampX = binaryOpLogicNode.getSucceedingStampForX(thisGuard.isNegated(), getSafeStamp(x), getOtherSafeStamp(y));
-                        if (newStampX != null && foldPendingTest(thisGuard, x, newStampX, rewireGuardFunction)) {
-                            return true;
-                        }
-                    }
-                    if (!y.isConstant()) {
-                        Stamp newStampY = binaryOpLogicNode.getSucceedingStampForY(thisGuard.isNegated(), getOtherSafeStamp(x), getSafeStamp(y));
-                        if (newStampY != null && foldPendingTest(thisGuard, y, newStampY, rewireGuardFunction)) {
-                            return true;
-                        }
-                    }
-                }
-            } else if (node instanceof ShortCircuitOrNode) {
-                final ShortCircuitOrNode shortCircuitOrNode = (ShortCircuitOrNode) node;
-                return tryProveCondition(shortCircuitOrNode.getX(), (guard, result, guardedValueStamp, newInput) -> {
-                    if (result == !shortCircuitOrNode.isXNegated()) {
-                        return rewireGuards(guard, true, newInput, guardedValueStamp, rewireGuardFunction);
-                    } else {
-                        return tryProveCondition(shortCircuitOrNode.getY(), (innerGuard, innerResult, innerGuardedValueStamp, innerNewInput) -> {
-                            ValueNode proxifiedInput = newInput;
-                            if (proxifiedInput == null) {
-                                proxifiedInput = innerNewInput;
-                            } else if (innerNewInput != null) {
-                                if (innerNewInput != newInput) {
-                                    // Cannot canonicalize due to different proxied inputs.
-                                    return false;
-                                }
-                            }
-                            // Can only canonicalize if the guards are equal.
-                            if (innerGuard == guard) {
-                                return rewireGuards(guard, innerResult ^ shortCircuitOrNode.isYNegated(), proxifiedInput, guardedValueStamp, rewireGuardFunction);
-                            }
-                            return false;
-                        });
-                    }
-                });
-            }
-
-            return false;
-        }
-
         protected void registerNewStamp(ValueNode maybeProxiedValue, Stamp newStamp, GuardingNode guard) {
             registerNewStamp(maybeProxiedValue, newStamp, guard, false);
         }
@@ -1179,95 +945,6 @@ public class ConditionalEliminationPhase extends BasePhase<CoreProviders> {
         }
     }
 
-    @FunctionalInterface
-    protected interface InfoElementProvider {
-        Iterable<InfoElement> getInfoElements(ValueNode value);
-    }
-
-    /**
-     * Checks for safe nodes when moving pending tests up.
-     */
-    static class InputFilter extends Node.EdgeVisitor {
-        boolean ok;
-        private ValueNode value;
-
-        InputFilter(ValueNode value) {
-            this.value = value;
-            this.ok = true;
-        }
-
-        @Override
-        public Node apply(Node node, Node curNode) {
-            if (!ok) {
-                // Abort the recursion
-                return curNode;
-            }
-            if (!(curNode instanceof ValueNode)) {
-                ok = false;
-                return curNode;
-            }
-            ValueNode curValue = (ValueNode) curNode;
-            if (curValue.isConstant() || curValue == value || curValue instanceof ParameterNode) {
-                return curNode;
-            }
-            if (curValue instanceof BinaryNode || curValue instanceof UnaryNode) {
-                curValue.applyInputs(this);
-            } else {
-                ok = false;
-            }
-            return curNode;
-        }
-    }
-
-    @FunctionalInterface
-    protected interface GuardRewirer {
-        /**
-         * Called if the condition could be proven to have a constant value ({@code result}) under
-         * {@code guard}.
-         *
-         * @param guard the guard whose result is proven
-         * @param result the known result of the guard
-         * @param newInput new input to pi nodes depending on the new guard
-         * @return whether the transformation could be applied
-         */
-        boolean rewire(GuardingNode guard, boolean result, Stamp guardedValueStamp, ValueNode newInput);
-    }
-
-    protected static final class InfoElement {
-        private final Stamp stamp;
-        private final GuardingNode guard;
-        private final ValueNode proxifiedInput;
-        private final InfoElement parent;
-
-        public InfoElement(Stamp stamp, GuardingNode guard, ValueNode proxifiedInput, InfoElement parent) {
-            this.stamp = stamp;
-            this.guard = guard;
-            this.proxifiedInput = proxifiedInput;
-            this.parent = parent;
-        }
-
-        public InfoElement getParent() {
-            return parent;
-        }
-
-        public Stamp getStamp() {
-            return stamp;
-        }
-
-        public GuardingNode getGuard() {
-            return guard;
-        }
-
-        public ValueNode getProxifiedInput() {
-            return proxifiedInput;
-        }
-
-        @Override
-        public String toString() {
-            return stamp + " -> " + guard;
-        }
-    }
-
     @Override
     public float codeSizeIncrease() {
         return 1.5f;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/ConditionalEliminationUtil.java

@@ -0,0 +1,441 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package org.graalvm.compiler.phases.common;
+
+import java.util.ArrayDeque;
+
+import jdk.internal.vm.compiler.collections.Pair;
+import org.graalvm.compiler.core.common.type.ArithmeticOpTable;
+import org.graalvm.compiler.core.common.type.ArithmeticOpTable.BinaryOp;
+import org.graalvm.compiler.core.common.type.ArithmeticOpTable.BinaryOp.Or;
+import org.graalvm.compiler.core.common.type.IntegerStamp;
+import org.graalvm.compiler.core.common.type.Stamp;
+import org.graalvm.compiler.graph.Node;
+import org.graalvm.compiler.nodes.BinaryOpLogicNode;
+import org.graalvm.compiler.nodes.DeoptimizingGuard;
+import org.graalvm.compiler.nodes.LogicNode;
+import org.graalvm.compiler.nodes.NodeView;
+import org.graalvm.compiler.nodes.ParameterNode;
+import org.graalvm.compiler.nodes.PiNode;
+import org.graalvm.compiler.nodes.ShortCircuitOrNode;
+import org.graalvm.compiler.nodes.UnaryOpLogicNode;
+import org.graalvm.compiler.nodes.ValueNode;
+import org.graalvm.compiler.nodes.calc.AndNode;
+import org.graalvm.compiler.nodes.calc.BinaryArithmeticNode;
+import org.graalvm.compiler.nodes.calc.BinaryNode;
+import org.graalvm.compiler.nodes.calc.IntegerEqualsNode;
+import org.graalvm.compiler.nodes.calc.UnaryNode;
+import org.graalvm.compiler.nodes.extended.GuardingNode;
+
+import jdk.vm.ci.meta.JavaConstant;
+import jdk.vm.ci.meta.TriState;
+
+public class ConditionalEliminationUtil {
+
+    public static final class Marks {
+
+        final int infoElementOperations;
+        final int conditions;
+
+        public Marks(int infoElementOperations, int conditions) {
+            this.infoElementOperations = infoElementOperations;
+            this.conditions = conditions;
+        }
+
+        public int getInfoElementOperations() {
+            return infoElementOperations;
+        }
+
+        public int getConditions() {
+            return conditions;
+        }
+    }
+
+    public static final class GuardedCondition {
+        private final GuardingNode guard;
+        private final LogicNode condition;
+        private final boolean negated;
+
+        public GuardedCondition(GuardingNode guard, LogicNode condition, boolean negated) {
+            this.guard = guard;
+            this.condition = condition;
+            this.negated = negated;
+        }
+
+        public GuardingNode getGuard() {
+            return guard;
+        }
+
+        public LogicNode getCondition() {
+            return condition;
+        }
+
+        public boolean isNegated() {
+            return negated;
+        }
+    }
+
+    @FunctionalInterface
+    public interface GuardRewirer {
+        /**
+         * Called if the condition could be proven to have a constant value ({@code result}) under
+         * {@code guard}.
+         *
+         * @param guard the guard whose result is proven
+         * @param result the known result of the guard
+         * @param newInput new input to pi nodes depending on the new guard
+         * @return whether the transformation could be applied
+         */
+        boolean rewire(GuardingNode guard, boolean result, Stamp guardedValueStamp, ValueNode newInput);
+    }
+
+    /**
+     * Checks for safe nodes when moving pending tests up.
+     */
+    public static class InputFilter extends Node.EdgeVisitor {
+        boolean ok;
+        private ValueNode value;
+
+        InputFilter(ValueNode value) {
+            this.value = value;
+            this.ok = true;
+        }
+
+        @Override
+        public Node apply(Node node, Node curNode) {
+            if (!ok) {
+                // Abort the recursion
+                return curNode;
+            }
+            if (!(curNode instanceof ValueNode)) {
+                ok = false;
+                return curNode;
+            }
+            ValueNode curValue = (ValueNode) curNode;
+            if (curValue.isConstant() || curValue == value || curValue instanceof ParameterNode) {
+                return curNode;
+            }
+            if (curValue instanceof BinaryNode || curValue instanceof UnaryNode) {
+                curValue.applyInputs(this);
+            } else {
+                ok = false;
+            }
+            return curNode;
+        }
+    }
+
+    public static final class InfoElement {
+        private final Stamp stamp;
+        private final GuardingNode guard;
+        private final ValueNode proxifiedInput;
+        private final InfoElement parent;
+
+        public InfoElement(Stamp stamp, GuardingNode guard, ValueNode proxifiedInput, InfoElement parent) {
+            this.stamp = stamp;
+            this.guard = guard;
+            this.proxifiedInput = proxifiedInput;
+            this.parent = parent;
+        }
+
+        public InfoElement getParent() {
+            return parent;
+        }
+
+        public Stamp getStamp() {
+            return stamp;
+        }
+
+        public GuardingNode getGuard() {
+            return guard;
+        }
+
+        public ValueNode getProxifiedInput() {
+            return proxifiedInput;
+        }
+
+        @Override
+        public String toString() {
+            return stamp + " -> " + guard;
+        }
+    }
+
+    /**
+     * Get the stamp that may be used for the value for which we are registering the condition. We
+     * may directly use the stamp here without restriction, because any later lookup of the
+     * registered info elements is in the same chain of pi nodes.
+     */
+    public static Stamp getSafeStamp(ValueNode x) {
+        return x.stamp(NodeView.DEFAULT);
+    }
+
+    /**
+     * We can only use the stamp of a second value involved in the condition if we are sure that we
+     * are not implicitly creating a dependency on a pi node that is responsible for that stamp. For
+     * now, we are conservatively only using the stamps of constants. Under certain circumstances,
+     * we may also be able to use the stamp of the value after skipping pi nodes (e.g., the stamp of
+     * a parameter after inlining, or the stamp of a fixed node that can never be replaced with a pi
+     * node via canonicalization).
+     */
+    public static Stamp getOtherSafeStamp(ValueNode x) {
+        if (x.isConstant() || x.graph().isAfterFixedReadPhase()) {
+            return x.stamp(NodeView.DEFAULT);
+        }
+        return x.stamp(NodeView.DEFAULT).unrestricted();
+    }
+
+    @FunctionalInterface
+    public interface InfoElementProvider {
+        InfoElement infoElements(ValueNode value);
+
+        default InfoElement nextElement(InfoElement current) {
+            InfoElement parent = current.getParent();
+            if (parent != null) {
+                return parent;
+            } else {
+                ValueNode proxifiedInput = current.getProxifiedInput();
+                if (proxifiedInput instanceof PiNode) {
+                    PiNode piNode = (PiNode) proxifiedInput;
+                    return infoElements(piNode.getOriginalNode());
+                }
+            }
+            return null;
+        }
+    }
+
+    public static Pair<InfoElement, Stamp> recursiveFoldStamp(InfoElementProvider infoElementProvider, Node node) {
+        if (node instanceof UnaryNode) {
+            UnaryNode unary = (UnaryNode) node;
+            ValueNode value = unary.getValue();
+            InfoElement infoElement = infoElementProvider.infoElements(value);
+            while (infoElement != null) {
+                Stamp result = unary.foldStamp(infoElement.getStamp());
+                if (result != null) {
+                    return Pair.create(infoElement, result);
+                }
+                infoElement = infoElementProvider.nextElement(infoElement);
+            }
+        } else if (node instanceof BinaryNode) {
+            BinaryNode binary = (BinaryNode) node;
+            ValueNode y = binary.getY();
+            ValueNode x = binary.getX();
+            if (y.isConstant()) {
+                InfoElement infoElement = infoElementProvider.infoElements(x);
+                while (infoElement != null) {
+                    Stamp result = binary.foldStamp(infoElement.getStamp(), y.stamp(NodeView.DEFAULT));
+                    if (result != null) {
+                        return Pair.create(infoElement, result);
+                    }
+                    infoElement = infoElementProvider.nextElement(infoElement);
+                }
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Recursively try to fold stamps within this expression using information from
+     * {@link InfoElementProvider#infoElements(ValueNode)}. It's only safe to use constants and one
+     * {@link InfoElement} otherwise more than one guard would be required.
+     *
+     * @param node
+     * @return the pair of the @{link InfoElement} used and the stamp produced for the whole
+     *         expression
+     */
+    public static Pair<InfoElement, Stamp> recursiveFoldStampFromInfo(InfoElementProvider infoElementProvider, Node node) {
+        return recursiveFoldStamp(infoElementProvider, node);
+    }
+
+    public static boolean rewireGuards(GuardingNode guard, boolean result, ValueNode proxifiedInput, Stamp guardedValueStamp, GuardRewirer rewireGuardFunction) {
+        return rewireGuardFunction.rewire(guard, result, guardedValueStamp, proxifiedInput);
+    }
+
+    @FunctionalInterface
+    public interface GuardFolding {
+        boolean foldGuard(DeoptimizingGuard thisGuard, ValueNode original, Stamp newStamp, GuardRewirer rewireGuardFunction);
+    }
+
+    public static boolean tryProveGuardCondition(InfoElementProvider infoElementProvider, ArrayDeque<GuardedCondition> conditions, GuardFolding guardFolding, DeoptimizingGuard thisGuard,
+                    LogicNode node,
+                    GuardRewirer rewireGuardFunction) {
+        InfoElement infoElement = infoElementProvider.infoElements(node);
+        while (infoElement != null) {
+            Stamp stamp = infoElement.getStamp();
+            JavaConstant constant = (JavaConstant) stamp.asConstant();
+            if (constant != null) {
+                // No proxified input and stamp required.
+                return rewireGuards(infoElement.getGuard(), constant.asBoolean(), null, null, rewireGuardFunction);
+            }
+            infoElement = infoElementProvider.nextElement(infoElement);
+        }
+
+        for (GuardedCondition guardedCondition : conditions) {
+            TriState result = guardedCondition.getCondition().implies(guardedCondition.isNegated(), node);
+            if (result.isKnown()) {
+                return rewireGuards(guardedCondition.getGuard(), result.toBoolean(), null, null, rewireGuardFunction);
+            }
+        }
+
+        if (node instanceof UnaryOpLogicNode) {
+            UnaryOpLogicNode unaryLogicNode = (UnaryOpLogicNode) node;
+            ValueNode value = unaryLogicNode.getValue();
+            infoElement = infoElementProvider.infoElements(value);
+            while (infoElement != null) {
+                Stamp stamp = infoElement.getStamp();
+                TriState result = unaryLogicNode.tryFold(stamp);
+                if (result.isKnown()) {
+                    return rewireGuards(infoElement.getGuard(), result.toBoolean(), infoElement.getProxifiedInput(), infoElement.getStamp(), rewireGuardFunction);
+                }
+                infoElement = infoElementProvider.nextElement(infoElement);
+            }
+            Pair<InfoElement, Stamp> foldResult = recursiveFoldStampFromInfo(infoElementProvider, value);
+            if (foldResult != null) {
+                TriState result = unaryLogicNode.tryFold(foldResult.getRight());
+                if (result.isKnown()) {
+                    return rewireGuards(foldResult.getLeft().getGuard(), result.toBoolean(), foldResult.getLeft().getProxifiedInput(), foldResult.getRight(), rewireGuardFunction);
+                }
+            }
+            if (thisGuard != null && guardFolding != null) {
+                Stamp newStamp = unaryLogicNode.getSucceedingStampForValue(thisGuard.isNegated());
+                if (newStamp != null && guardFolding.foldGuard(thisGuard, value, newStamp, rewireGuardFunction)) {
+                    return true;
+                }
+
+            }
+        } else if (node instanceof BinaryOpLogicNode) {
+            BinaryOpLogicNode binaryOpLogicNode = (BinaryOpLogicNode) node;
+            ValueNode x = binaryOpLogicNode.getX();
+            ValueNode y = binaryOpLogicNode.getY();
+            infoElement = infoElementProvider.infoElements(x);
+            while (infoElement != null) {
+                TriState result = binaryOpLogicNode.tryFold(infoElement.getStamp(), y.stamp(NodeView.DEFAULT));
+                if (result.isKnown()) {
+                    return rewireGuards(infoElement.getGuard(), result.toBoolean(), infoElement.getProxifiedInput(), infoElement.getStamp(), rewireGuardFunction);
+                }
+                infoElement = infoElementProvider.nextElement(infoElement);
+            }
+
+            if (y.isConstant()) {
+                Pair<InfoElement, Stamp> foldResult = recursiveFoldStampFromInfo(infoElementProvider, x);
+                if (foldResult != null) {
+                    TriState result = binaryOpLogicNode.tryFold(foldResult.getRight(), y.stamp(NodeView.DEFAULT));
+                    if (result.isKnown()) {
+                        return rewireGuards(foldResult.getLeft().getGuard(), result.toBoolean(), foldResult.getLeft().getProxifiedInput(), foldResult.getRight(), rewireGuardFunction);
+                    }
+                }
+            } else {
+                infoElement = infoElementProvider.infoElements(y);
+                while (infoElement != null) {
+                    TriState result = binaryOpLogicNode.tryFold(x.stamp(NodeView.DEFAULT), infoElement.getStamp());
+                    if (result.isKnown()) {
+                        return rewireGuards(infoElement.getGuard(), result.toBoolean(), infoElement.getProxifiedInput(), infoElement.getStamp(), rewireGuardFunction);
+                    }
+                    infoElement = infoElementProvider.nextElement(infoElement);
+                }
+            }
+
+            /*
+             * For complex expressions involving constants, see if it's possible to fold the tests
+             * by using stamps one level up in the expression. For instance, (x + n < y) might fold
+             * if something is known about x and all other values are constants. The reason for the
+             * constant restriction is that if more than 1 real value is involved the code might
+             * need to adopt multiple guards to have proper dependences.
+             */
+            if (x instanceof BinaryArithmeticNode<?> && y.isConstant()) {
+                BinaryArithmeticNode<?> binary = (BinaryArithmeticNode<?>) x;
+                if (binary.getY().isConstant()) {
+                    infoElement = infoElementProvider.infoElements(binary.getX());
+                    while (infoElement != null) {
+                        Stamp newStampX = binary.foldStamp(infoElement.getStamp(), binary.getY().stamp(NodeView.DEFAULT));
+                        TriState result = binaryOpLogicNode.tryFold(newStampX, y.stamp(NodeView.DEFAULT));
+                        if (result.isKnown()) {
+                            return rewireGuards(infoElement.getGuard(), result.toBoolean(), infoElement.getProxifiedInput(), newStampX, rewireGuardFunction);
+                        }
+                        infoElement = infoElementProvider.nextElement(infoElement);
+                    }
+                }
+            }
+
+            if (thisGuard != null && guardFolding != null && binaryOpLogicNode instanceof IntegerEqualsNode && !thisGuard.isNegated()) {
+                if (y.isConstant() && x instanceof AndNode) {
+                    AndNode and = (AndNode) x;
+                    if (and.getY() == y) {
+                        /*
+                         * This 'and' proves something about some of the bits in and.getX(). It's
+                         * equivalent to or'ing in the mask value since those values are known to be
+                         * set.
+                         */
+                        BinaryOp<Or> op = ArithmeticOpTable.forStamp(x.stamp(NodeView.DEFAULT)).getOr();
+                        IntegerStamp newStampX = (IntegerStamp) op.foldStamp(getSafeStamp(and.getX()), getOtherSafeStamp(y));
+                        if (guardFolding.foldGuard(thisGuard, and.getX(), newStampX, rewireGuardFunction)) {
+                            return true;
+                        }
+                    }
+                }
+            }
+
+            if (thisGuard != null && guardFolding != null) {
+                if (!x.isConstant()) {
+                    Stamp newStampX = binaryOpLogicNode.getSucceedingStampForX(thisGuard.isNegated(), getSafeStamp(x), getOtherSafeStamp(y));
+                    if (newStampX != null && guardFolding.foldGuard(thisGuard, x, newStampX, rewireGuardFunction)) {
+                        return true;
+                    }
+                }
+                if (!y.isConstant() && guardFolding != null) {
+                    Stamp newStampY = binaryOpLogicNode.getSucceedingStampForY(thisGuard.isNegated(), getOtherSafeStamp(x), getSafeStamp(y));
+                    if (newStampY != null && guardFolding.foldGuard(thisGuard, y, newStampY, rewireGuardFunction)) {
+                        return true;
+                    }
+                }
+            }
+        } else if (node instanceof ShortCircuitOrNode) {
+            final ShortCircuitOrNode shortCircuitOrNode = (ShortCircuitOrNode) node;
+            return tryProveGuardCondition(infoElementProvider, conditions, guardFolding, null, shortCircuitOrNode.getX(), (guard, result, guardedValueStamp, newInput) -> {
+                if (result == !shortCircuitOrNode.isXNegated()) {
+                    return rewireGuards(guard, true, newInput, guardedValueStamp, rewireGuardFunction);
+                } else {
+                    return tryProveGuardCondition(infoElementProvider, conditions, guardFolding, null, shortCircuitOrNode.getY(), (innerGuard, innerResult, innerGuardedValueStamp, innerNewInput) -> {
+                        ValueNode proxifiedInput = newInput;
+                        if (proxifiedInput == null) {
+                            proxifiedInput = innerNewInput;
+                        } else if (innerNewInput != null) {
+                            if (innerNewInput != newInput) {
+                                // Cannot canonicalize due to different proxied inputs.
+                                return false;
+                            }
+                        }
+                        // Can only canonicalize if the guards are equal.
+                        if (innerGuard == guard) {
+                            return rewireGuards(guard, innerResult ^ shortCircuitOrNode.isYNegated(), proxifiedInput, guardedValueStamp, rewireGuardFunction);
+                        }
+                        return false;
+                    });
+                }
+            });
+        }
+
+        return false;
+    }
+
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/DefaultJavaLoweringProvider.java

@@ -43,7 +43,6 @@ import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.List;
 
-import jdk.vm.ci.meta.JavaConstant;
 import org.graalvm.compiler.api.directives.GraalDirectives;
 import org.graalvm.compiler.api.replacements.Snippet;
 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
@@ -135,6 +134,7 @@ import org.graalvm.compiler.nodes.memory.WriteNode;
 import org.graalvm.compiler.nodes.memory.address.AddressNode;
 import org.graalvm.compiler.nodes.memory.address.IndexAddressNode;
 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
+import org.graalvm.compiler.nodes.spi.GCProvider;
 import org.graalvm.compiler.nodes.spi.Lowerable;
 import org.graalvm.compiler.nodes.spi.LoweringProvider;
 import org.graalvm.compiler.nodes.spi.LoweringTool;
@@ -157,6 +157,7 @@ import jdk.vm.ci.code.MemoryBarriers;
 import jdk.vm.ci.code.TargetDescription;
 import jdk.vm.ci.meta.DeoptimizationAction;
 import jdk.vm.ci.meta.DeoptimizationReason;
+import jdk.vm.ci.meta.JavaConstant;
 import jdk.vm.ci.meta.JavaKind;
 import jdk.vm.ci.meta.MetaAccessProvider;
 import jdk.vm.ci.meta.ResolvedJavaField;
@@ -673,24 +674,25 @@ public abstract class DefaultJavaLoweringProvider implements LoweringProvider {
      * @param tool utility for performing the lowering
      */
     protected void lowerUnsafeLoadNode(RawLoadNode load, LoweringTool tool) {
+        GCProvider gc = tool.getProviders().getGC();
         StructuredGraph graph = load.graph();
         if (load instanceof GuardedUnsafeLoadNode) {
             GuardedUnsafeLoadNode guardedLoad = (GuardedUnsafeLoadNode) load;
             GuardingNode guard = guardedLoad.getGuard();
             if (guard == null) {
                 // can float freely if the guard folded away
-                ReadNode memoryRead = createUnsafeRead(graph, load, null);
+                ReadNode memoryRead = createUnsafeRead(gc, graph, load, null);
                 memoryRead.setForceFixed(false);
                 graph.replaceFixedWithFixed(load, memoryRead);
             } else {
                 // must be guarded, but flows below the guard
-                ReadNode memoryRead = createUnsafeRead(graph, load, guard);
+                ReadNode memoryRead = createUnsafeRead(gc, graph, load, guard);
                 graph.replaceFixedWithFixed(load, memoryRead);
             }
         } else {
             // never had a guarding condition so it must be fixed, creation of the read will force
             // it to be fixed
-            ReadNode memoryRead = createUnsafeRead(graph, load, null);
+            ReadNode memoryRead = createUnsafeRead(gc, graph, load, null);
             graph.replaceFixedWithFixed(load, memoryRead);
         }
     }
@@ -703,12 +705,12 @@ public abstract class DefaultJavaLoweringProvider implements LoweringProvider {
         }
     }
 
-    protected ReadNode createUnsafeRead(StructuredGraph graph, RawLoadNode load, GuardingNode guard) {
+    protected ReadNode createUnsafeRead(GCProvider gc, StructuredGraph graph, RawLoadNode load, GuardingNode guard) {
         boolean compressible = load.accessKind() == JavaKind.Object;
         JavaKind readKind = load.accessKind();
         Stamp loadStamp = loadStamp(load.stamp(NodeView.DEFAULT), readKind, compressible);
         AddressNode address = createUnsafeAddress(graph, load.object(), load.offset());
-        ReadNode memoryRead = graph.add(new ReadNode(address, load.getLocationIdentity(), loadStamp, BarrierType.NONE));
+        ReadNode memoryRead = graph.add(new ReadNode(address, load.getLocationIdentity(), loadStamp, gc.getBarrierSet().readBarrierType(load)));
         if (guard == null) {
             // An unsafe read must not float otherwise it may float above
             // a test guaranteeing the read is safe.

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/IntrinsicGraphBuilder.java

@@ -155,10 +155,18 @@ public class IntrinsicGraphBuilder implements GraphBuilderContext, Receiver {
     @Override
     public void push(JavaKind kind, ValueNode value) {
         assert kind != JavaKind.Void;
-        assert returnValue == null;
+        GraalError.guarantee(returnValue == null, "can only push one value");
         returnValue = value;
     }
 
+    @Override
+    public ValueNode pop(JavaKind slotKind) {
+        GraalError.guarantee(returnValue != null, "no value pushed");
+        ValueNode result = returnValue;
+        returnValue = null;
+        return result;
+    }
+
     @Override
     public Invoke handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean forceInlineEverything) {
         throw GraalError.shouldNotReachHere();

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StandardGraphBuilderPlugins.java

@@ -84,6 +84,7 @@ import org.graalvm.compiler.nodes.calc.ZeroExtendNode;
 import org.graalvm.compiler.nodes.debug.BindToRegisterNode;
 import org.graalvm.compiler.nodes.debug.BlackholeNode;
 import org.graalvm.compiler.nodes.debug.ControlFlowAnchorNode;
+import org.graalvm.compiler.nodes.debug.SideEffectNode;
 import org.graalvm.compiler.nodes.debug.SpillRegistersNode;
 import org.graalvm.compiler.nodes.extended.BoxNode;
 import org.graalvm.compiler.nodes.extended.BranchProbabilityNode;
@@ -155,20 +156,22 @@ import sun.misc.Unsafe;
 public class StandardGraphBuilderPlugins {
 
     public static void registerInvocationPlugins(MetaAccessProvider metaAccess, SnippetReflectionProvider snippetReflection, InvocationPlugins plugins, Replacements replacements,
-                    boolean allowDeoptimization, boolean explicitUnsafeNullChecks) {
+                    boolean allowDeoptimization, boolean explicitUnsafeNullChecks, boolean arrayEqualsSubstitution) {
         registerObjectPlugins(plugins);
         registerClassPlugins(plugins);
         registerMathPlugins(plugins, allowDeoptimization);
         registerStrictMathPlugins(plugins);
         registerUnsignedMathPlugins(plugins);
-        registerStringPlugins(plugins, replacements, snippetReflection);
+        registerStringPlugins(plugins, replacements, snippetReflection, arrayEqualsSubstitution);
         registerCharacterPlugins(plugins);
         registerShortPlugins(plugins);
         registerIntegerLongPlugins(plugins, JavaKind.Int);
         registerIntegerLongPlugins(plugins, JavaKind.Long);
         registerFloatPlugins(plugins);
         registerDoublePlugins(plugins);
-        registerArraysPlugins(plugins, replacements);
+        if (arrayEqualsSubstitution) {
+            registerArraysPlugins(plugins, replacements);
+        }
         registerArrayPlugins(plugins, replacements);
         registerUnsafePlugins(plugins, replacements, explicitUnsafeNullChecks);
         registerEdgesPlugins(metaAccess, plugins);
@@ -196,7 +199,7 @@ public class StandardGraphBuilderPlugins {
         STRING_CODER_FIELD = coder;
     }
 
-    private static void registerStringPlugins(InvocationPlugins plugins, Replacements replacements, SnippetReflectionProvider snippetReflection) {
+    private static void registerStringPlugins(InvocationPlugins plugins, Replacements replacements, SnippetReflectionProvider snippetReflection, boolean arrayEqualsSubstitution) {
         final Registration r = new Registration(plugins, String.class, replacements);
         r.register1("hashCode", Receiver.class, new InvocationPlugin() {
             @Override
@@ -227,7 +230,9 @@ public class StandardGraphBuilderPlugins {
         });
 
         if (JavaVersionUtil.JAVA_SPEC <= 8) {
-            r.registerMethodSubstitution(StringSubstitutions.class, "equals", Receiver.class, Object.class);
+            if (arrayEqualsSubstitution) {
+                r.registerMethodSubstitution(StringSubstitutions.class, "equals", Receiver.class, Object.class);
+            }
 
             r.register7("indexOf", char[].class, int.class, int.class, char[].class, int.class, int.class, int.class, new StringIndexOfConstantPlugin());
 
@@ -242,7 +247,9 @@ public class StandardGraphBuilderPlugins {
                 }
             });
         } else {
-            r.registerMethodSubstitution(JDK9StringSubstitutions.class, "equals", Receiver.class, Object.class);
+            if (arrayEqualsSubstitution) {
+                r.registerMethodSubstitution(JDK9StringSubstitutions.class, "equals", Receiver.class, Object.class);
+            }
             Registration utf16sub = new Registration(plugins, StringUTF16Substitutions.class, replacements);
             utf16sub.register2("getCharDirect", byte[].class, int.class, new InvocationPlugin() {
                 @Override
@@ -1245,7 +1252,20 @@ public class StandardGraphBuilderPlugins {
                 return true;
             }
         });
-
+        r.register0("sideEffect", new InvocationPlugin() {
+            @Override
+            public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) {
+                b.add(new SideEffectNode());
+                return true;
+            }
+        });
+        r.register1("sideEffect", int.class, new InvocationPlugin() {
+            @Override
+            public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode a) {
+                b.addPush(JavaKind.Int, new SideEffectNode(a));
+                return true;
+            }
+        });
         r.register2("injectBranchProbability", double.class, boolean.class, new InvocationPlugin() {
             @Override
             public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode probability, ValueNode condition) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/gc/G1WriteBarrierSnippets.java

@@ -48,6 +48,7 @@ import org.graalvm.compiler.nodes.gc.G1ArrayRangePreWriteBarrier;
 import org.graalvm.compiler.nodes.gc.G1PostWriteBarrier;
 import org.graalvm.compiler.nodes.gc.G1PreWriteBarrier;
 import org.graalvm.compiler.nodes.gc.G1ReferentFieldReadBarrier;
+import org.graalvm.compiler.nodes.java.InstanceOfNode;
 import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType;
 import org.graalvm.compiler.nodes.memory.address.AddressNode;
 import org.graalvm.compiler.nodes.memory.address.AddressNode.Address;
@@ -69,6 +70,8 @@ import jdk.internal.vm.compiler.word.Pointer;
 import jdk.internal.vm.compiler.word.UnsignedWord;
 import jdk.internal.vm.compiler.word.WordFactory;
 
+import jdk.vm.ci.meta.ResolvedJavaType;
+
 public abstract class G1WriteBarrierSnippets extends WriteBarrierSnippets implements Snippets {
 
     public static final LocationIdentity GC_LOG_LOCATION = NamedLocationIdentity.mutable("GC-Log");
@@ -164,6 +167,15 @@ public abstract class G1WriteBarrierSnippets extends WriteBarrierSnippets implem
         }
     }
 
+    @Snippet
+    public void g1ReferentReadBarrier(Address address, Object object, Object expectedObject, @ConstantParameter boolean isDynamicCheck, Word offset,
+                    @ConstantParameter int traceStartCycle, @ConstantParameter Counters counters) {
+        if (!isDynamicCheck ||
+                        (offset == WordFactory.unsigned(referentOffset()) && InstanceOfNode.doInstanceof(referenceType(), object))) {
+            g1PreWriteBarrier(address, object, expectedObject, false, false, traceStartCycle, counters);
+        }
+    }
+
     @Snippet
     public void g1PostWriteBarrier(Address address, Object object, Object value, @ConstantParameter boolean usePrecise, @ConstantParameter int traceStartCycle,
                     @ConstantParameter Counters counters) {
@@ -359,6 +371,10 @@ public abstract class G1WriteBarrierSnippets extends WriteBarrierSnippets implem
 
     protected abstract ForeignCallDescriptor printfCallDescriptor();
 
+    protected abstract ResolvedJavaType referenceType();
+
+    protected abstract long referentOffset();
+
     private boolean isTracingActive(int traceStartCycle) {
         return traceStartCycle > 0 && ((Pointer) WordFactory.pointer(gcTotalCollectionsAddress())).readLong(0) > traceStartCycle;
     }
@@ -446,13 +462,10 @@ public abstract class G1WriteBarrierSnippets extends WriteBarrierSnippets implem
 
         public void lower(AbstractTemplates templates, SnippetInfo snippet, G1ReferentFieldReadBarrier barrier, LoweringTool tool) {
             Arguments args = new Arguments(snippet, barrier.graph().getGuardsStage(), tool.getLoweringStage());
-            AddressNode address = barrier.getAddress();
+            // This is expected to be lowered before address lowering
+            OffsetAddressNode address = (OffsetAddressNode) barrier.getAddress();
             args.add("address", address);
-            if (address instanceof OffsetAddressNode) {
-                args.add("object", ((OffsetAddressNode) address).getBase());
-            } else {
-                args.add("object", null);
-            }
+            args.add("object", address.getBase());
 
             ValueNode expected = barrier.getExpectedObject();
             if (expected != null && expected.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp) {
@@ -460,8 +473,8 @@ public abstract class G1WriteBarrierSnippets extends WriteBarrierSnippets implem
             }
 
             args.add("expectedObject", expected);
-            args.addConst("doLoad", barrier.doLoad());
-            args.addConst("nullCheck", false);
+            args.addConst("isDynamicCheck", barrier.isDynamicCheck());
+            args.add("offset", address.getOffset());
             args.addConst("traceStartCycle", traceStartCycle(barrier.graph()));
             args.addConst("counters", counters);
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerAddExactNode.java

@@ -42,12 +42,12 @@ import org.graalvm.compiler.nodes.ConstantNode;
 import org.graalvm.compiler.nodes.NodeView;
 import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.calc.AddNode;
+import org.graalvm.compiler.nodes.extended.GuardedNode;
+import org.graalvm.compiler.nodes.extended.GuardingNode;
 
 import jdk.vm.ci.code.CodeUtil;
 import jdk.vm.ci.meta.JavaConstant;
 import jdk.vm.ci.meta.JavaKind;
-import org.graalvm.compiler.nodes.extended.GuardedNode;
-import org.graalvm.compiler.nodes.extended.GuardingNode;
 
 /**
  * Node representing an exact integer addition that will throw an {@link ArithmeticException} in
@@ -140,6 +140,9 @@ public final class IntegerAddExactNode extends AddNode implements GuardedNode, I
         if (!IntegerStamp.addCanOverflow((IntegerStamp) forX.stamp(NodeView.DEFAULT), (IntegerStamp) forY.stamp(NodeView.DEFAULT))) {
             return new AddNode(forX, forY).canonical(tool);
         }
+        if (getGuard() == null) {
+            return new AddNode(forX, forY).canonical(tool);
+        }
         return this;
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerAddExactOverflowNode.java

@@ -25,10 +25,12 @@
 
 package org.graalvm.compiler.replacements.nodes.arithmetic;
 
-import jdk.vm.ci.meta.JavaConstant;
-import jdk.vm.ci.meta.JavaKind;
+import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_2;
+import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_2;
+
 import org.graalvm.compiler.core.common.type.IntegerStamp;
 import org.graalvm.compiler.core.common.type.Stamp;
+import org.graalvm.compiler.graph.IterableNodeType;
 import org.graalvm.compiler.graph.NodeClass;
 import org.graalvm.compiler.graph.spi.Canonicalizable.BinaryCommutative;
 import org.graalvm.compiler.graph.spi.CanonicalizerTool;
@@ -40,11 +42,11 @@ import org.graalvm.compiler.nodes.NodeView;
 import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.calc.BinaryNode;
 
-import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_2;
-import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_2;
+import jdk.vm.ci.meta.JavaConstant;
+import jdk.vm.ci.meta.JavaKind;
 
 @NodeInfo(cycles = CYCLES_2, size = SIZE_2)
-public final class IntegerAddExactOverflowNode extends IntegerExactOverflowNode implements Simplifiable, BinaryCommutative<ValueNode> {
+public final class IntegerAddExactOverflowNode extends IntegerExactOverflowNode implements Simplifiable, BinaryCommutative<ValueNode>, IterableNodeType {
     public static final NodeClass<IntegerAddExactOverflowNode> TYPE = NodeClass.create(IntegerAddExactOverflowNode.class);
 
     public IntegerAddExactOverflowNode(ValueNode x, ValueNode y) {