8190710: Update Graal

Reviewed-by: kvn

make/CompileToolsHotspot.gmk

@@ -67,6 +67,7 @@ ifeq ($(INCLUDE_GRAAL), true)
           $(SRC_DIR)/org.graalvm.compiler.phases.common/src \
           $(SRC_DIR)/org.graalvm.compiler.serviceprovider/src \
           $(SRC_DIR)/org.graalvm.compiler.virtual/src \
+          $(SRC_DIR)/org.graalvm.graphio/src \
           $(SRC_DIR)/org.graalvm.util/src \
           $(VM_CI_SRC_DIR)/jdk.vm.ci.code/src \
           $(VM_CI_SRC_DIR)/jdk.vm.ci.common/src \
@@ -125,6 +126,7 @@ ifeq ($(INCLUDE_GRAAL), true)
           $(SRC_DIR)/org.graalvm.compiler.nodeinfo/src \
           $(SRC_DIR)/org.graalvm.compiler.options/src \
           $(SRC_DIR)/org.graalvm.compiler.serviceprovider/src \
+          $(SRC_DIR)/org.graalvm.graphio/src \
           $(SRC_DIR)/org.graalvm.util/src \
           $(VM_CI_SRC_DIR)/jdk.vm.ci.code/src \
           $(VM_CI_SRC_DIR)/jdk.vm.ci.common/src \

src/hotspot/share/aot/aotCodeHeap.cpp

@@ -490,6 +490,8 @@ void AOTCodeHeap::link_stub_routines_symbols() {
 
     SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_checkcast_arraycopy", address, StubRoutines::_checkcast_arraycopy);
 
+    SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_generic_arraycopy", address, StubRoutines::_generic_arraycopy);
+
     SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_aescrypt_encryptBlock", address, StubRoutines::_aescrypt_encryptBlock);
     SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_aescrypt_decryptBlock", address, StubRoutines::_aescrypt_decryptBlock);
     SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_cipherBlockChaining_encryptAESCrypt", address, StubRoutines::_cipherBlockChaining_encryptAESCrypt);

src/jdk.aot/share/classes/jdk.tools.jaotc.binformat/src/jdk/tools/jaotc/binformat/BinaryContainer.java

@@ -192,6 +192,8 @@ public final class BinaryContainer implements SymbolTable {
 
         {"StubRoutines::_checkcast_arraycopy", "_aot_stub_routines_checkcast_arraycopy"},
 
+        {"StubRoutines::_generic_arraycopy", "_aot_stub_routines_generic_arraycopy"},
+
         {"StubRoutines::_aescrypt_encryptBlock", "_aot_stub_routines_aescrypt_encryptBlock"},
         {"StubRoutines::_aescrypt_decryptBlock", "_aot_stub_routines_aescrypt_decryptBlock"},
         {"StubRoutines::_cipherBlockChaining_encryptAESCrypt", "_aot_stub_routines_cipherBlockChaining_encryptAESCrypt"},

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.api.directives.test/src/org/graalvm/compiler/api/directives/test/BlackholeDirectiveTest.java

@@ -34,6 +34,9 @@ import org.graalvm.compiler.api.directives.GraalDirectives;
 import org.graalvm.compiler.core.test.GraalCompilerTest;
 import org.graalvm.compiler.nodes.ParameterNode;
 import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.phases.OptimisticOptimizations;
+import org.graalvm.compiler.phases.OptimisticOptimizations.Optimization;
+import org.graalvm.compiler.phases.tiers.HighTierContext;
 
 /**
  * Tests for {@link GraalDirectives#blackhole}.
@@ -128,6 +131,11 @@ public class BlackholeDirectiveTest extends GraalCompilerTest {
         test("blackholeObjectSnippet", 37);
     }
 
+    @Override
+    protected HighTierContext getDefaultHighTierContext() {
+        return new HighTierContext(getProviders(), getDefaultGraphBuilderSuite(), OptimisticOptimizations.ALL.remove(Optimization.RemoveNeverExecutedCode));
+    }
+
     @Override
     protected boolean checkLowTierGraph(StructuredGraph graph) {
         BlackholeSnippet snippet = graph.method().getAnnotation(BlackholeSnippet.class);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.api.directives.test/src/org/graalvm/compiler/api/directives/test/ControlFlowAnchorDirectiveTest.java

@@ -46,6 +46,9 @@ import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions;
 import org.graalvm.compiler.nodes.debug.ControlFlowAnchorNode;
 
 import jdk.vm.ci.meta.ResolvedJavaMethod;
+import org.graalvm.compiler.phases.OptimisticOptimizations;
+import org.graalvm.compiler.phases.OptimisticOptimizations.Optimization;
+import org.graalvm.compiler.phases.tiers.HighTierContext;
 
 public class ControlFlowAnchorDirectiveTest extends GraalCompilerTest {
 
@@ -238,6 +241,11 @@ public class ControlFlowAnchorDirectiveTest extends GraalCompilerTest {
         return Collections.emptyList();
     }
 
+    @Override
+    protected HighTierContext getDefaultHighTierContext() {
+        return new HighTierContext(getProviders(), getDefaultGraphBuilderSuite(), OptimisticOptimizations.ALL.remove(Optimization.RemoveNeverExecutedCode));
+    }
+
     @Override
     protected boolean checkLowTierGraph(StructuredGraph graph) {
         List<ControlFlowAnchorNode> anchors = graph.getNodes().filter(ControlFlowAnchorNode.class).snapshot();

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.api.directives.test/src/org/graalvm/compiler/api/directives/test/OpaqueDirectiveTest.java

@@ -37,6 +37,9 @@ import org.graalvm.compiler.nodes.ReturnNode;
 import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.nodes.calc.AddNode;
 import org.graalvm.compiler.nodes.calc.ConditionalNode;
+import org.graalvm.compiler.phases.OptimisticOptimizations;
+import org.graalvm.compiler.phases.OptimisticOptimizations.Optimization;
+import org.graalvm.compiler.phases.tiers.HighTierContext;
 
 /**
  * Tests for {@link GraalDirectives#opaque}.
@@ -127,6 +130,11 @@ public class OpaqueDirectiveTest extends GraalCompilerTest {
         test("opaqueObjectSnippet");
     }
 
+    @Override
+    protected HighTierContext getDefaultHighTierContext() {
+        return new HighTierContext(getProviders(), getDefaultGraphBuilderSuite(), OptimisticOptimizations.ALL.remove(Optimization.RemoveNeverExecutedCode));
+    }
+
     @Override
     protected boolean checkLowTierGraph(StructuredGraph graph) {
         OpaqueSnippet snippet = graph.method().getAnnotation(OpaqueSnippet.class);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.api.replacements/src/org/graalvm/compiler/api/replacements/Snippet.java

@@ -35,6 +35,13 @@ import java.lang.annotation.Target;
 @Target(ElementType.METHOD)
 public @interface Snippet {
 
+    /**
+     * A partial intrinsic exits by (effectively) calling the intrinsified method. Normally, this
+     * call must use exactly the same arguments as the call that is being intrinsified. For well
+     * known snippets that are used after frame state assignment, we want to relax this restriction.
+     */
+    boolean allowPartialIntrinsicArgumentMismatch() default false;
+
     /**
      * Denotes a snippet parameter representing 0 or more arguments that will be bound during
      * snippet template instantiation. During snippet template creation, its value must be an array

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Assembler.java

@@ -22,10 +22,14 @@
  */
 package org.graalvm.compiler.asm.amd64;
 
-import static org.graalvm.compiler.core.common.NumUtil.isByte;
-import static org.graalvm.compiler.core.common.NumUtil.isInt;
-import static org.graalvm.compiler.core.common.NumUtil.isShiftCount;
-import static org.graalvm.compiler.core.common.NumUtil.isUByte;
+import static jdk.vm.ci.amd64.AMD64.CPU;
+import static jdk.vm.ci.amd64.AMD64.XMM;
+import static jdk.vm.ci.amd64.AMD64.r12;
+import static jdk.vm.ci.amd64.AMD64.r13;
+import static jdk.vm.ci.amd64.AMD64.rbp;
+import static jdk.vm.ci.amd64.AMD64.rip;
+import static jdk.vm.ci.amd64.AMD64.rsp;
+import static jdk.vm.ci.code.MemoryBarriers.STORE_LOAD;
 import static org.graalvm.compiler.asm.amd64.AMD64AsmOptions.UseAddressNop;
 import static org.graalvm.compiler.asm.amd64.AMD64AsmOptions.UseNormalNop;
 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64BinaryArithmetic.ADD;
@@ -47,25 +51,24 @@ import static org.graalvm.compiler.asm.amd64.AMD64Assembler.OperandSize.QWORD;
 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.OperandSize.SD;
 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.OperandSize.SS;
 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.OperandSize.WORD;
-import static jdk.vm.ci.amd64.AMD64.CPU;
-import static jdk.vm.ci.amd64.AMD64.XMM;
-import static jdk.vm.ci.amd64.AMD64.r12;
-import static jdk.vm.ci.amd64.AMD64.r13;
-import static jdk.vm.ci.amd64.AMD64.rbp;
-import static jdk.vm.ci.amd64.AMD64.rip;
-import static jdk.vm.ci.amd64.AMD64.rsp;
-import static jdk.vm.ci.code.MemoryBarriers.STORE_LOAD;
+import static org.graalvm.compiler.core.common.NumUtil.isByte;
+import static org.graalvm.compiler.core.common.NumUtil.isInt;
+import static org.graalvm.compiler.core.common.NumUtil.isShiftCount;
+import static org.graalvm.compiler.core.common.NumUtil.isUByte;
 
 import org.graalvm.compiler.asm.Assembler;
 import org.graalvm.compiler.asm.Label;
-import org.graalvm.compiler.core.common.NumUtil;
 import org.graalvm.compiler.asm.amd64.AMD64Address.Scale;
+import org.graalvm.compiler.core.common.NumUtil;
+import org.graalvm.compiler.debug.GraalError;
 
 import jdk.vm.ci.amd64.AMD64;
 import jdk.vm.ci.amd64.AMD64.CPUFeature;
+import jdk.vm.ci.amd64.AMD64Kind;
 import jdk.vm.ci.code.Register;
 import jdk.vm.ci.code.Register.RegisterCategory;
 import jdk.vm.ci.code.TargetDescription;
+import jdk.vm.ci.meta.PlatformKind;
 
 /**
  * This class implements an assembler that can encode most X86 instructions.
@@ -225,7 +228,7 @@ public class AMD64Assembler extends Assembler {
      * The x86 operand sizes.
      */
     public enum OperandSize {
-        BYTE(1) {
+        BYTE(1, AMD64Kind.BYTE) {
             @Override
             protected void emitImmediate(AMD64Assembler asm, int imm) {
                 assert imm == (byte) imm;
@@ -238,7 +241,7 @@ public class AMD64Assembler extends Assembler {
             }
         },
 
-        WORD(2, 0x66) {
+        WORD(2, AMD64Kind.WORD, 0x66) {
             @Override
             protected void emitImmediate(AMD64Assembler asm, int imm) {
                 assert imm == (short) imm;
@@ -251,7 +254,7 @@ public class AMD64Assembler extends Assembler {
             }
         },
 
-        DWORD(4) {
+        DWORD(4, AMD64Kind.DWORD) {
             @Override
             protected void emitImmediate(AMD64Assembler asm, int imm) {
                 asm.emitInt(imm);
@@ -263,7 +266,7 @@ public class AMD64Assembler extends Assembler {
             }
         },
 
-        QWORD(8) {
+        QWORD(8, AMD64Kind.QWORD) {
             @Override
             protected void emitImmediate(AMD64Assembler asm, int imm) {
                 asm.emitInt(imm);
@@ -275,34 +278,35 @@ public class AMD64Assembler extends Assembler {
             }
         },
 
-        SS(4, 0xF3, true),
+        SS(4, AMD64Kind.SINGLE, 0xF3, true),
 
-        SD(8, 0xF2, true),
+        SD(8, AMD64Kind.DOUBLE, 0xF2, true),
 
-        PS(16, true),
+        PS(16, AMD64Kind.V128_SINGLE, true),
 
-        PD(16, 0x66, true);
+        PD(16, AMD64Kind.V128_DOUBLE, 0x66, true);
 
         private final int sizePrefix;
-
         private final int bytes;
         private final boolean xmm;
+        private final AMD64Kind kind;
 
-        OperandSize(int bytes) {
-            this(bytes, 0);
+        OperandSize(int bytes, AMD64Kind kind) {
+            this(bytes, kind, 0);
         }
 
-        OperandSize(int bytes, int sizePrefix) {
-            this(bytes, sizePrefix, false);
+        OperandSize(int bytes, AMD64Kind kind, int sizePrefix) {
+            this(bytes, kind, sizePrefix, false);
         }
 
-        OperandSize(int bytes, boolean xmm) {
-            this(bytes, 0, xmm);
+        OperandSize(int bytes, AMD64Kind kind, boolean xmm) {
+            this(bytes, kind, 0, xmm);
         }
 
-        OperandSize(int bytes, int sizePrefix, boolean xmm) {
+        OperandSize(int bytes, AMD64Kind kind, int sizePrefix, boolean xmm) {
             this.sizePrefix = sizePrefix;
             this.bytes = bytes;
+            this.kind = kind;
             this.xmm = xmm;
         }
 
@@ -314,6 +318,19 @@ public class AMD64Assembler extends Assembler {
             return xmm;
         }
 
+        public AMD64Kind getKind() {
+            return kind;
+        }
+
+        public static OperandSize get(PlatformKind kind) {
+            for (OperandSize operandSize : OperandSize.values()) {
+                if (operandSize.kind.equals(kind)) {
+                    return operandSize;
+                }
+            }
+            throw GraalError.shouldNotReachHere("Unexpected kind: " + kind.toString());
+        }
+
         /**
          * Emit an immediate of this size. Note that immediate {@link #QWORD} operands are encoded
          * as sign-extended 32-bit values.
@@ -2230,6 +2247,14 @@ public class AMD64Assembler extends Assembler {
         emitOperandHelper(dst, src, 0);
     }
 
+    public final void movzbl(Register dst, Register src) {
+        AMD64RMOp.MOVZXB.emit(this, OperandSize.DWORD, dst, src);
+    }
+
+    public final void movzbq(Register dst, Register src) {
+        AMD64RMOp.MOVZXB.emit(this, OperandSize.QWORD, dst, src);
+    }
+
     public final void movzwl(Register dst, AMD64Address src) {
         prefix(src, dst);
         emitByte(0x0F);
@@ -3198,6 +3223,13 @@ public class AMD64Assembler extends Assembler {
         emitByte(0xC0 | encode);
     }
 
+    public final void setb(ConditionFlag cc, Register dst) {
+        int encode = prefixAndEncode(dst.encoding, true);
+        emitByte(0x0F);
+        emitByte(0x90 | cc.getValue());
+        emitByte(0xC0 | encode);
+    }
+
     public final void cmovq(ConditionFlag cc, Register dst, AMD64Address src) {
         prefixq(src, dst);
         emitByte(0x0F);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64MacroAssembler.java

@@ -31,8 +31,8 @@ import static org.graalvm.compiler.asm.amd64.AMD64AsmOptions.UseXmmLoadAndClearU
 import static org.graalvm.compiler.asm.amd64.AMD64AsmOptions.UseXmmRegToRegMoveAll;
 
 import org.graalvm.compiler.asm.Label;
-import org.graalvm.compiler.core.common.NumUtil;
 import org.graalvm.compiler.asm.amd64.AMD64Address.Scale;
+import org.graalvm.compiler.core.common.NumUtil;
 
 import jdk.vm.ci.amd64.AMD64;
 import jdk.vm.ci.amd64.AMD64Kind;
@@ -281,6 +281,16 @@ public class AMD64MacroAssembler extends AMD64Assembler {
 
     }
 
+    public final void setl(ConditionFlag cc, Register dst) {
+        setb(cc, dst);
+        movzbl(dst, dst);
+    }
+
+    public final void setq(ConditionFlag cc, Register dst) {
+        setb(cc, dst);
+        movzbq(dst, dst);
+    }
+
     public final void flog(Register dest, Register value, boolean base10) {
         if (base10) {
             fldlg2();

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64.test/src/org/graalvm/compiler/core/amd64/test/AMD64AddressLoweringTest.java

@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.core.amd64.test;
+
+import static org.junit.Assume.assumeTrue;
+
+import org.graalvm.compiler.asm.amd64.AMD64Address.Scale;
+import org.graalvm.compiler.core.amd64.AMD64AddressLowering;
+import org.graalvm.compiler.core.amd64.AMD64AddressNode;
+import org.graalvm.compiler.core.test.GraalCompilerTest;
+import org.graalvm.compiler.nodes.ConstantNode;
+import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.nodes.ValueNode;
+import org.graalvm.compiler.nodes.calc.AddNode;
+import org.graalvm.compiler.nodes.calc.LeftShiftNode;
+import org.graalvm.compiler.nodes.calc.NegateNode;
+import org.graalvm.compiler.nodes.memory.address.AddressNode;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import jdk.vm.ci.amd64.AMD64;
+
+public class AMD64AddressLoweringTest extends GraalCompilerTest {
+
+    private StructuredGraph graph;
+    private AMD64AddressLowering lowering;
+
+    @Before
+    public void checkAMD64() {
+        assumeTrue("skipping AMD64 specific test", getTarget().arch instanceof AMD64);
+        graph = new StructuredGraph.Builder(getInitialOptions(), getDebugContext()).build();
+        lowering = new AMD64AddressLowering();
+    }
+
+    @Test
+    public void convertBaseAndIndexToDisplacement() {
+        ValueNode base = graph.unique(const64(1000));
+        ValueNode index = graph.unique(const64(10));
+        AddressNode result = lowering.lower(base, index);
+        assertAddress(result, null, null, Scale.Times1, 1010);
+    }
+
+    @Test
+    public void convertBaseToDisplacement() {
+        ValueNode constantAddress = graph.addOrUniqueWithInputs(const64(1000));
+        AddressNode result = lowering.lower(constantAddress, null);
+        assertAddress(result, null, null, Scale.Times1, 1000);
+    }
+
+    @Test
+    public void convertBaseAndShiftedIndexToDisplacement() {
+        ValueNode base = graph.addOrUniqueWithInputs(const64(1000));
+        ValueNode index = graph.addOrUniqueWithInputs(new LeftShiftNode(const64(10), const32(1)));
+        AddressNode result = lowering.lower(base, index);
+        assertAddress(result, null, null, Scale.Times2, 1020);
+    }
+
+    @Test
+    public void convertBaseAndNegatedShiftedIndexToDisplacement() {
+        ValueNode base = graph.addOrUniqueWithInputs(const64(1000));
+        ValueNode index = graph.addOrUniqueWithInputs(new NegateNode(new LeftShiftNode(const64(10), const32(2))));
+        AddressNode result = lowering.lower(base, index);
+        assertAddress(result, null, null, Scale.Times4, 960);
+    }
+
+    @Test
+    public void convertNegatedBaseAndNegatedShiftedIndexToDisplacement() {
+        ValueNode base = graph.addOrUniqueWithInputs(new NegateNode(const64(1000)));
+        ValueNode index = graph.addOrUniqueWithInputs(new NegateNode(new LeftShiftNode(const64(10), const32(2))));
+        AddressNode result = lowering.lower(base, index);
+        assertAddress(result, null, null, Scale.Times4, -1040);
+    }
+
+    @Test
+    public void convertNegatedShiftedBaseAndNegatedIndexToDisplacement() {
+        ValueNode base = graph.addOrUniqueWithInputs(new NegateNode(new LeftShiftNode(const64(10), const32(2))));
+        ValueNode index = graph.addOrUniqueWithInputs(new NegateNode(const64(1000)));
+        AddressNode result = lowering.lower(base, index);
+        assertAddress(result, null, null, Scale.Times4, -1040);
+    }
+
+    @Test
+    public void convertTwoLevelsOfNegatedShiftedBaseAndNegatedIndexToDisplacement() {
+        ValueNode base = graph.addOrUniqueWithInputs(new NegateNode(new LeftShiftNode(new NegateNode(new LeftShiftNode(const64(500), const32(1))), const32(1))));
+        ValueNode index = graph.addOrUniqueWithInputs(new NegateNode(new AddNode(new NegateNode(const64(13)), const64(3))));
+        AddressNode result = lowering.lower(base, index);
+        assertAddress(result, null, null, Scale.Times4, 2010);
+    }
+
+    private static ConstantNode const64(long value) {
+        return ConstantNode.forIntegerBits(Long.SIZE, value);
+    }
+
+    private static ConstantNode const32(long value) {
+        return ConstantNode.forIntegerBits(Integer.SIZE, value);
+    }
+
+    private static void assertAddress(AddressNode actual, ValueNode expectedBase, ValueNode expectedIndex, Scale expectedScale, int expectedDisplacement) {
+        AMD64AddressNode address = (AMD64AddressNode) actual;
+        Assert.assertEquals(expectedBase, address.getBase());
+        Assert.assertEquals(expectedIndex, address.getIndex());
+        Assert.assertEquals(expectedScale, address.getScale());
+        Assert.assertEquals(expectedDisplacement, address.getDisplacement());
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64AddressLowering.java

@@ -23,21 +23,22 @@
 
 package org.graalvm.compiler.core.amd64;
 
-import jdk.vm.ci.meta.JavaConstant;
-
-import org.graalvm.compiler.core.common.NumUtil;
 import org.graalvm.compiler.asm.amd64.AMD64Address.Scale;
+import org.graalvm.compiler.core.common.NumUtil;
 import org.graalvm.compiler.core.common.type.IntegerStamp;
 import org.graalvm.compiler.debug.DebugContext;
+import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.calc.AddNode;
 import org.graalvm.compiler.nodes.calc.LeftShiftNode;
+import org.graalvm.compiler.nodes.calc.NegateNode;
 import org.graalvm.compiler.nodes.calc.ZeroExtendNode;
 import org.graalvm.compiler.nodes.memory.address.AddressNode;
 import org.graalvm.compiler.phases.common.AddressLoweringPhase.AddressLowering;
 
-public class AMD64AddressLowering extends AddressLowering {
+import jdk.vm.ci.meta.JavaConstant;
 
+public class AMD64AddressLowering extends AddressLowering {
     @Override
     public AddressNode lower(ValueNode address) {
         return lower(address, null);
@@ -46,24 +47,37 @@ public class AMD64AddressLowering extends AddressLowering {
     @Override
     public AddressNode lower(ValueNode base, ValueNode offset) {
         AMD64AddressNode ret = new AMD64AddressNode(base, offset);
+        StructuredGraph graph = base.graph();
+
         boolean changed;
         do {
-            changed = improve(base.getDebug(), ret);
+            changed = improve(graph, base.getDebug(), ret, false, false);
         } while (changed);
-        return base.graph().unique(ret);
+
+        return graph.unique(ret);
     }
 
     /**
-     * @param debug
+     * Tries to optimize addresses so that they match the AMD64-specific addressing mode better
+     * (base + index * scale + displacement).
+     *
+     * @param graph the current graph
+     * @param debug the current debug context
+     * @param ret the address that should be optimized
+     * @param isBaseNegated determines if the address base is negated. if so, all values that are
+     *            extracted from the base will be negated as well
+     * @param isIndexNegated determines if the index is negated. if so, all values that are
+     *            extracted from the index will be negated as well
+     * @return true if the address was modified
      */
-    protected boolean improve(DebugContext debug, AMD64AddressNode ret) {
-        ValueNode newBase = improveInput(ret, ret.getBase(), 0);
+    protected boolean improve(StructuredGraph graph, DebugContext debug, AMD64AddressNode ret, boolean isBaseNegated, boolean isIndexNegated) {
+        ValueNode newBase = improveInput(ret, ret.getBase(), 0, isBaseNegated);
         if (newBase != ret.getBase()) {
             ret.setBase(newBase);
             return true;
         }
 
-        ValueNode newIdx = improveInput(ret, ret.getIndex(), ret.getScale().log2);
+        ValueNode newIdx = improveInput(ret, ret.getIndex(), ret.getScale().log2, isIndexNegated);
         if (newIdx != ret.getIndex()) {
             ret.setIndex(newIdx);
             return true;
@@ -83,55 +97,122 @@ public class AMD64AddressLowering extends AddressLowering {
         }
 
         if (ret.getScale() == Scale.Times1) {
-            if (ret.getBase() == null || ret.getIndex() == null) {
-                if (ret.getBase() instanceof AddNode) {
+            if (ret.getIndex() == null && ret.getBase() instanceof AddNode) {
                 AddNode add = (AddNode) ret.getBase();
                 ret.setBase(add.getX());
-                    ret.setIndex(add.getY());
+                ret.setIndex(considerNegation(graph, add.getY(), isBaseNegated));
                 return true;
-                } else if (ret.getIndex() instanceof AddNode) {
+            } else if (ret.getBase() == null && ret.getIndex() instanceof AddNode) {
                 AddNode add = (AddNode) ret.getIndex();
-                    ret.setBase(add.getX());
+                ret.setBase(considerNegation(graph, add.getX(), isIndexNegated));
                 ret.setIndex(add.getY());
                 return true;
             }
-            }
 
             if (ret.getBase() instanceof LeftShiftNode && !(ret.getIndex() instanceof LeftShiftNode)) {
                 ValueNode tmp = ret.getBase();
-                ret.setBase(ret.getIndex());
-                ret.setIndex(tmp);
+                ret.setBase(considerNegation(graph, ret.getIndex(), isIndexNegated != isBaseNegated));
+                ret.setIndex(considerNegation(graph, tmp, isIndexNegated != isBaseNegated));
                 return true;
             }
         }
 
+        return improveNegation(graph, debug, ret, isBaseNegated, isIndexNegated);
+    }
+
+    private boolean improveNegation(StructuredGraph graph, DebugContext debug, AMD64AddressNode ret, boolean originalBaseNegated, boolean originalIndexNegated) {
+        boolean baseNegated = originalBaseNegated;
+        boolean indexNegated = originalIndexNegated;
+
+        ValueNode originalBase = ret.getBase();
+        ValueNode originalIndex = ret.getIndex();
+
+        if (ret.getBase() instanceof NegateNode) {
+            NegateNode negate = (NegateNode) ret.getBase();
+            ret.setBase(negate.getValue());
+            baseNegated = !baseNegated;
+        }
+
+        if (ret.getIndex() instanceof NegateNode) {
+            NegateNode negate = (NegateNode) ret.getIndex();
+            ret.setIndex(negate.getValue());
+            indexNegated = !indexNegated;
+        }
+
+        if (baseNegated != originalBaseNegated || indexNegated != originalIndexNegated) {
+            ValueNode base = ret.getBase();
+            ValueNode index = ret.getIndex();
+
+            boolean improved = improve(graph, debug, ret, baseNegated, indexNegated);
+            if (baseNegated != originalBaseNegated) {
+                if (base == ret.getBase()) {
+                    ret.setBase(originalBase);
+                } else if (ret.getBase() != null) {
+                    ret.setBase(graph.maybeAddOrUnique(NegateNode.create(ret.getBase())));
+                }
+            }
+
+            if (indexNegated != originalIndexNegated) {
+                if (index == ret.getIndex()) {
+                    ret.setIndex(originalIndex);
+                } else if (ret.getIndex() != null) {
+                    ret.setIndex(graph.maybeAddOrUnique(NegateNode.create(ret.getIndex())));
+                }
+            }
+            return improved;
+        } else {
+            assert ret.getBase() == originalBase && ret.getIndex() == originalIndex;
+        }
         return false;
     }
 
-    private static ValueNode improveInput(AMD64AddressNode address, ValueNode node, int shift) {
+    private static ValueNode considerNegation(StructuredGraph graph, ValueNode value, boolean negate) {
+        if (negate && value != null) {
+            return graph.maybeAddOrUnique(NegateNode.create(value));
+        }
+        return value;
+    }
+
+    private ValueNode improveInput(AMD64AddressNode address, ValueNode node, int shift, boolean negateExtractedDisplacement) {
         if (node == null) {
             return null;
         }
 
         JavaConstant c = node.asJavaConstant();
         if (c != null) {
-            return improveConstDisp(address, node, c, null, shift);
+            return improveConstDisp(address, node, c, null, shift, negateExtractedDisplacement);
         } else {
-            if (node.stamp() instanceof IntegerStamp && ((IntegerStamp) node.stamp()).getBits() == 64) {
-                if (node instanceof ZeroExtendNode) {
-                    if (((ZeroExtendNode) node).getInputBits() == 32) {
+            if (node.stamp() instanceof IntegerStamp) {
+                if (node instanceof ZeroExtendNode && (((ZeroExtendNode) node).getInputBits() == 32)) {
                     /*
-                         * We can just swallow a zero-extend from 32 bit to 64 bit because the upper
-                         * half of the register will always be zero.
+                     * we can't just swallow all zero-extends as we might encounter something like
+                     * the following: ZeroExtend(Add(negativeValue, positiveValue)).
+                     *
+                     * if we swallow the zero-extend in this case and subsequently optimize the add,
+                     * we might end up with a negative value that has less than 64 bits in base or
+                     * index. such a value would require sign extension instead of zero-extension
+                     * but the backend can only do zero-extension. if we ever want to optimize that
+                     * further, we would also need to be careful about over-/underflows.
+                     *
+                     * furthermore, we also can't swallow zero-extends with less than 32 bits as
+                     * most of these values are immediately sign-extended to 32 bit by the backend
+                     * (therefore, the subsequent implicit zero-extension to 64 bit won't do what we
+                     * expect).
                      */
-                        return ((ZeroExtendNode) node).getValue();
+                    ValueNode value = ((ZeroExtendNode) node).getValue();
+                    if (!mightBeOptimized(value)) {
+                        // if the value is not optimized further by the address lowering, then we
+                        // can safely rely on the backend doing the implicitly zero-extension.
+                        return value;
                     }
-                } else if (node instanceof AddNode) {
+                }
+
+                if (node instanceof AddNode) {
                     AddNode add = (AddNode) node;
                     if (add.getX().isConstant()) {
-                        return improveConstDisp(address, node, add.getX().asJavaConstant(), add.getY(), shift);
+                        return improveConstDisp(address, node, add.getX().asJavaConstant(), add.getY(), shift, negateExtractedDisplacement);
                     } else if (add.getY().isConstant()) {
-                        return improveConstDisp(address, node, add.getY().asJavaConstant(), add.getX(), shift);
+                        return improveConstDisp(address, node, add.getY().asJavaConstant(), add.getX(), shift, negateExtractedDisplacement);
                     }
                 }
             }
@@ -140,15 +221,30 @@ public class AMD64AddressLowering extends AddressLowering {
         return node;
     }
 
-    private static ValueNode improveConstDisp(AMD64AddressNode address, ValueNode original, JavaConstant c, ValueNode other, int shift) {
+    /**
+     * This method returns true for all nodes that might be optimized by the address lowering.
+     */
+    protected boolean mightBeOptimized(ValueNode value) {
+        return value instanceof AddNode || value instanceof LeftShiftNode || value instanceof NegateNode || value instanceof ZeroExtendNode;
+    }
+
+    private static ValueNode improveConstDisp(AMD64AddressNode address, ValueNode original, JavaConstant c, ValueNode other, int shift, boolean negateExtractedDisplacement) {
         if (c.getJavaKind().isNumericInteger()) {
-            long disp = address.getDisplacement();
-            disp += c.asLong() << shift;
-            if (NumUtil.isInt(disp)) {
-                address.setDisplacement((int) disp);
+            long delta = c.asLong() << shift;
+            if (updateDisplacement(address, delta, negateExtractedDisplacement)) {
                 return other;
             }
         }
         return original;
     }
+
+    protected static boolean updateDisplacement(AMD64AddressNode address, long displacementDelta, boolean negateDelta) {
+        long sign = negateDelta ? -1 : 1;
+        long disp = address.getDisplacement() + displacementDelta * sign;
+        if (NumUtil.isInt(disp)) {
+            address.setDisplacement((int) disp);
+            return true;
+        }
+        return false;
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64LIRGenerator.java

@@ -33,15 +33,17 @@ import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
 import static org.graalvm.compiler.lir.LIRValueUtil.asConstantValue;
 import static org.graalvm.compiler.lir.LIRValueUtil.asJavaConstant;
 import static org.graalvm.compiler.lir.LIRValueUtil.isConstantValue;
+import static org.graalvm.compiler.lir.LIRValueUtil.isIntConstant;
 import static org.graalvm.compiler.lir.LIRValueUtil.isJavaConstant;
 
-import org.graalvm.compiler.core.common.NumUtil;
+import org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64BinaryArithmetic;
 import org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64MIOp;
 import org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64RMOp;
 import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag;
 import org.graalvm.compiler.asm.amd64.AMD64Assembler.OperandSize;
 import org.graalvm.compiler.asm.amd64.AMD64Assembler.SSEOp;
 import org.graalvm.compiler.core.common.LIRKind;
+import org.graalvm.compiler.core.common.NumUtil;
 import org.graalvm.compiler.core.common.calc.Condition;
 import org.graalvm.compiler.core.common.spi.ForeignCallLinkage;
 import org.graalvm.compiler.core.common.spi.LIRKindTool;
@@ -58,13 +60,17 @@ import org.graalvm.compiler.lir.Variable;
 import org.graalvm.compiler.lir.amd64.AMD64AddressValue;
 import org.graalvm.compiler.lir.amd64.AMD64ArithmeticLIRGeneratorTool;
 import org.graalvm.compiler.lir.amd64.AMD64ArrayEqualsOp;
+import org.graalvm.compiler.lir.amd64.AMD64Binary;
 import org.graalvm.compiler.lir.amd64.AMD64BinaryConsumer;
 import org.graalvm.compiler.lir.amd64.AMD64ByteSwapOp;
 import org.graalvm.compiler.lir.amd64.AMD64Call;
+import org.graalvm.compiler.lir.amd64.AMD64ControlFlow;
 import org.graalvm.compiler.lir.amd64.AMD64ControlFlow.BranchOp;
 import org.graalvm.compiler.lir.amd64.AMD64ControlFlow.CondMoveOp;
+import org.graalvm.compiler.lir.amd64.AMD64ControlFlow.CondSetOp;
 import org.graalvm.compiler.lir.amd64.AMD64ControlFlow.FloatBranchOp;
 import org.graalvm.compiler.lir.amd64.AMD64ControlFlow.FloatCondMoveOp;
+import org.graalvm.compiler.lir.amd64.AMD64ControlFlow.FloatCondSetOp;
 import org.graalvm.compiler.lir.amd64.AMD64ControlFlow.ReturnOp;
 import org.graalvm.compiler.lir.amd64.AMD64ControlFlow.StrategySwitchOp;
 import org.graalvm.compiler.lir.amd64.AMD64ControlFlow.TableSwitchOp;
@@ -257,8 +263,7 @@ public abstract class AMD64LIRGenerator extends LIRGenerator {
 
     @Override
     public void emitCompareBranch(PlatformKind cmpKind, Value left, Value right, Condition cond, boolean unorderedIsTrue, LabelRef trueLabel, LabelRef falseLabel, double trueLabelProbability) {
-        boolean mirrored = emitCompare(cmpKind, left, right);
-        Condition finalCondition = mirrored ? cond.mirror() : cond;
+        Condition finalCondition = emitCompare(cmpKind, left, right, cond);
         if (cmpKind == AMD64Kind.SINGLE || cmpKind == AMD64Kind.DOUBLE) {
             append(new FloatBranchOp(finalCondition, unorderedIsTrue, trueLabel, falseLabel, trueLabelProbability));
         } else {
@@ -290,14 +295,60 @@ public abstract class AMD64LIRGenerator extends LIRGenerator {
 
     @Override
     public Variable emitConditionalMove(PlatformKind cmpKind, Value left, Value right, Condition cond, boolean unorderedIsTrue, Value trueValue, Value falseValue) {
-        boolean mirrored = emitCompare(cmpKind, left, right);
-        Condition finalCondition = mirrored ? cond.mirror() : cond;
+        boolean isFloatComparison = cmpKind == AMD64Kind.SINGLE || cmpKind == AMD64Kind.DOUBLE;
 
-        Variable result = newVariable(trueValue.getValueKind());
-        if (cmpKind == AMD64Kind.SINGLE || cmpKind == AMD64Kind.DOUBLE) {
-            append(new FloatCondMoveOp(result, finalCondition, unorderedIsTrue, load(trueValue), load(falseValue)));
+        Condition finalCondition = cond;
+        Value finalTrueValue = trueValue;
+        Value finalFalseValue = falseValue;
+        if (isFloatComparison) {
+            // eliminate the parity check in case of a float comparison
+            Value finalLeft = left;
+            Value finalRight = right;
+            if (unorderedIsTrue != AMD64ControlFlow.trueOnUnordered(finalCondition)) {
+                if (unorderedIsTrue == AMD64ControlFlow.trueOnUnordered(finalCondition.mirror())) {
+                    finalCondition = finalCondition.mirror();
+                    finalLeft = right;
+                    finalRight = left;
+                } else if (finalCondition != Condition.EQ && finalCondition != Condition.NE) {
+                    // negating EQ and NE does not make any sense as we would need to negate
+                    // unorderedIsTrue as well (otherwise, we would no longer fulfill the Java
+                    // NaN semantics)
+                    assert unorderedIsTrue == AMD64ControlFlow.trueOnUnordered(finalCondition.negate());
+                    finalCondition = finalCondition.negate();
+                    finalTrueValue = falseValue;
+                    finalFalseValue = trueValue;
+                }
+            }
+            emitRawCompare(cmpKind, finalLeft, finalRight);
         } else {
-            append(new CondMoveOp(result, finalCondition, load(trueValue), loadNonConst(falseValue)));
+            finalCondition = emitCompare(cmpKind, left, right, cond);
+        }
+
+        boolean isParityCheckNecessary = isFloatComparison && unorderedIsTrue != AMD64ControlFlow.trueOnUnordered(finalCondition);
+        Variable result = newVariable(finalTrueValue.getValueKind());
+        if (!isParityCheckNecessary && isIntConstant(finalTrueValue, 1) && isIntConstant(finalFalseValue, 0)) {
+            if (isFloatComparison) {
+                append(new FloatCondSetOp(result, finalCondition));
+            } else {
+                append(new CondSetOp(result, finalCondition));
+            }
+        } else if (!isParityCheckNecessary && isIntConstant(finalTrueValue, 0) && isIntConstant(finalFalseValue, 1)) {
+            if (isFloatComparison) {
+                if (unorderedIsTrue == AMD64ControlFlow.trueOnUnordered(finalCondition.negate())) {
+                    append(new FloatCondSetOp(result, finalCondition.negate()));
+                } else {
+                    append(new FloatCondSetOp(result, finalCondition));
+                    Variable negatedResult = newVariable(result.getValueKind());
+                    append(new AMD64Binary.ConstOp(AMD64BinaryArithmetic.XOR, OperandSize.get(result.getPlatformKind()), negatedResult, result, 1));
+                    result = negatedResult;
+                }
+            } else {
+                append(new CondSetOp(result, finalCondition.negate()));
+            }
+        } else if (isFloatComparison) {
+            append(new FloatCondMoveOp(result, finalCondition, unorderedIsTrue, load(finalTrueValue), load(finalFalseValue)));
+        } else {
+            append(new CondMoveOp(result, finalCondition, load(finalTrueValue), loadNonConst(finalFalseValue)));
         }
         return result;
     }
@@ -394,23 +445,21 @@ public abstract class AMD64LIRGenerator extends LIRGenerator {
      *
      * @param a the left operand of the comparison
      * @param b the right operand of the comparison
+     * @param cond the condition of the comparison
      * @return true if the left and right operands were switched, false otherwise
      */
-    private boolean emitCompare(PlatformKind cmpKind, Value a, Value b) {
-        Variable left;
-        Value right;
-        boolean mirrored;
+    private Condition emitCompare(PlatformKind cmpKind, Value a, Value b, Condition cond) {
         if (LIRValueUtil.isVariable(b)) {
-            left = load(b);
-            right = loadNonConst(a);
-            mirrored = true;
+            emitRawCompare(cmpKind, b, a);
+            return cond.mirror();
         } else {
-            left = load(a);
-            right = loadNonConst(b);
-            mirrored = false;
+            emitRawCompare(cmpKind, a, b);
+            return cond;
         }
-        ((AMD64ArithmeticLIRGeneratorTool) arithmeticLIRGen).emitCompareOp((AMD64Kind) cmpKind, left, right);
-        return mirrored;
+    }
+
+    private void emitRawCompare(PlatformKind cmpKind, Value left, Value right) {
+        ((AMD64ArithmeticLIRGeneratorTool) arithmeticLIRGen).emitCompareOp((AMD64Kind) cmpKind, load(left), loadNonConst(right));
     }
 
     @Override

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64LIRKindTool.java

@@ -28,7 +28,7 @@ import org.graalvm.compiler.debug.GraalError;
 
 import jdk.vm.ci.amd64.AMD64Kind;
 
-public class AMD64LIRKindTool implements LIRKindTool {
+public abstract class AMD64LIRKindTool implements LIRKindTool {
 
     @Override
     public LIRKind getIntegerKind(int bits) {
@@ -67,12 +67,8 @@ public class AMD64LIRKindTool implements LIRKindTool {
     }
 
     @Override
-    public LIRKind getNarrowOopKind() {
-        return LIRKind.reference(AMD64Kind.DWORD);
-    }
+    public abstract LIRKind getNarrowOopKind();
 
     @Override
-    public LIRKind getNarrowPointerKind() {
-        return LIRKind.value(AMD64Kind.DWORD);
-    }
+    public abstract LIRKind getNarrowPointerKind();
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/CompressEncoding.java

@@ -34,14 +34,6 @@ public final class CompressEncoding {
         this.shift = shift;
     }
 
-    public int compress(long ptr) {
-        if (ptr == 0L) {
-            return 0;
-        } else {
-            return (int) ((ptr - base) >>> shift);
-        }
-    }
-
     public boolean hasBase() {
         return base != 0;
     }
@@ -58,14 +50,6 @@ public final class CompressEncoding {
         return shift;
     }
 
-    public long uncompress(int ptr) {
-        if (ptr == 0) {
-            return 0L;
-        } else {
-            return ((ptr & 0xFFFFFFFFL) << shift) + base;
-        }
-    }
-
     @Override
     public String toString() {
         return "base: " + base + " shift: " + shift;
@@ -85,8 +69,7 @@ public final class CompressEncoding {
         if (obj instanceof CompressEncoding) {
             CompressEncoding other = (CompressEncoding) obj;
             return base == other.base && shift == other.shift;
-        } else {
+        }
         return false;
     }
 }
-}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/calc/FloatConvert.java

@@ -25,21 +25,23 @@ package org.graalvm.compiler.core.common.calc;
 import org.graalvm.compiler.debug.GraalError;
 
 public enum FloatConvert {
-    F2I(FloatConvertCategory.FloatingPointToInteger),
-    D2I(FloatConvertCategory.FloatingPointToInteger),
-    F2L(FloatConvertCategory.FloatingPointToInteger),
-    D2L(FloatConvertCategory.FloatingPointToInteger),
-    I2F(FloatConvertCategory.IntegerToFloatingPoint),
-    L2F(FloatConvertCategory.IntegerToFloatingPoint),
-    D2F(FloatConvertCategory.FloatingPointToFloatingPoint),
-    I2D(FloatConvertCategory.IntegerToFloatingPoint),
-    L2D(FloatConvertCategory.IntegerToFloatingPoint),
-    F2D(FloatConvertCategory.FloatingPointToFloatingPoint);
+    F2I(FloatConvertCategory.FloatingPointToInteger, 32),
+    D2I(FloatConvertCategory.FloatingPointToInteger, 64),
+    F2L(FloatConvertCategory.FloatingPointToInteger, 32),
+    D2L(FloatConvertCategory.FloatingPointToInteger, 64),
+    I2F(FloatConvertCategory.IntegerToFloatingPoint, 32),
+    L2F(FloatConvertCategory.IntegerToFloatingPoint, 64),
+    D2F(FloatConvertCategory.FloatingPointToFloatingPoint, 64),
+    I2D(FloatConvertCategory.IntegerToFloatingPoint, 32),
+    L2D(FloatConvertCategory.IntegerToFloatingPoint, 64),
+    F2D(FloatConvertCategory.FloatingPointToFloatingPoint, 32);
 
-    private FloatConvertCategory category;
+    private final FloatConvertCategory category;
+    private final int inputBits;
 
-    FloatConvert(FloatConvertCategory category) {
+    FloatConvert(FloatConvertCategory category, int inputBits) {
         this.category = category;
+        this.inputBits = inputBits;
     }
 
     public FloatConvertCategory getCategory() {
@@ -72,4 +74,8 @@ public enum FloatConvert {
                 throw GraalError.shouldNotReachHere();
         }
     }
+
+    public int getInputBits() {
+        return inputBits;
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/cfg/Loop.java

@@ -41,7 +41,6 @@ public abstract class Loop<T extends AbstractBlockBase<T>> {
         this.parent = parent;
         if (parent != null) {
             this.depth = parent.getDepth() + 1;
-            parent.getChildren().add(this);
         } else {
             this.depth = 1;
         }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/ArithmeticOpTable.java

@@ -96,6 +96,22 @@ public final class ArithmeticOpTable {
         }
     }
 
+    public BinaryOp<?>[] getBinaryOps() {
+        return new BinaryOp<?>[]{add, sub, mul, mulHigh, umulHigh, div, rem, and, or, xor};
+    }
+
+    public UnaryOp<?>[] getUnaryOps() {
+        return new UnaryOp<?>[]{neg, not, abs, sqrt};
+    }
+
+    public ShiftOp<?>[] getShiftOps() {
+        return new ShiftOp<?>[]{shl, shr, ushr};
+    }
+
+    public IntegerConvertOp<?>[] getIntegerConvertOps() {
+        return new IntegerConvertOp<?>[]{zeroExtend, signExtend, narrow};
+    }
+
     public static final ArithmeticOpTable EMPTY = new ArithmeticOpTable(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);
 
     public interface ArithmeticOpWrapper {
@@ -562,7 +578,10 @@ public final class ArithmeticOpTable {
         }
 
         /**
-         * Apply the operation to two {@linkplain Constant Constants}.
+         * Applies this operation to {@code a} and {@code b}.
+         *
+         * @return the result of applying this operation or {@code null} if applying it would raise
+         *         an exception (e.g., {@link ArithmeticException} for dividing by 0)
          */
         public abstract Constant foldConstant(Constant a, Constant b);
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/FloatStamp.java

@@ -291,7 +291,7 @@ public class FloatStamp extends PrimitiveStamp {
 
     @Override
     public JavaConstant asConstant() {
-        if (nonNaN && Double.compare(lowerBound, upperBound) == 0) {
+        if (isConstant()) {
             switch (getBits()) {
                 case 32:
                     return JavaConstant.forFloat((float) lowerBound);
@@ -302,6 +302,68 @@ public class FloatStamp extends PrimitiveStamp {
         return null;
     }
 
+    private boolean isConstant() {
+        /*
+         * There are many forms of NaNs and any operations on them can silently convert them into
+         * the canonical NaN.
+         */
+        return (Double.compare(lowerBound, upperBound) == 0 && nonNaN);
+    }
+
+    private static FloatStamp stampForConstant(Constant constant) {
+        FloatStamp result;
+        PrimitiveConstant value = (PrimitiveConstant) constant;
+        switch (value.getJavaKind()) {
+            case Float:
+                if (Float.isNaN(value.asFloat())) {
+                    result = new FloatStamp(32, Double.NaN, Double.NaN, false);
+                } else {
+                    result = new FloatStamp(32, value.asFloat(), value.asFloat(), !Float.isNaN(value.asFloat()));
+                }
+                break;
+            case Double:
+                if (Double.isNaN(value.asDouble())) {
+                    result = new FloatStamp(64, Double.NaN, Double.NaN, false);
+                } else {
+                    result = new FloatStamp(64, value.asDouble(), value.asDouble(), !Double.isNaN(value.asDouble()));
+                }
+                break;
+            default:
+                throw GraalError.shouldNotReachHere();
+        }
+        if (result.isConstant()) {
+            return result;
+        }
+        return null;
+    }
+
+    private static Stamp maybeFoldConstant(UnaryOp<?> op, FloatStamp stamp) {
+        if (stamp.isConstant()) {
+            JavaConstant constant = stamp.asConstant();
+            Constant folded = op.foldConstant(constant);
+            if (folded != null) {
+                return FloatStamp.stampForConstant(folded);
+            }
+        }
+        return null;
+    }
+
+    private static Stamp maybeFoldConstant(BinaryOp<?> op, FloatStamp stamp1, FloatStamp stamp2) {
+        if (stamp1.isConstant() && stamp2.isConstant()) {
+            JavaConstant constant1 = stamp1.asConstant();
+            JavaConstant constant2 = stamp2.asConstant();
+            Constant folded = op.foldConstant(constant1, constant2);
+            if (folded != null) {
+                FloatStamp stamp = stampForConstant(folded);
+                if (stamp != null && stamp.isConstant()) {
+                    assert stamp.asConstant().equals(folded);
+                    return stamp;
+                }
+            }
+        }
+        return null;
+    }
+
     public static final ArithmeticOpTable OPS = new ArithmeticOpTable(
 
                     new UnaryOp.Neg() {
@@ -322,8 +384,13 @@ public class FloatStamp extends PrimitiveStamp {
                         @Override
                         public Stamp foldStamp(Stamp s) {
                             FloatStamp stamp = (FloatStamp) s;
+                            Stamp folded = maybeFoldConstant(this, stamp);
+                            if (folded != null) {
+                                return folded;
+                            }
                             return new FloatStamp(stamp.getBits(), -stamp.upperBound(), -stamp.lowerBound(), stamp.isNonNaN());
                         }
+
                     },
 
                     new BinaryOp.Add(false, true) {
@@ -344,8 +411,13 @@ public class FloatStamp extends PrimitiveStamp {
                         }
 
                         @Override
-                        public Stamp foldStamp(Stamp stamp1, Stamp stamp2) {
-                            // TODO
+                        public Stamp foldStamp(Stamp s1, Stamp s2) {
+                            FloatStamp stamp1 = (FloatStamp) s1;
+                            FloatStamp stamp2 = (FloatStamp) s2;
+                            Stamp folded = maybeFoldConstant(this, stamp1, stamp2);
+                            if (folded != null) {
+                                return folded;
+                            }
                             return stamp1.unrestricted();
                         }
 
@@ -381,8 +453,13 @@ public class FloatStamp extends PrimitiveStamp {
                         }
 
                         @Override
-                        public Stamp foldStamp(Stamp stamp1, Stamp stamp2) {
-                            // TODO
+                        public Stamp foldStamp(Stamp s1, Stamp s2) {
+                            FloatStamp stamp1 = (FloatStamp) s1;
+                            FloatStamp stamp2 = (FloatStamp) s2;
+                            Stamp folded = maybeFoldConstant(this, stamp1, stamp2);
+                            if (folded != null) {
+                                return folded;
+                            }
                             return stamp1.unrestricted();
                         }
 
@@ -418,9 +495,14 @@ public class FloatStamp extends PrimitiveStamp {
                         }
 
                         @Override
-                        public Stamp foldStamp(Stamp a, Stamp b) {
-                            // TODO
-                            return a.unrestricted();
+                        public Stamp foldStamp(Stamp s1, Stamp s2) {
+                            FloatStamp stamp1 = (FloatStamp) s1;
+                            FloatStamp stamp2 = (FloatStamp) s2;
+                            Stamp folded = maybeFoldConstant(this, stamp1, stamp2);
+                            if (folded != null) {
+                                return folded;
+                            }
+                            return stamp1.unrestricted();
                         }
 
                         @Override
@@ -450,17 +532,24 @@ public class FloatStamp extends PrimitiveStamp {
                             assert a.getJavaKind() == b.getJavaKind();
                             switch (a.getJavaKind()) {
                                 case Float:
-                                    return JavaConstant.forFloat(a.asFloat() / b.asFloat());
+                                    float floatDivisor = b.asFloat();
+                                    return (floatDivisor == 0) ? null : JavaConstant.forFloat(a.asFloat() / floatDivisor);
                                 case Double:
-                                    return JavaConstant.forDouble(a.asDouble() / b.asDouble());
+                                    double doubleDivisor = b.asDouble();
+                                    return (doubleDivisor == 0) ? null : JavaConstant.forDouble(a.asDouble() / doubleDivisor);
                                 default:
                                     throw GraalError.shouldNotReachHere();
                             }
                         }
 
                         @Override
-                        public Stamp foldStamp(Stamp stamp1, Stamp stamp2) {
-                            // TODO
+                        public Stamp foldStamp(Stamp s1, Stamp s2) {
+                            FloatStamp stamp1 = (FloatStamp) s1;
+                            FloatStamp stamp2 = (FloatStamp) s2;
+                            Stamp folded = maybeFoldConstant(this, stamp1, stamp2);
+                            if (folded != null) {
+                                return folded;
+                            }
                             return stamp1.unrestricted();
                         }
 
@@ -496,8 +585,13 @@ public class FloatStamp extends PrimitiveStamp {
                         }
 
                         @Override
-                        public Stamp foldStamp(Stamp stamp1, Stamp stamp2) {
-                            // TODO
+                        public Stamp foldStamp(Stamp s1, Stamp s2) {
+                            FloatStamp stamp1 = (FloatStamp) s1;
+                            FloatStamp stamp2 = (FloatStamp) s2;
+                            Stamp folded = maybeFoldConstant(this, stamp1, stamp2);
+                            if (folded != null) {
+                                return folded;
+                            }
                             return stamp1.unrestricted();
                         }
                     },
@@ -521,6 +615,17 @@ public class FloatStamp extends PrimitiveStamp {
 
                         @Override
                         public Stamp foldStamp(Stamp s) {
+                            FloatStamp stamp = (FloatStamp) s;
+                            JavaConstant constant = stamp.asConstant();
+                            if (constant != null) {
+                                Constant folded = foldConstant(constant);
+                                if (folded != null) {
+                                    FloatStamp result = stampForConstant(folded);
+                                    if (result != null && result.isConstant()) {
+                                        return result;
+                                    }
+                                }
+                            }
                             return s.unrestricted();
                         }
                     },
@@ -547,7 +652,13 @@ public class FloatStamp extends PrimitiveStamp {
                         }
 
                         @Override
-                        public Stamp foldStamp(Stamp stamp1, Stamp stamp2) {
+                        public Stamp foldStamp(Stamp s1, Stamp s2) {
+                            FloatStamp stamp1 = (FloatStamp) s1;
+                            FloatStamp stamp2 = (FloatStamp) s2;
+                            Stamp folded = maybeFoldConstant(this, stamp1, stamp2);
+                            if (folded != null) {
+                                return folded;
+                            }
                             return stamp1.unrestricted();
                         }
 
@@ -576,7 +687,9 @@ public class FloatStamp extends PrimitiveStamp {
                                 case Float:
                                     int fa = Float.floatToRawIntBits(a.asFloat());
                                     int fb = Float.floatToRawIntBits(b.asFloat());
-                                    return JavaConstant.forFloat(Float.intBitsToFloat(fa | fb));
+                                    float floatOr = Float.intBitsToFloat(fa | fb);
+                                    assert (fa | fb) == Float.floatToRawIntBits((floatOr));
+                                    return JavaConstant.forFloat(floatOr);
                                 case Double:
                                     long da = Double.doubleToRawLongBits(a.asDouble());
                                     long db = Double.doubleToRawLongBits(b.asDouble());
@@ -587,7 +700,13 @@ public class FloatStamp extends PrimitiveStamp {
                         }
 
                         @Override
-                        public Stamp foldStamp(Stamp stamp1, Stamp stamp2) {
+                        public Stamp foldStamp(Stamp s1, Stamp s2) {
+                            FloatStamp stamp1 = (FloatStamp) s1;
+                            FloatStamp stamp2 = (FloatStamp) s2;
+                            Stamp folded = maybeFoldConstant(this, stamp1, stamp2);
+                            if (folded != null) {
+                                return folded;
+                            }
                             return stamp1.unrestricted();
                         }
 
@@ -627,7 +746,13 @@ public class FloatStamp extends PrimitiveStamp {
                         }
 
                         @Override
-                        public Stamp foldStamp(Stamp stamp1, Stamp stamp2) {
+                        public Stamp foldStamp(Stamp s1, Stamp s2) {
+                            FloatStamp stamp1 = (FloatStamp) s1;
+                            FloatStamp stamp2 = (FloatStamp) s2;
+                            Stamp folded = maybeFoldConstant(this, stamp1, stamp2);
+                            if (folded != null) {
+                                return folded;
+                            }
                             return stamp1.unrestricted();
                         }
 
@@ -665,6 +790,10 @@ public class FloatStamp extends PrimitiveStamp {
                         @Override
                         public Stamp foldStamp(Stamp s) {
                             FloatStamp stamp = (FloatStamp) s;
+                            Stamp folded = maybeFoldConstant(this, stamp);
+                            if (folded != null) {
+                                return folded;
+                            }
                             if (stamp.isNaN()) {
                                 return stamp;
                             }
@@ -689,6 +818,11 @@ public class FloatStamp extends PrimitiveStamp {
 
                         @Override
                         public Stamp foldStamp(Stamp s) {
+                            FloatStamp stamp = (FloatStamp) s;
+                            Stamp folded = maybeFoldConstant(this, stamp);
+                            if (folded != null) {
+                                return folded;
+                            }
                             return s.unrestricted();
                         }
                     },

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/IntegerStamp.java

@@ -597,6 +597,10 @@ public final class IntegerStamp extends PrimitiveStamp {
                         public Stamp foldStamp(Stamp s) {
                             IntegerStamp stamp = (IntegerStamp) s;
                             int bits = stamp.getBits();
+                            if (stamp.lowerBound == stamp.upperBound) {
+                                long value = CodeUtil.convert(-stamp.lowerBound(), stamp.getBits(), false);
+                                return StampFactory.forInteger(stamp.getBits(), value, value);
+                            }
                             if (stamp.lowerBound() != CodeUtil.minValue(bits)) {
                                 // TODO(ls) check if the mask calculation is correct...
                                 return StampFactory.forInteger(bits, -stamp.upperBound(), -stamp.lowerBound());
@@ -624,6 +628,11 @@ public final class IntegerStamp extends PrimitiveStamp {
                             int bits = a.getBits();
                             assert bits == b.getBits();
 
+                            if (a.lowerBound == a.upperBound && b.lowerBound == b.upperBound) {
+                                long value = CodeUtil.convert(a.lowerBound() + b.lowerBound(), a.getBits(), false);
+                                return StampFactory.forInteger(a.getBits(), value, value);
+                            }
+
                             if (a.isUnrestricted()) {
                                 return a;
                             } else if (b.isUnrestricted()) {
@@ -711,6 +720,12 @@ public final class IntegerStamp extends PrimitiveStamp {
 
                             int bits = a.getBits();
                             assert bits == b.getBits();
+
+                            if (a.lowerBound == a.upperBound && b.lowerBound == b.upperBound) {
+                                long value = CodeUtil.convert(a.lowerBound() * b.lowerBound(), a.getBits(), false);
+                                return StampFactory.forInteger(a.getBits(), value, value);
+                            }
+
                             // if a==0 or b==0 result of a*b is always 0
                             if (a.upMask() == 0) {
                                 return a;
@@ -791,7 +806,7 @@ public final class IntegerStamp extends PrimitiveStamp {
                                 long maxPosB = b.upperBound();
 
                                 // multiplication has shift semantics
-                                long newUpMask = ~CodeUtil.mask(Long.numberOfTrailingZeros(a.upMask) + Long.numberOfTrailingZeros(b.upMask)) & CodeUtil.mask(bits);
+                                long newUpMask = ~CodeUtil.mask(Math.min(64, Long.numberOfTrailingZeros(a.upMask) + Long.numberOfTrailingZeros(b.upMask))) & CodeUtil.mask(bits);
 
                                 if (a.canBePositive()) {
                                     if (b.canBePositive()) {
@@ -1023,6 +1038,9 @@ public final class IntegerStamp extends PrimitiveStamp {
                             PrimitiveConstant a = (PrimitiveConstant) const1;
                             PrimitiveConstant b = (PrimitiveConstant) const2;
                             assert a.getJavaKind() == b.getJavaKind();
+                            if (b.asLong() == 0) {
+                                return null;
+                            }
                             return JavaConstant.forIntegerKind(a.getJavaKind(), a.asLong() / b.asLong());
                         }
 
@@ -1031,9 +1049,12 @@ public final class IntegerStamp extends PrimitiveStamp {
                             IntegerStamp a = (IntegerStamp) stamp1;
                             IntegerStamp b = (IntegerStamp) stamp2;
                             assert a.getBits() == b.getBits();
-                            if (b.isStrictlyPositive()) {
-                                long newLowerBound = a.lowerBound() / b.upperBound();
-                                long newUpperBound = a.upperBound() / b.lowerBound();
+                            if (a.lowerBound == a.upperBound && b.lowerBound == b.upperBound && b.lowerBound != 0) {
+                                long value = CodeUtil.convert(a.lowerBound() / b.lowerBound(), a.getBits(), false);
+                                return StampFactory.forInteger(a.getBits(), value, value);
+                            } else if (b.isStrictlyPositive()) {
+                                long newLowerBound = a.lowerBound() < 0 ? a.lowerBound() / b.lowerBound() : a.lowerBound() / b.upperBound();
+                                long newUpperBound = a.upperBound() < 0 ? a.upperBound() / b.upperBound() : a.upperBound() / b.lowerBound();
                                 return StampFactory.forInteger(a.getBits(), newLowerBound, newUpperBound);
                             } else {
                                 return a.unrestricted();
@@ -1054,6 +1075,9 @@ public final class IntegerStamp extends PrimitiveStamp {
                             PrimitiveConstant a = (PrimitiveConstant) const1;
                             PrimitiveConstant b = (PrimitiveConstant) const2;
                             assert a.getJavaKind() == b.getJavaKind();
+                            if (b.asLong() == 0) {
+                                return null;
+                            }
                             return JavaConstant.forIntegerKind(a.getJavaKind(), a.asLong() % b.asLong());
                         }
 
@@ -1062,6 +1086,12 @@ public final class IntegerStamp extends PrimitiveStamp {
                             IntegerStamp a = (IntegerStamp) stamp1;
                             IntegerStamp b = (IntegerStamp) stamp2;
                             assert a.getBits() == b.getBits();
+
+                            if (a.lowerBound == a.upperBound && b.lowerBound == b.upperBound && b.lowerBound != 0) {
+                                long value = CodeUtil.convert(a.lowerBound() % b.lowerBound(), a.getBits(), false);
+                                return StampFactory.forInteger(a.getBits(), value, value);
+                            }
+
                             // zero is always possible
                             long newLowerBound = Math.min(a.lowerBound(), 0);
                             long newUpperBound = Math.max(a.upperBound(), 0);
@@ -1364,6 +1394,10 @@ public final class IntegerStamp extends PrimitiveStamp {
                         public Stamp foldStamp(Stamp input) {
                             IntegerStamp stamp = (IntegerStamp) input;
                             int bits = stamp.getBits();
+                            if (stamp.lowerBound == stamp.upperBound) {
+                                long value = CodeUtil.convert(Math.abs(stamp.lowerBound()), stamp.getBits(), false);
+                                return StampFactory.forInteger(stamp.getBits(), value, value);
+                            }
                             if (stamp.lowerBound() == CodeUtil.minValue(bits)) {
                                 return input.unrestricted();
                             } else {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/CheckGraalInvariants.java

@@ -49,8 +49,8 @@ import org.graalvm.compiler.core.CompilerThreadFactory;
 import org.graalvm.compiler.core.common.LIRKind;
 import org.graalvm.compiler.core.common.type.ArithmeticOpTable;
 import org.graalvm.compiler.debug.DebugCloseable;
-import org.graalvm.compiler.debug.DebugHandlersFactory;
 import org.graalvm.compiler.debug.DebugContext;
+import org.graalvm.compiler.debug.DebugHandlersFactory;
 import org.graalvm.compiler.debug.GraalError;
 import org.graalvm.compiler.graph.Node;
 import org.graalvm.compiler.graph.NodeClass;
@@ -73,6 +73,8 @@ import org.graalvm.compiler.phases.util.Providers;
 import org.graalvm.compiler.phases.verify.VerifyBailoutUsage;
 import org.graalvm.compiler.phases.verify.VerifyCallerSensitiveMethods;
 import org.graalvm.compiler.phases.verify.VerifyDebugUsage;
+import org.graalvm.compiler.phases.verify.VerifyGetOptionsUsage;
+import org.graalvm.compiler.phases.verify.VerifyGraphAddUsage;
 import org.graalvm.compiler.phases.verify.VerifyInstanceOfUsage;
 import org.graalvm.compiler.phases.verify.VerifyUpdateUsages;
 import org.graalvm.compiler.phases.verify.VerifyUsageWithEquals;
@@ -381,6 +383,8 @@ public class CheckGraalInvariants extends GraalCompilerTest {
         new VerifyUpdateUsages().apply(graph, context);
         new VerifyBailoutUsage().apply(graph, context);
         new VerifyInstanceOfUsage().apply(graph, context);
+        new VerifyGraphAddUsage().apply(graph, context);
+        new VerifyGetOptionsUsage().apply(graph, context);
         if (graph.method().isBridge()) {
             BridgeMethodUtils.getBridgedMethod(graph.method());
         }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ConditionalEliminationTest14.java

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2015, 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.core.test;
+
+import org.graalvm.compiler.graph.iterators.NodeIterable;
+import org.graalvm.compiler.nodes.ConstantNode;
+import org.graalvm.compiler.nodes.FixedGuardNode;
+import org.graalvm.compiler.nodes.GuardNode;
+import org.graalvm.compiler.nodes.LogicNode;
+import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions;
+import org.graalvm.compiler.nodes.calc.IntegerBelowNode;
+import org.graalvm.compiler.nodes.java.LoadIndexedNode;
+import org.graalvm.compiler.nodes.memory.FloatingReadNode;
+import org.graalvm.compiler.nodes.memory.ReadNode;
+import org.graalvm.compiler.nodes.spi.LoweringTool;
+import org.graalvm.compiler.phases.common.CanonicalizerPhase;
+import org.graalvm.compiler.phases.common.FloatingReadPhase;
+import org.graalvm.compiler.phases.common.IterativeConditionalEliminationPhase;
+import org.graalvm.compiler.phases.common.LoweringPhase;
+import org.graalvm.compiler.phases.tiers.PhaseContext;
+import org.junit.Assert;
+import org.junit.Test;
+
+import jdk.vm.ci.meta.DeoptimizationReason;
+
+/**
+ * Check that multiple bounds checks are correctly grouped together.
+ */
+public class ConditionalEliminationTest14 extends ConditionalEliminationTestBase {
+
+    public static void test1Snippet(Object[] args) {
+        Object a5 = args[5];
+        Object a7 = args[7];
+        Object a6 = args[6];
+
+        /*
+         * The order of the conditions matters: The scheduler processes the floating reads for the
+         * array loads in the order of the conditions here, and we want the index 7 access to be
+         * processed before the index 6 access.
+         */
+        if (a5 != null && a7 != null && a6 != null) {
+            sink1 = 1;
+        }
+        sink0 = 0;
+    }
+
+    @Test
+    public void test1() {
+        StructuredGraph graph = parseEager("test1Snippet", AllowAssumptions.YES);
+        CanonicalizerPhase canonicalizer = new CanonicalizerPhase();
+        PhaseContext context = new PhaseContext(getProviders());
+
+        /* Convert the LoadIndexNode to ReadNode with floating guards. */
+        new LoweringPhase(canonicalizer, LoweringTool.StandardLoweringStage.HIGH_TIER).apply(graph, context);
+        /* Convert the ReadNode to FloatingReadNode. */
+        new FloatingReadPhase().apply(graph);
+        /* Apply the phase that we want to test. */
+        new IterativeConditionalEliminationPhase(canonicalizer, true).apply(graph, context);
+
+        Assert.assertEquals("All guards must be floating", 0, graph.getNodes(FixedGuardNode.TYPE).count());
+        Assert.assertEquals("All array accesses must have been lowered", 0, graph.getNodes().filter(LoadIndexedNode.class).count());
+        Assert.assertEquals("All reads must be floating", 0, graph.getNodes().filter(ReadNode.class).count());
+        Assert.assertEquals("Must have floating reads (3 array accesses, 1 array length)", 4, graph.getNodes().filter(FloatingReadNode.class).count());
+
+        NodeIterable<GuardNode> boundsChecks = graph.getNodes(GuardNode.TYPE).filter(n -> ((GuardNode) n).getReason() == DeoptimizationReason.BoundsCheckException);
+        Assert.assertEquals("Must have only 1 bounds check remaining", 1, boundsChecks.count());
+        LogicNode condition = boundsChecks.first().getCondition();
+        Assert.assertTrue("Bounds check must check for array length 8", condition instanceof IntegerBelowNode && ((IntegerBelowNode) condition).getY().valueEquals(ConstantNode.forInt(8)));
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ConditionalEliminationTestBase.java

@@ -27,12 +27,15 @@ import org.graalvm.compiler.nodes.ProxyNode;
 import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions;
 import org.graalvm.compiler.nodes.spi.LoweringTool;
+import org.graalvm.compiler.phases.OptimisticOptimizations;
+import org.graalvm.compiler.phases.OptimisticOptimizations.Optimization;
 import org.graalvm.compiler.phases.common.CanonicalizerPhase;
 import org.graalvm.compiler.phases.common.ConditionalEliminationPhase;
 import org.graalvm.compiler.phases.common.ConvertDeoptimizeToGuardPhase;
 import org.graalvm.compiler.phases.common.IterativeConditionalEliminationPhase;
 import org.graalvm.compiler.phases.common.LoweringPhase;
 import org.graalvm.compiler.phases.schedule.SchedulePhase;
+import org.graalvm.compiler.phases.tiers.HighTierContext;
 import org.graalvm.compiler.phases.tiers.PhaseContext;
 import org.junit.Assert;
 
@@ -45,6 +48,15 @@ public class ConditionalEliminationTestBase extends GraalCompilerTest {
     protected static int sink1;
     protected static int sink2;
 
+    /**
+     * These tests assume all code paths in called routines are reachable so disable removal of dead
+     * code based on method profiles.
+     */
+    @Override
+    protected HighTierContext getDefaultHighTierContext() {
+        return new HighTierContext(getProviders(), getDefaultGraphBuilderSuite(), OptimisticOptimizations.ALL.remove(Optimization.RemoveNeverExecutedCode));
+    }
+
     protected void testConditionalElimination(String snippet, String referenceSnippet) {
         testConditionalElimination(snippet, referenceSnippet, false, false);
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/DumpPathTest.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.core.test;
+
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.graalvm.compiler.debug.DebugOptions;
+import org.graalvm.compiler.options.OptionKey;
+import org.graalvm.compiler.options.OptionValues;
+import org.graalvm.util.EconomicMap;
+import org.junit.Test;
+
+/**
+ * Check that setting the dump path results in files ending up in the right directory with matching
+ * names.
+ */
+public class DumpPathTest extends GraalCompilerTest {
+
+    public static Object snippet() {
+        return new String("snippet");
+    }
+
+    @Test
+    public void testDump() throws IOException {
+        Path dumpDirectoryPath = Files.createTempDirectory("DumpPathTest");
+        String[] extensions = new String[]{".cfg", ".bgv", ".graph-strings"};
+        EconomicMap<OptionKey<?>, Object> overrides = OptionValues.newOptionMap();
+        overrides.put(DebugOptions.DumpPath, dumpDirectoryPath.toString());
+        overrides.put(DebugOptions.PrintGraphFile, true);
+        overrides.put(DebugOptions.PrintCanonicalGraphStrings, true);
+        overrides.put(DebugOptions.Dump, "*");
+
+        // Generate dump files.
+        test(new OptionValues(getInitialOptions(), overrides), "snippet");
+        // Check that Ideal files got created, in the right place.
+        checkForFiles(dumpDirectoryPath, extensions);
+
+        // Clean up the generated files.
+        scrubDirectory(dumpDirectoryPath);
+    }
+
+    /**
+     * Check that the given directory contains file or directory names with all the given
+     * extensions.
+     */
+    private static void checkForFiles(Path directoryPath, String[] extensions) throws IOException {
+        String[] paths = new String[extensions.length];
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(directoryPath)) {
+            for (Path filePath : stream) {
+                String fileName = filePath.getFileName().toString();
+                for (int i = 0; i < extensions.length; i++) {
+                    String extension = extensions[i];
+                    if (fileName.endsWith(extensions[i])) {
+                        assertTrue(paths[i] == null, "multiple files found for %s in %s", extension, directoryPath);
+                        paths[i] = fileName.replace(extensions[i], "");
+                    }
+                }
+            }
+        }
+        for (int i = 0; i < paths.length; i++) {
+            assertTrue(paths[i] != null, "missing file for extension %s in %s", extensions[i], directoryPath);
+        }
+        // Ensure that all file names are the same.
+        for (int i = 1; i < paths.length; i++) {
+            assertTrue(paths[0].equals(paths[i]), paths[0] + " != " + paths[i]);
+        }
+    }
+
+    /**
+     * Remove the temporary directory.
+     */
+    private static void scrubDirectory(Path directoryPath) {
+        try {
+            try (DirectoryStream<Path> stream = Files.newDirectoryStream(directoryPath)) {
+                for (Path filePath : stream) {
+                    if (Files.isRegularFile(filePath)) {
+                        Files.delete(filePath);
+                    } else if (Files.isDirectory(filePath)) {
+                        scrubDirectory(filePath);
+                    }
+                }
+            }
+            Files.delete(directoryPath);
+        } catch (IOException ioe) {
+            ioe.printStackTrace();
+        }
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/FinalizableSubclassTest.java

@@ -72,7 +72,7 @@ public class FinalizableSubclassTest extends GraalCompilerTest {
         Assert.assertTrue(constructors.length == 1);
         final ResolvedJavaMethod javaMethod = getMetaAccess().lookupJavaMethod(constructors[0]);
         OptionValues options = getInitialOptions();
-        StructuredGraph graph = new StructuredGraph.Builder(options, getDebugContext(options), allowAssumptions).method(javaMethod).build();
+        StructuredGraph graph = new StructuredGraph.Builder(options, getDebugContext(options, null, javaMethod), allowAssumptions).method(javaMethod).build();
 
         GraphBuilderConfiguration conf = GraphBuilderConfiguration.getSnippetDefault(getDefaultGraphBuilderPlugins());
         new GraphBuilderPhase.Instance(getMetaAccess(), getProviders().getStampProvider(), getProviders().getConstantReflection(), getProviders().getConstantFieldProvider(), conf,

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GraalCompilerTest.java

@@ -389,7 +389,7 @@ public abstract class GraalCompilerTest extends GraalTest {
      * {@link DebugDumpHandler}s closed in {@link #afterTest()}.
      */
     protected DebugContext getDebugContext() {
-        return getDebugContext(getInitialOptions());
+        return getDebugContext(getInitialOptions(), null, null);
     }
 
     @Override
@@ -862,7 +862,7 @@ public abstract class GraalCompilerTest extends GraalTest {
         Result actual = executeActual(options, method, receiver, args);
         profile = method.getProfilingInfo(); // profile can change after execution
         for (DeoptimizationReason reason : shouldNotDeopt) {
-            Assert.assertEquals((int) deoptCounts.get(reason), profile.getDeoptimizationCount(reason));
+            Assert.assertEquals("wrong number of deopt counts for " + reason, (int) deoptCounts.get(reason), profile.getDeoptimizationCount(reason));
         }
         return actual;
     }
@@ -1216,15 +1216,15 @@ public abstract class GraalCompilerTest extends GraalTest {
 
     protected final Builder builder(ResolvedJavaMethod method, AllowAssumptions allowAssumptions) {
         OptionValues options = getInitialOptions();
-        return new Builder(options, getDebugContext(options), allowAssumptions).method(method).compilationId(getCompilationId(method));
+        return new Builder(options, getDebugContext(options, null, method), allowAssumptions).method(method).compilationId(getCompilationId(method));
     }
 
     protected final Builder builder(ResolvedJavaMethod method, AllowAssumptions allowAssumptions, CompilationIdentifier compilationId, OptionValues options) {
-        return new Builder(options, getDebugContext(options), allowAssumptions).method(method).compilationId(compilationId);
+        return new Builder(options, getDebugContext(options, compilationId.toString(CompilationIdentifier.Verbosity.ID), method), allowAssumptions).method(method).compilationId(compilationId);
     }
 
     protected final Builder builder(ResolvedJavaMethod method, AllowAssumptions allowAssumptions, OptionValues options) {
-        return new Builder(options, getDebugContext(options), allowAssumptions).method(method).compilationId(getCompilationId(method));
+        return new Builder(options, getDebugContext(options, null, method), allowAssumptions).method(method).compilationId(getCompilationId(method));
     }
 
     protected PhaseSuite<HighTierContext> getDebugGraphBuilderSuite() {
@@ -1234,6 +1234,7 @@ public abstract class GraalCompilerTest extends GraalTest {
     @SuppressWarnings("try")
     protected StructuredGraph parse(StructuredGraph.Builder builder, PhaseSuite<HighTierContext> graphBuilderSuite) {
         ResolvedJavaMethod javaMethod = builder.getMethod();
+        builder.speculationLog(getSpeculationLog());
         if (builder.getCancellable() == null) {
             builder.cancellable(getCancellable(javaMethod));
         }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GraalDebugHandlersFactoryTest.java

@@ -31,8 +31,12 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Comparator;
 
-import org.graalvm.compiler.printer.GraalDebugHandlersFactory;
+import org.graalvm.compiler.debug.DebugOptions;
+import org.graalvm.compiler.debug.PathUtilities;
+import org.graalvm.compiler.options.OptionKey;
+import org.graalvm.compiler.options.OptionValues;
 import org.graalvm.compiler.test.AddExports;
+import org.junit.Assume;
 import org.junit.Test;
 
 @AddExports("jdk.internal.vm.compiler/org.graalvm.compiler.printer")
@@ -40,23 +44,28 @@ public class GraalDebugHandlersFactoryTest extends GraalCompilerTest {
 
     @Test
     public void createUniqueTest() throws Exception {
-        Field maxFileNameLengthField = GraalDebugHandlersFactory.class.getDeclaredField("MAX_FILE_NAME_LENGTH");
+        Field maxFileNameLengthField = PathUtilities.class.getDeclaredField("MAX_FILE_NAME_LENGTH");
+        try {
             maxFileNameLengthField.setAccessible(true);
+        } catch (RuntimeException ex) {
+            Assume.assumeFalse("If InaccessibleObjectException is thrown, skip the test, we are on JDK9", ex.getClass().getSimpleName().equals("InaccessibleObjectException"));
+        }
         int maxFileNameLength = maxFileNameLengthField.getInt(null);
-        Method createUniqueMethod = GraalDebugHandlersFactory.class.getDeclaredMethod("createUnique", Path.class, String.class, String.class, String.class, boolean.class);
+        Method createUniqueMethod = PathUtilities.class.getDeclaredMethod("createUnique", OptionValues.class, OptionKey.class, String.class, String.class, String.class, boolean.class);
         createUniqueMethod.setAccessible(true);
         Path tmpDir = Files.createTempDirectory(Paths.get("."), "createUniqueTest");
+        OptionValues options = new OptionValues(OptionValues.asMap(DebugOptions.DumpPath, tmpDir.toString()));
         try {
             for (boolean createDirectory : new boolean[]{true, false}) {
                 for (String ext : new String[]{"", ".bgv", ".graph-strings"}) {
                     for (int i = 0; i < maxFileNameLength + 5; i++) {
                         String id = new String(new char[i]).replace('\0', 'i');
                         String label = "";
-                        createUniqueMethod.invoke(null, tmpDir, id, label, ext, createDirectory);
+                        createUniqueMethod.invoke(null, options, null, id, label, ext, createDirectory);
 
                         id = "";
                         label = new String(new char[i]).replace('\0', 'l');
-                        createUniqueMethod.invoke(null, tmpDir, id, label, ext, createDirectory);
+                        createUniqueMethod.invoke(null, options, null, id, label, ext, createDirectory);
                     }
                 }
             }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GuardedIntrinsicTest.java

@@ -144,6 +144,11 @@ public class GuardedIntrinsicTest extends GraalCompilerTest {
     public void test01() {
         Super inheritsHC = new Super();
         Person overridesHC = new Person(0);
+
+        // Ensure the profile for getSuperAge includes both receiver types
+        getSuperAge(inheritsHC);
+        getSuperAge(overridesHC);
+
         test("getSuperAge", inheritsHC);
         test("getSuperAge", overridesHC);
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/HashCodeTest.java

@@ -22,6 +22,8 @@
  */
 package org.graalvm.compiler.core.test;
 
+import java.util.HashMap;
+
 import org.graalvm.compiler.core.phases.HighTier;
 import org.graalvm.compiler.core.phases.MidTier;
 import org.graalvm.compiler.nodes.InvokeNode;
@@ -139,6 +141,10 @@ public class HashCodeTest extends GraalCompilerTest {
     public void test08() {
         initialize(Appendable.class);
         checkForGuardedIntrinsicPattern("hashCodeInterface");
+
+        // Ensure the profile for the dispatch in hashCodeSnippet01
+        // has a receiver type that does not select Object.hashCode intrinsic
+        hashCodeSnippet01(new HashMap<>());
         checkForGuardedIntrinsicPattern("hashCodeSnippet01");
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/MarkUnsafeAccessTest.java

@@ -25,6 +25,7 @@ package org.graalvm.compiler.core.test;
 import static java.nio.file.StandardOpenOption.READ;
 import static java.nio.file.StandardOpenOption.WRITE;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.MappedByteBuffer;
@@ -33,22 +34,20 @@ import java.nio.channels.FileChannel.MapMode;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
-import jdk.vm.ci.code.InstalledCode;
-import jdk.vm.ci.code.InvalidInstalledCodeException;
-import jdk.vm.ci.meta.ResolvedJavaMethod;
-import jdk.vm.ci.meta.ResolvedJavaType;
-
-import org.junit.Assert;
-import org.junit.Assume;
-import org.junit.Test;
-
-import sun.misc.Unsafe;
-
 import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.phases.common.CanonicalizerPhase;
 import org.graalvm.compiler.phases.common.inlining.InliningPhase;
 import org.graalvm.compiler.phases.common.inlining.policy.InlineEverythingPolicy;
 import org.graalvm.compiler.phases.tiers.HighTierContext;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+
+import jdk.vm.ci.code.InstalledCode;
+import jdk.vm.ci.code.InvalidInstalledCodeException;
+import jdk.vm.ci.meta.ResolvedJavaMethod;
+import jdk.vm.ci.meta.ResolvedJavaType;
+import sun.misc.Unsafe;
 
 public class MarkUnsafeAccessTest extends GraalCompilerTest {
 
@@ -170,7 +169,9 @@ public class MarkUnsafeAccessTest extends GraalCompilerTest {
         try {
             mbb.position(BLOCK_SIZE);
             getter.get(mbb);
-            System.currentTimeMillis(); // materialize async exception
+
+            // Make a call that goes into native code to materialize async exception
+            new File("").exists();
         } catch (InternalError e) {
             return;
         }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/MergeCanonicalizerTest.java

@@ -26,12 +26,23 @@ import org.graalvm.compiler.debug.DebugContext;
 import org.graalvm.compiler.nodes.ReturnNode;
 import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions;
+import org.graalvm.compiler.phases.OptimisticOptimizations;
+import org.graalvm.compiler.phases.OptimisticOptimizations.Optimization;
 import org.graalvm.compiler.phases.common.CanonicalizerPhase;
+import org.graalvm.compiler.phases.tiers.HighTierContext;
 import org.graalvm.compiler.phases.tiers.PhaseContext;
 import org.junit.Test;
 
 public class MergeCanonicalizerTest extends GraalCompilerTest {
 
+    /**
+     * These tests assume all code paths are reachable so disable profile based dead code removal.
+     */
+    @Override
+    protected HighTierContext getDefaultHighTierContext() {
+        return new HighTierContext(getProviders(), getDefaultGraphBuilderSuite(), OptimisticOptimizations.ALL.remove(Optimization.RemoveNeverExecutedCode));
+    }
+
     public static int staticField;
 
     private int field;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ProfilingInfoTest.java

@@ -24,15 +24,17 @@ package org.graalvm.compiler.core.test;
 
 import java.io.Serializable;
 
+import org.graalvm.compiler.test.SubprocessUtil;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+
 import jdk.vm.ci.meta.JavaTypeProfile;
 import jdk.vm.ci.meta.ProfilingInfo;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
 import jdk.vm.ci.meta.ResolvedJavaType;
 import jdk.vm.ci.meta.TriState;
 
-import org.junit.Assert;
-import org.junit.Test;
-
 /**
  * Tests profiling information provided by the runtime.
  * <p>
@@ -40,7 +42,7 @@ import org.junit.Test;
  * information may be gathered for any given method. For example, HotSpot's advanced compilation
  * policy can decide to only gather partial profiles in a first level compilation (see
  * AdvancedThresholdPolicy::common(...) in advancedThresholdPolicy.cpp). Because of this,
- * occasionally tests for {@link ProfilingInfo#getNullSeen(int)} can fail since HotSpot only set's
+ * occasionally tests for {@link ProfilingInfo#getNullSeen(int)} can fail since HotSpot only sets
  * the null_seen bit when doing full profiling.
  */
 public class ProfilingInfoTest extends GraalCompilerTest {
@@ -182,6 +184,14 @@ public class ProfilingInfoTest extends GraalCompilerTest {
         Assert.assertNull(typeProfile);
     }
 
+    public ProfilingInfoTest() {
+        // These tests are explicitly testing the profiling behavior of the
+        // interpreter. C1-based profiling differs slightly and when -Xcomp
+        // is present, profiles will be created by C1 compiled code, not the
+        // interpreter.
+        Assume.assumeTrue(!SubprocessUtil.getVMCommandLine().contains("-Xcomp"));
+    }
+
     @Test
     public void testExceptionSeen() {
         // NullPointerException

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/SubWordReturnTest.java

@@ -0,0 +1,143 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.core.test;
+
+import java.util.ArrayList;
+import java.util.List;
+import jdk.vm.ci.meta.JavaKind;
+import jdk.vm.ci.meta.ResolvedJavaMethod;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.FieldVisitor;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+
+@RunWith(Parameterized.class)
+public class SubWordReturnTest extends GraalCompilerTest {
+
+    private final JavaKind kind;
+    private final int value;
+
+    private final String generatedClassName;
+    private final String generatedClassNameInternal;
+
+    private final String testMethodName;
+
+    /**
+     * The {@link AsmLoader} generates a class looking like this for the types byte, short, int and
+     * char.
+     */
+    static class ByteGetter {
+
+        // private static int intField = 1000000;
+
+        private static byte get() {
+            // GETSTATIC intField
+            // IRETURN
+            return 0;
+        }
+
+        public static int testByteSnippet() {
+            return get();
+        }
+    }
+
+    @Parameters(name = "{0}, {1}")
+    public static List<Object[]> data() {
+        ArrayList<Object[]> ret = new ArrayList<>();
+        for (int i : new int[]{1000000, 1000001, -1000000, -1}) {
+            ret.add(new Object[]{JavaKind.Boolean, i});
+            ret.add(new Object[]{JavaKind.Byte, i});
+            ret.add(new Object[]{JavaKind.Short, i});
+            ret.add(new Object[]{JavaKind.Char, i});
+        }
+        return ret;
+    }
+
+    public SubWordReturnTest(JavaKind kind, int value) {
+        this.kind = kind;
+        this.value = value;
+
+        this.generatedClassName = SubWordReturnTest.class.getName() + "$" + kind.toString() + "Getter";
+        this.generatedClassNameInternal = generatedClassName.replace('.', '/');
+        this.testMethodName = "test" + kind.name() + "Snippet";
+    }
+
+    @Test
+    public void test() throws ClassNotFoundException {
+        Class<?> testClass = new AsmLoader(SubWordReturnTest.class.getClassLoader()).findClass(generatedClassName);
+        ResolvedJavaMethod method = getResolvedJavaMethod(testClass, testMethodName);
+        test(method, null);
+    }
+
+    class AsmLoader extends ClassLoader implements Opcodes {
+
+        Class<?> loaded;
+
+        AsmLoader(ClassLoader parent) {
+            super(parent);
+        }
+
+        @Override
+        protected Class<?> findClass(String name) throws ClassNotFoundException {
+            if (name.equals(generatedClassName)) {
+                if (loaded == null) {
+                    byte[] gen = generateClass();
+                    loaded = defineClass(name, gen, 0, gen.length);
+                }
+                return loaded;
+            } else {
+                return super.findClass(name);
+            }
+        }
+
+        private byte[] generateClass() {
+            ClassWriter cw = new ClassWriter(0);
+            cw.visit(52, ACC_SUPER | ACC_PUBLIC, generatedClassNameInternal, null, "java/lang/Object", null);
+
+            FieldVisitor intField = cw.visitField(ACC_PRIVATE | ACC_STATIC, "intField", "I", null, value);
+            intField.visitEnd();
+
+            MethodVisitor get = cw.visitMethod(ACC_PRIVATE | ACC_STATIC, "get", "()" + kind.getTypeChar(), null, null);
+            get.visitCode();
+            get.visitFieldInsn(GETSTATIC, generatedClassNameInternal, "intField", "I");
+            get.visitInsn(IRETURN);
+            get.visitMaxs(1, 0);
+            get.visitEnd();
+
+            MethodVisitor snippet = cw.visitMethod(ACC_PUBLIC | ACC_STATIC, testMethodName, "()I", null, null);
+            snippet.visitCode();
+            snippet.visitMethodInsn(INVOKESTATIC, generatedClassNameInternal, "get", "()" + kind.getTypeChar(), false);
+            snippet.visitInsn(IRETURN);
+            snippet.visitMaxs(1, 0);
+            snippet.visitEnd();
+
+            cw.visitEnd();
+            return cw.toByteArray();
+        }
+    }
+
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/UnbalancedMonitorsTest.java

@@ -87,7 +87,7 @@ public class UnbalancedMonitorsTest extends GraalCompilerTest {
         ResolvedJavaMethod method = getResolvedJavaMethod(LOADER.findClass(INNER_CLASS_NAME), name);
         try {
             OptionValues options = getInitialOptions();
-            StructuredGraph graph = new StructuredGraph.Builder(options, getDebugContext(options)).method(method).build();
+            StructuredGraph graph = new StructuredGraph.Builder(options, getDebugContext(options, null, method)).method(method).build();
             Plugins plugins = new Plugins(new InvocationPlugins());
             GraphBuilderConfiguration graphBuilderConfig = GraphBuilderConfiguration.getDefault(plugins).withEagerResolving(true).withUnresolvedIsError(true);
             OptimisticOptimizations optimisticOpts = OptimisticOptimizations.NONE;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/UnsafeVirtualizationTest.java

@@ -32,9 +32,20 @@ import org.graalvm.compiler.phases.tiers.PhaseContext;
 import org.graalvm.compiler.virtual.phases.ea.PartialEscapePhase;
 import org.junit.Test;
 
+import jdk.vm.ci.code.InstalledCode;
+import jdk.vm.ci.meta.ResolvedJavaMethod;
+
 public class UnsafeVirtualizationTest extends GraalCompilerTest {
 
-    public static class A {
+    public static class Base {
+        /*
+         * This padding ensure that the size of the Base class ends up as a multiple of 8, which
+         * makes the first field of the subclass 8-byte aligned.
+         */
+        double padding;
+    }
+
+    public static class A extends Base {
         int f1;
         int f2;
     }
@@ -56,39 +67,96 @@ public class UnsafeVirtualizationTest extends GraalCompilerTest {
         AF2Offset = o2;
     }
 
-    public static int unsafeSnippet0(int i1, int i2) {
+    public static int unsafeSnippet1(double i1) {
         A a = new A();
-        UNSAFE.putDouble(a, AF1Offset, i1 + i2);
+        UNSAFE.putDouble(a, AF1Offset, i1);
         return UNSAFE.getInt(a, AF1Offset) + UNSAFE.getInt(a, AF2Offset);
     }
 
-    public static int unsafeSnippet1(int i1, int i2) {
+    public static long unsafeSnippet2a(int i1) {
         A a = new A();
-        UNSAFE.putDouble(a, AF1Offset, i1 + i2);
+        UNSAFE.putDouble(a, AF1Offset, i1);
+        a.f1 = i1;
+        return UNSAFE.getLong(a, AF1Offset);
+    }
+
+    public static long unsafeSnippet2b(int i1) {
+        A a = new A();
+        UNSAFE.putDouble(a, AF1Offset, i1);
         a.f2 = i1;
-        return (int) UNSAFE.getDouble(a, AF1Offset);
+        return UNSAFE.getLong(a, AF1Offset);
+    }
+
+    public static long unsafeSnippet3a(int i1) {
+        A a = new A();
+        UNSAFE.putDouble(a, AF1Offset, i1);
+        UNSAFE.putInt(a, AF1Offset, i1);
+        return UNSAFE.getLong(a, AF1Offset);
+    }
+
+    public static long unsafeSnippet3b(int i1) {
+        A a = new A();
+        UNSAFE.putDouble(a, AF1Offset, i1);
+        UNSAFE.putInt(a, AF2Offset, i1);
+        return UNSAFE.getLong(a, AF1Offset);
+    }
+
+    public static int unsafeSnippet4(double i1) {
+        A a = new A();
+        UNSAFE.putDouble(a, AF1Offset, i1);
+        UNSAFE.putDouble(a, AF1Offset, i1);
+        return UNSAFE.getInt(a, AF1Offset) + UNSAFE.getInt(a, AF2Offset);
     }
 
     @Test
     public void testUnsafePEA01() {
-        testPartialEscapeReadElimination(parseEager("unsafeSnippet0", AllowAssumptions.NO), false);
-        testPartialEscapeReadElimination(parseEager("unsafeSnippet0", AllowAssumptions.NO), true);
+        testPartialEscapeReadElimination("unsafeSnippet1", false, 1.0);
+        testPartialEscapeReadElimination("unsafeSnippet1", true, 1.0);
     }
 
     @Test
     public void testUnsafePEA02() {
-        testPartialEscapeReadElimination(parseEager("unsafeSnippet1", AllowAssumptions.NO), false);
-        testPartialEscapeReadElimination(parseEager("unsafeSnippet1", AllowAssumptions.NO), true);
+        testPartialEscapeReadElimination("unsafeSnippet2a", false, 1);
+        testPartialEscapeReadElimination("unsafeSnippet2a", true, 1);
+
+        testPartialEscapeReadElimination("unsafeSnippet2b", false, 1);
+        testPartialEscapeReadElimination("unsafeSnippet2b", true, 1);
     }
 
-    public void testPartialEscapeReadElimination(StructuredGraph graph, boolean canonicalizeBefore) {
+    @Test
+    public void testUnsafePEA03() {
+        testPartialEscapeReadElimination("unsafeSnippet3a", false, 1);
+        testPartialEscapeReadElimination("unsafeSnippet3a", true, 1);
+
+        testPartialEscapeReadElimination("unsafeSnippet3b", false, 1);
+        testPartialEscapeReadElimination("unsafeSnippet3b", true, 1);
+    }
+
+    @Test
+    public void testUnsafePEA04() {
+        testPartialEscapeReadElimination("unsafeSnippet4", false, 1.0);
+        testPartialEscapeReadElimination("unsafeSnippet4", true, 1.0);
+    }
+
+    public void testPartialEscapeReadElimination(String snippet, boolean canonicalizeBefore, Object... args) {
+        assert AF1Offset % 8 == 0 : "First of the two int-fields must be 8-byte aligned";
+
+        ResolvedJavaMethod method = getResolvedJavaMethod(snippet);
+        StructuredGraph graph = parseEager(snippet, AllowAssumptions.NO);
         OptionValues options = graph.getOptions();
         PhaseContext context = getDefaultHighTierContext();
         CanonicalizerPhase canonicalizer = new CanonicalizerPhase();
         if (canonicalizeBefore) {
             canonicalizer.apply(graph, context);
         }
+        Result r = executeExpected(method, null, args);
         new PartialEscapePhase(true, true, canonicalizer, null, options).apply(graph, context);
+        try {
+            InstalledCode code = getCode(method, graph);
+            Object result = code.executeVarargs(args);
+            assertEquals(r, new Result(result, null));
+        } catch (Throwable e) {
+            assertFalse(true, e.toString());
+        }
     }
-
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ea/EATestBase.java

@@ -77,7 +77,7 @@ public class EATestBase extends GraalCompilerTest {
 
         @Override
         public String toString() {
-            return "{" + x + "," + y + "}";
+            return "{" + x + "," + y + "," + z + "}";
         }
 
         @Override
@@ -158,11 +158,19 @@ public class EATestBase extends GraalCompilerTest {
             context = getDefaultHighTierContext();
             new InliningPhase(new CanonicalizerPhase()).apply(graph, context);
             new DeadCodeEliminationPhase().apply(graph);
-            new CanonicalizerPhase().apply(graph, context);
+            canonicalizeGraph();
             new PartialEscapePhase(iterativeEscapeAnalysis, false, new CanonicalizerPhase(), null, graph.getOptions()).apply(graph, context);
+            postEACanonicalizeGraph();
             returnNodes = graph.getNodes(ReturnNode.TYPE).snapshot();
         } catch (Throwable e) {
             throw debug.handle(e);
         }
     }
+
+    protected void postEACanonicalizeGraph() {
+    }
+
+    protected void canonicalizeGraph() {
+        new CanonicalizerPhase().apply(graph, context);
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ea/PEAAssertionsTest.java

@@ -27,9 +27,20 @@ import org.junit.Test;
 import org.graalvm.compiler.api.directives.GraalDirectives;
 import org.graalvm.compiler.code.SourceStackTraceBailoutException;
 import org.graalvm.compiler.core.test.GraalCompilerTest;
+import org.graalvm.compiler.phases.OptimisticOptimizations;
+import org.graalvm.compiler.phases.OptimisticOptimizations.Optimization;
+import org.graalvm.compiler.phases.tiers.HighTierContext;
 
 public class PEAAssertionsTest extends GraalCompilerTest {
 
+    /**
+     * These tests assume all code paths are reachable so disable profile based dead code removal.
+     */
+    @Override
+    protected HighTierContext getDefaultHighTierContext() {
+        return new HighTierContext(getProviders(), getDefaultGraphBuilderSuite(), OptimisticOptimizations.ALL.remove(Optimization.RemoveNeverExecutedCode));
+    }
+
     public static Object field;
 
     public static void snippet1(int i) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ea/TrufflePEATest.java

@@ -22,6 +22,8 @@
  */
 package org.graalvm.compiler.core.test.ea;
 
+import java.lang.reflect.Field;
+
 import org.graalvm.compiler.core.common.GraalOptions;
 import org.graalvm.compiler.core.test.GraalCompilerTest;
 import org.graalvm.compiler.nodes.StructuredGraph;
@@ -33,9 +35,8 @@ import org.graalvm.compiler.phases.common.inlining.InliningPhase;
 import org.graalvm.compiler.phases.tiers.HighTierContext;
 import org.graalvm.compiler.virtual.phases.ea.PartialEscapePhase;
 import org.junit.Test;
-import sun.misc.Unsafe;
 
-import java.lang.reflect.Field;
+import sun.misc.Unsafe;
 
 public class TrufflePEATest extends GraalCompilerTest {
 
@@ -56,6 +57,7 @@ public class TrufflePEATest extends GraalCompilerTest {
     static class DynamicObject {
         int primitiveField0;
         int primitiveField1;
+        int primitiveField2;
     }
 
     private static final long offsetLong1 = Unsafe.ARRAY_LONG_BASE_OFFSET + Unsafe.ARRAY_LONG_INDEX_SCALE * 1;
@@ -66,7 +68,15 @@ public class TrufflePEATest extends GraalCompilerTest {
     static {
         try {
             Field primitiveField0 = DynamicObject.class.getDeclaredField("primitiveField0");
-            primitiveField0Offset = UNSAFE.objectFieldOffset(primitiveField0);
+            long offset = UNSAFE.objectFieldOffset(primitiveField0);
+            if (offset % 8 == 0) {
+                primitiveField0Offset = offset;
+            } else {
+                Field primitiveField1 = DynamicObject.class.getDeclaredField("primitiveField1");
+                offset = UNSAFE.objectFieldOffset(primitiveField1);
+                assert offset % 8 == 0;
+                primitiveField0Offset = offset;
+            }
         } catch (NoSuchFieldException | SecurityException e) {
             throw new AssertionError(e);
         }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ea/UnsafeEATest.java

@@ -22,14 +22,26 @@
  */
 package org.graalvm.compiler.core.test.ea;
 
-import jdk.vm.ci.meta.JavaConstant;
+import java.nio.ByteBuffer;
 
+import org.graalvm.compiler.api.directives.GraalDirectives;
+import org.graalvm.compiler.graph.Graph;
+import org.graalvm.compiler.graph.Node;
+import org.graalvm.compiler.nodes.NamedLocationIdentity;
+import org.graalvm.compiler.nodes.PhiNode;
+import org.graalvm.compiler.nodes.ValuePhiNode;
+import org.graalvm.compiler.nodes.calc.UnpackEndianHalfNode;
+import org.graalvm.compiler.nodes.extended.RawLoadNode;
+import org.graalvm.compiler.nodes.extended.RawStoreNode;
+import org.graalvm.compiler.nodes.extended.UnsafeAccessNode;
+import org.graalvm.compiler.nodes.java.LoadFieldNode;
+import org.graalvm.compiler.phases.common.CanonicalizerPhase;
 import org.junit.Assert;
 import org.junit.Test;
 
-import org.graalvm.compiler.nodes.PhiNode;
-import org.graalvm.compiler.nodes.ValuePhiNode;
-import org.graalvm.compiler.nodes.java.LoadFieldNode;
+import jdk.vm.ci.meta.JavaConstant;
+import jdk.vm.ci.meta.JavaKind;
+import jdk.vm.ci.meta.ResolvedJavaMethod;
 
 public class UnsafeEATest extends EATestBase {
 
@@ -56,6 +68,64 @@ public class UnsafeEATest extends EATestBase {
         }
     }
 
+    @Override
+    protected void testEscapeAnalysis(String snippet, JavaConstant expectedConstantResult, boolean iterativeEscapeAnalysis) {
+        // Exercise both a graph containing UnsafeAccessNodes and one which has been possibly been
+        // canonicalized into AccessFieldNodes.
+        testingUnsafe = true;
+        super.testEscapeAnalysis(snippet, expectedConstantResult, iterativeEscapeAnalysis);
+        testingUnsafe = false;
+        super.testEscapeAnalysis(snippet, expectedConstantResult, iterativeEscapeAnalysis);
+        if (expectedConstantResult != null) {
+            // Check that a compiled version of this method returns the same value if we expect a
+            // constant result.
+            ResolvedJavaMethod method = getResolvedJavaMethod(snippet);
+            JavaKind[] javaKinds = method.getSignature().toParameterKinds(false);
+            Object[] args = new Object[javaKinds.length];
+            int i = 0;
+            for (JavaKind k : javaKinds) {
+                args[i++] = JavaConstant.defaultForKind(k).asBoxedPrimitive();
+            }
+            Result result = executeExpected(method, null, args);
+            assertTrue(result.returnValue.equals(expectedConstantResult.asBoxedPrimitive()));
+        }
+    }
+
+    @Override
+    protected void canonicalizeGraph() {
+        if (testingUnsafe) {
+            // For testing purposes we'd like to ensure that our raw unsafe operations stay as
+            // unsafe nodes, so force them to appear to have LocationIdentity.any to disable
+            // transformation into field access nodes.
+            for (Node node : graph.getNodes().filter(x -> x instanceof UnsafeAccessNode).snapshot()) {
+                if (node instanceof RawStoreNode) {
+                    RawStoreNode store = (RawStoreNode) node;
+                    RawStoreNode newStore = graph.add(new RawStoreNode(store.object(), store.offset(), store.value(), store.accessKind(), NamedLocationIdentity.any(),
+                                    store.needsBarrier(), store.stateAfter(), true));
+                    graph.replaceFixedWithFixed(store, newStore);
+                } else if (node instanceof RawLoadNode) {
+                    RawLoadNode load = (RawLoadNode) node;
+                    RawLoadNode newLoad = graph.add(new RawLoadNode(load.object(), load.offset(), load.accessKind(), NamedLocationIdentity.any(),
+                                    true));
+                    graph.replaceFixedWithFixed(load, newLoad);
+                }
+            }
+        }
+        super.canonicalizeGraph();
+    }
+
+    @Override
+    protected void postEACanonicalizeGraph() {
+        // Simplify any UnpackEndianHalfNode so we end up with constants.
+        Graph.Mark mark = graph.getMark();
+        for (UnpackEndianHalfNode node : graph.getNodes().filter(UnpackEndianHalfNode.class)) {
+            node.lower(getTarget().arch.getByteOrder());
+        }
+        new CanonicalizerPhase().applyIncremental(graph, context, mark);
+    }
+
+    private boolean testingUnsafe;
+
     @Test
     public void testSimpleInt() {
         testEscapeAnalysis("testSimpleIntSnippet", JavaConstant.forInt(101), false);
@@ -89,6 +159,82 @@ public class UnsafeEATest extends EATestBase {
         return UNSAFE.getDouble(x, fieldOffset1);
     }
 
+    @Test
+    public void testSimpleDoubleOverwriteWithInt() {
+        testEscapeAnalysis("testSimpleDoubleOverwriteWithIntSnippet", JavaConstant.forInt(10), false);
+    }
+
+    public static int testSimpleDoubleOverwriteWithIntSnippet() {
+        TestClassInt x = new TestClassInt();
+        UNSAFE.putDouble(x, fieldOffset1, 10.1);
+        UNSAFE.putInt(x, fieldOffset1, 10);
+        return UNSAFE.getInt(x, fieldOffset1);
+    }
+
+    @Test
+    public void testSimpleDoubleOverwriteWithSecondInt() {
+        ByteBuffer bb = ByteBuffer.allocate(8).order(getTarget().arch.getByteOrder());
+        bb.putDouble(10.1);
+        int value = bb.getInt(4);
+
+        testEscapeAnalysis("testSimpleDoubleOverwriteWithSecondIntSnippet", JavaConstant.forInt(value), false);
+    }
+
+    public static int testSimpleDoubleOverwriteWithSecondIntSnippet() {
+        TestClassInt x = new TestClassInt();
+        UNSAFE.putDouble(x, fieldOffset1, 10.1);
+        UNSAFE.putInt(x, fieldOffset1, 10);
+        return UNSAFE.getInt(x, fieldOffset2);
+    }
+
+    @Test
+    public void testSimpleDoubleOverwriteWithFirstInt() {
+        ByteBuffer bb = ByteBuffer.allocate(8).order(getTarget().arch.getByteOrder());
+        bb.putDouble(10.1);
+        int value = bb.getInt(0);
+
+        testEscapeAnalysis("testSimpleDoubleOverwriteWithFirstIntSnippet", JavaConstant.forInt(value), false);
+    }
+
+    public static int testSimpleDoubleOverwriteWithFirstIntSnippet() {
+        TestClassInt x = new TestClassInt();
+        UNSAFE.putDouble(x, fieldOffset1, 10.1);
+        UNSAFE.putInt(x, fieldOffset2, 10);
+        return UNSAFE.getInt(x, fieldOffset1);
+    }
+
+    @Test
+    public void testSimpleLongOverwriteWithSecondInt() {
+        ByteBuffer bb = ByteBuffer.allocate(8).order(getTarget().arch.getByteOrder());
+        bb.putLong(0, 0x1122334455667788L);
+        int value = bb.getInt(4);
+
+        testEscapeAnalysis("testSimpleLongOverwriteWithSecondIntSnippet", JavaConstant.forInt(value), false);
+    }
+
+    public static int testSimpleLongOverwriteWithSecondIntSnippet() {
+        TestClassInt x = new TestClassInt();
+        UNSAFE.putLong(x, fieldOffset1, 0x1122334455667788L);
+        UNSAFE.putInt(x, fieldOffset1, 10);
+        return UNSAFE.getInt(x, fieldOffset2);
+    }
+
+    @Test
+    public void testSimpleLongOverwriteWithFirstInt() {
+        ByteBuffer bb = ByteBuffer.allocate(8).order(getTarget().arch.getByteOrder());
+        bb.putLong(0, 0x1122334455667788L);
+        int value = bb.getInt(0);
+
+        testEscapeAnalysis("testSimpleLongOverwriteWithFirstIntSnippet", JavaConstant.forInt(value), false);
+    }
+
+    public static int testSimpleLongOverwriteWithFirstIntSnippet() {
+        TestClassInt x = new TestClassInt();
+        UNSAFE.putLong(x, fieldOffset1, 0x1122334455667788L);
+        UNSAFE.putInt(x, fieldOffset2, 10);
+        return UNSAFE.getInt(x, fieldOffset1);
+    }
+
     @Test
     public void testMergedDouble() {
         testEscapeAnalysis("testMergedDoubleSnippet", null, false);
@@ -111,6 +257,32 @@ public class UnsafeEATest extends EATestBase {
         return UNSAFE.getDouble(x, fieldOffset1);
     }
 
+    static class ExtendedTestClassInt extends TestClassInt {
+        public long l;
+    }
+
+    @Test
+    public void testMergedVirtualObjects() {
+        testEscapeAnalysis("testMergedVirtualObjectsSnippet", null, false);
+    }
+
+    public static TestClassInt testMergedVirtualObjectsSnippet(int value) {
+        TestClassInt x;
+        if (value == 1) {
+            x = new TestClassInt();
+            UNSAFE.putDouble(x, fieldOffset1, 10);
+        } else {
+            x = new TestClassInt();
+            UNSAFE.putInt(x, fieldOffset1, 0);
+        }
+        UNSAFE.putInt(x, fieldOffset1, 0);
+        if (value == 2) {
+            UNSAFE.putInt(x, fieldOffset2, 0);
+        }
+        GraalDirectives.deoptimizeAndInvalidate();
+        return x;
+    }
+
     @Test
     public void testMaterializedDouble() {
         test("testMaterializedDoubleSnippet");

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/inlining/NestedLoopEffectsPhaseComplexityTest.java

@@ -146,7 +146,7 @@ public class NestedLoopEffectsPhaseComplexityTest extends GraalCompilerTest {
 
     private StructuredGraph parseBytecodes(ResolvedJavaMethod method, HighTierContext context, CanonicalizerPhase canonicalizer) {
         OptionValues options = getInitialOptions();
-        StructuredGraph newGraph = new StructuredGraph.Builder(options, getDebugContext(options), AllowAssumptions.NO).method(method).build();
+        StructuredGraph newGraph = new StructuredGraph.Builder(options, getDebugContext(options, null, method), AllowAssumptions.NO).method(method).build();
         context.getGraphBuilderSuite().apply(newGraph, context);
         new DeadCodeEliminationPhase(Optional).apply(newGraph);
         canonicalizer.apply(newGraph, context);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/GraalCompilerOptions.java

@@ -47,7 +47,7 @@ public class GraalCompilerOptions {
      public static final EnumOptionKey<ExceptionAction> CompilationFailureAction = new EnumOptionKey<>(ExceptionAction.Diagnose);
     @Option(help = "The maximum number of compilation failures or bailouts to handle with the action specified " +
                    "by CompilationFailureAction or CompilationBailoutAction before changing to a less verbose action.", type = OptionType.User)
-    public static final OptionKey<Integer> MaxCompilationProblemsPerAction = new OptionKey<>(5);
+    public static final OptionKey<Integer> MaxCompilationProblemsPerAction = new OptionKey<>(2);
     @Option(help = "Alias for CompilationFailureAction=ExitVM.", type = OptionType.User)
     public static final OptionKey<Boolean> ExitVMOnException = new OptionKey<>(false);
     // @formatter:on

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/DebugInfoBuilder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -132,8 +132,17 @@ public class DebugInfoBuilder {
                             slotKinds[pos] = toSlotKind(value);
                             pos++;
                         } else {
-                            assert currentField.values().get(i - 1).getStackKind() == JavaKind.Double || currentField.values().get(i - 1).getStackKind() == JavaKind.Long : vobjNode + " " + i + " " +
-                                            currentField.values().get(i - 1);
+                            assert value.getStackKind() == JavaKind.Illegal;
+                            ValueNode previousValue = currentField.values().get(i - 1);
+                            assert (previousValue != null && previousValue.getStackKind().needsTwoSlots()) : vobjNode + " " + i +
+                                            " " + previousValue + " " + currentField.values().snapshot();
+                            if (previousValue == null || !previousValue.getStackKind().needsTwoSlots()) {
+                                // Don't allow the IllegalConstant to leak into the debug info
+                                JavaKind entryKind = vobjNode.entryKind(i);
+                                values[pos] = JavaConstant.defaultForKind(entryKind.getStackKind());
+                                slotKinds[pos] = entryKind.getStackKind();
+                                pos++;
+                            }
                         }
                     }
                     if (pos != entryCount) {
@@ -164,21 +173,21 @@ public class DebugInfoBuilder {
             if (!type.isArray()) {
                 ResolvedJavaField[] fields = type.getInstanceFields(true);
                 int fieldIndex = 0;
-                for (int i = 0; i < values.length; i++) {
-                    ResolvedJavaField field = fields[fieldIndex++];
-                    JavaKind valKind = slotKinds[i].getStackKind();
+                for (int valueIndex = 0; valueIndex < values.length; valueIndex++, fieldIndex++) {
+                    ResolvedJavaField field = fields[fieldIndex];
+                    JavaKind valKind = slotKinds[valueIndex].getStackKind();
                     JavaKind fieldKind = storageKind(field.getType());
-                    if (fieldKind == JavaKind.Object) {
-                        assert valKind.isObject() : field + ": " + valKind + " != " + fieldKind;
-                    } else {
                     if ((valKind == JavaKind.Double || valKind == JavaKind.Long) && fieldKind == JavaKind.Int) {
-                            assert storageKind(fields[fieldIndex].getType()) == JavaKind.Int;
+                        assert fieldIndex + 1 < fields.length : String.format("Not enough fields for fieldIndex = %d valueIndex = %d %s %s", fieldIndex, valueIndex, Arrays.toString(fields),
+                                        Arrays.toString(values));
+                        assert storageKind(fields[fieldIndex + 1].getType()) == JavaKind.Int : String.format("fieldIndex = %d valueIndex = %d %s %s %s", fieldIndex, valueIndex,
+                                        storageKind(fields[fieldIndex + 1].getType()), Arrays.toString(fields),
+                                        Arrays.toString(values));
                         fieldIndex++;
                     } else {
                         assert valKind == fieldKind.getStackKind() : field + ": " + valKind + " != " + fieldKind;
                     }
                 }
-                }
                 assert fields.length == fieldIndex : type + ": fields=" + Arrays.toString(fields) + ", field values=" + Arrays.toString(values);
             } else {
                 JavaKind componentKind = storageKind(type.getComponentType()).getStackKind();

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/NodeLIRBuilder.java

@@ -59,6 +59,7 @@ import org.graalvm.compiler.lir.StandardOp.LabelOp;
 import org.graalvm.compiler.lir.SwitchStrategy;
 import org.graalvm.compiler.lir.Variable;
 import org.graalvm.compiler.lir.debug.LIRGenerationDebugContext;
+import org.graalvm.compiler.lir.framemap.FrameMapBuilder;
 import org.graalvm.compiler.lir.gen.LIRGenerator;
 import org.graalvm.compiler.lir.gen.LIRGenerator.Options;
 import org.graalvm.compiler.lir.gen.LIRGeneratorTool;
@@ -577,9 +578,9 @@ public abstract class NodeLIRBuilder implements NodeLIRBuilderTool, LIRGeneratio
     @Override
     public void emitInvoke(Invoke x) {
         LoweredCallTargetNode callTarget = (LoweredCallTargetNode) x.callTarget();
-        CallingConvention invokeCc = gen.getResult().getFrameMapBuilder().getRegisterConfig().getCallingConvention(callTarget.callType(), x.asNode().stamp().javaType(gen.getMetaAccess()),
-                        callTarget.signature(), gen);
-        gen.getResult().getFrameMapBuilder().callsMethod(invokeCc);
+        FrameMapBuilder frameMapBuilder = gen.getResult().getFrameMapBuilder();
+        CallingConvention invokeCc = frameMapBuilder.getRegisterConfig().getCallingConvention(callTarget.callType(), x.asNode().stamp().javaType(gen.getMetaAccess()), callTarget.signature(), gen);
+        frameMapBuilder.callsMethod(invokeCc);
 
         Value[] parameters = visitInvokeArguments(invokeCc, callTarget.arguments());
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/target/Backend.java

@@ -195,10 +195,13 @@ public abstract class Backend implements TargetProvider, ValueKindFactory<LIRKin
     public InstalledCode createInstalledCode(DebugContext debug, ResolvedJavaMethod method, CompilationRequest compilationRequest, CompilationResult compilationResult,
                     SpeculationLog speculationLog, InstalledCode predefinedInstalledCode, boolean isDefault, Object[] context) {
         Object[] debugContext = context != null ? context : new Object[]{getProviders().getCodeCache(), method, compilationResult};
-        CodeInstallationTask[] tasks = new CodeInstallationTask[codeInstallationTaskFactories.size()];
+        CodeInstallationTask[] tasks;
+        synchronized (this) {
+            tasks = new CodeInstallationTask[codeInstallationTaskFactories.size()];
             for (int i = 0; i < codeInstallationTaskFactories.size(); i++) {
                 tasks[i] = codeInstallationTaskFactories.get(i).create();
             }
+        }
         try (DebugContext.Scope s2 = debug.scope("CodeInstall", debugContext);
                         DebugContext.Activation a = debug.activate()) {
             for (CodeInstallationTask task : tasks) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/DebugContext.java

@@ -29,9 +29,11 @@ import static org.graalvm.compiler.debug.DebugOptions.Counters;
 import static org.graalvm.compiler.debug.DebugOptions.Dump;
 import static org.graalvm.compiler.debug.DebugOptions.DumpOnError;
 import static org.graalvm.compiler.debug.DebugOptions.DumpOnPhaseChange;
+import static org.graalvm.compiler.debug.DebugOptions.DumpPath;
 import static org.graalvm.compiler.debug.DebugOptions.ListMetrics;
 import static org.graalvm.compiler.debug.DebugOptions.Log;
 import static org.graalvm.compiler.debug.DebugOptions.MemUseTrackers;
+import static org.graalvm.compiler.debug.DebugOptions.ShowDumpFiles;
 import static org.graalvm.compiler.debug.DebugOptions.Time;
 import static org.graalvm.compiler.debug.DebugOptions.Timers;
 import static org.graalvm.compiler.debug.DebugOptions.TrackMemUse;
@@ -56,6 +58,7 @@ import java.util.TreeMap;
 
 import org.graalvm.compiler.options.OptionKey;
 import org.graalvm.compiler.options.OptionValues;
+import org.graalvm.graphio.GraphOutput;
 import org.graalvm.util.EconomicMap;
 import org.graalvm.util.EconomicSet;
 import org.graalvm.util.Pair;
@@ -98,6 +101,8 @@ public final class DebugContext implements AutoCloseable {
     CloseableCounter currentMemUseTracker;
     Scope lastClosedScope;
     Throwable lastExceptionThrown;
+    private IgvDumpChannel sharedChannel;
+    private GraphOutput<?, ?> parentOutput;
 
     /**
      * Stores the {@link MetricKey} values.
@@ -111,6 +116,19 @@ public final class DebugContext implements AutoCloseable {
         return immutable.scopesEnabled;
     }
 
+    public <G, N, M> GraphOutput<G, M> buildOutput(GraphOutput.Builder<G, N, M> builder) throws IOException {
+        if (parentOutput != null) {
+            return builder.build(parentOutput);
+        } else {
+            if (sharedChannel == null) {
+                sharedChannel = new IgvDumpChannel(() -> getDumpPath(".bgv", false), immutable.options);
+            }
+            final GraphOutput<G, M> output = builder.build(sharedChannel);
+            parentOutput = output;
+            return output;
+        }
+    }
+
     /**
      * The immutable configuration that can be shared between {@link DebugContext} objects.
      */
@@ -323,6 +341,14 @@ public final class DebugContext implements AutoCloseable {
             String compilableName = compilable instanceof JavaMethod ? ((JavaMethod) compilable).format("%H.%n(%p)%R") : String.valueOf(compilable);
             return identifier + ":" + compilableName;
         }
+
+        final String getLabel() {
+            if (compilable instanceof JavaMethod) {
+                JavaMethod method = (JavaMethod) compilable;
+                return method.format("%h.%n(%p)%r");
+            }
+            return String.valueOf(compilable);
+        }
     }
 
     private final Description description;
@@ -394,6 +420,20 @@ public final class DebugContext implements AutoCloseable {
         }
     }
 
+    public Path getDumpPath(String extension, boolean directory) {
+        try {
+            String id = description == null ? null : description.identifier;
+            String label = description == null ? null : description.getLabel();
+            Path result = PathUtilities.createUnique(immutable.options, DumpPath, id, label, extension, directory);
+            if (ShowDumpFiles.getValue(immutable.options)) {
+                TTY.println("Dumping debug output to %s", result.toAbsolutePath().toString());
+            }
+            return result;
+        } catch (IOException ex) {
+            throw rethrowSilently(RuntimeException.class, ex);
+        }
+    }
+
     /**
      * A special dump level that indicates the dumping machinery is enabled but no dumps will be
      * produced except through other options.
@@ -2043,4 +2083,9 @@ public final class DebugContext implements AutoCloseable {
         }
         out.println();
     }
+
+    @SuppressWarnings({"unused", "unchecked"})
+    private static <E extends Exception> E rethrowSilently(Class<E> type, Throwable ex) throws E {
+        throw (E) ex;
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/DebugHandlersFactory.java

@@ -35,6 +35,9 @@ public interface DebugHandlersFactory {
 
     /**
      * Creates {@link DebugHandler}s based on {@code options}.
+     *
+     * @param options options to control type and name of the channel
+     * @return list of debug handers that have been created
      */
     List<DebugHandler> createHandlers(OptionValues options);
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/DebugOptions.java

@@ -128,8 +128,6 @@ public class DebugOptions {
     public static final OptionKey<Boolean> PrintGraphProbabilities = new OptionKey<>(false);
     @Option(help = "Enable dumping to the IdealGraphVisualizer.", type = OptionType.Debug)
     public static final OptionKey<Boolean> PrintGraph = new OptionKey<>(true);
-    @Option(help = "Dump graphs in binary format instead of XML format.", type = OptionType.Debug)
-    public static final OptionKey<Boolean> PrintBinaryGraphs = new OptionKey<>(true);
     @Option(help = "Print graphs to files instead of sending them over the network.", type = OptionType.Debug)
     public static final OptionKey<Boolean> PrintGraphFile = new OptionKey<>(false);
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/IgvDumpChannel.java

@@ -0,0 +1,119 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.debug;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedByInterruptException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.SocketChannel;
+import java.nio.channels.WritableByteChannel;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.function.Supplier;
+import static org.graalvm.compiler.debug.DebugOptions.PrintBinaryGraphPort;
+import static org.graalvm.compiler.debug.DebugOptions.PrintGraphHost;
+import org.graalvm.compiler.options.OptionValues;
+
+final class IgvDumpChannel implements WritableByteChannel {
+    private final Supplier<Path> pathProvider;
+    private final OptionValues options;
+    private WritableByteChannel sharedChannel;
+    private boolean closed;
+
+    IgvDumpChannel(Supplier<Path> pathProvider, OptionValues options) {
+        this.pathProvider = pathProvider;
+        this.options = options;
+    }
+
+    @Override
+    public int write(ByteBuffer src) throws IOException {
+        return channel().write(src);
+    }
+
+    @Override
+    public boolean isOpen() {
+        return !closed;
+    }
+
+    @Override
+    public void close() throws IOException {
+    }
+
+    void realClose() throws IOException {
+        closed = true;
+        if (sharedChannel != null) {
+            sharedChannel.close();
+            sharedChannel = null;
+        }
+    }
+
+    WritableByteChannel channel() throws IOException {
+        if (closed) {
+            throw new IOException();
+        }
+        if (sharedChannel == null) {
+            if (DebugOptions.PrintGraphFile.getValue(options)) {
+                sharedChannel = createFileChannel(pathProvider);
+            } else {
+                sharedChannel = createNetworkChannel(pathProvider, options);
+            }
+        }
+        return sharedChannel;
+    }
+
+    private static WritableByteChannel createNetworkChannel(Supplier<Path> pathProvider, OptionValues options) throws IOException {
+        String host = PrintGraphHost.getValue(options);
+        int port = PrintBinaryGraphPort.getValue(options);
+        try {
+            WritableByteChannel channel = SocketChannel.open(new InetSocketAddress(host, port));
+            TTY.println("Connected to the IGV on %s:%d", host, port);
+            return channel;
+        } catch (ClosedByInterruptException | InterruptedIOException e) {
+            /*
+             * Interrupts should not count as errors because they may be caused by a cancelled Graal
+             * compilation. ClosedByInterruptException occurs if the SocketChannel could not be
+             * opened. InterruptedIOException occurs if new Socket(..) was interrupted.
+             */
+            return null;
+        } catch (IOException e) {
+            if (!DebugOptions.PrintGraphFile.hasBeenSet(options)) {
+                return createFileChannel(pathProvider);
+            } else {
+                throw new IOException(String.format("Could not connect to the IGV on %s:%d", host, port), e);
+            }
+        }
+    }
+
+    private static WritableByteChannel createFileChannel(Supplier<Path> pathProvider) throws IOException {
+        Path path = pathProvider.get();
+        try {
+            return FileChannel.open(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE);
+        } catch (IOException e) {
+            throw new IOException(String.format("Failed to open %s to dump IGV graphs", path), e);
+        }
+    }
+
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/PathUtilities.java

@@ -22,12 +22,13 @@
  */
 package org.graalvm.compiler.debug;
 
+import java.io.File;
 import java.io.IOException;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
 import java.nio.file.InvalidPathException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.HashMap;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.graalvm.compiler.options.OptionKey;
@@ -39,54 +40,6 @@ import org.graalvm.compiler.options.OptionValues;
 public class PathUtilities {
 
     private static final AtomicLong globalTimeStamp = new AtomicLong();
-    /**
-     * This generates a per thread persistent id to aid mapping related dump files with each other.
-     */
-    private static final ThreadLocal<PerThreadSequence> threadDumpId = new ThreadLocal<>();
-    private static final AtomicInteger dumpId = new AtomicInteger();
-
-    static class PerThreadSequence {
-        final int threadID;
-        HashMap<String, Integer> sequences = new HashMap<>(2);
-
-        PerThreadSequence(int threadID) {
-            this.threadID = threadID;
-        }
-
-        String generateID(String extension) {
-            Integer box = sequences.get(extension);
-            if (box == null) {
-                sequences.put(extension, 1);
-                return Integer.toString(threadID);
-            } else {
-                sequences.put(extension, box + 1);
-                return Integer.toString(threadID) + '-' + box;
-            }
-        }
-    }
-
-    private static String getThreadDumpId(String extension) {
-        PerThreadSequence id = threadDumpId.get();
-        if (id == null) {
-            id = new PerThreadSequence(dumpId.incrementAndGet());
-            threadDumpId.set(id);
-        }
-        return id.generateID(extension);
-    }
-
-    /**
-     * Prepends a period (i.e., {@code '.'}) to an non-null, non-empty string representation a file
-     * extension if the string does not already start with a period.
-     *
-     * @return {@code ext} unmodified if it is null, empty or already starts with a period other
-     *         {@code "." + ext}
-     */
-    public static String formatExtension(String ext) {
-        if (ext == null || ext.length() == 0) {
-            return "";
-        }
-        return "." + ext;
-    }
 
     /**
      * Gets a time stamp for the current process. This method will always return the same value for
@@ -99,43 +52,6 @@ public class PathUtilities {
         return globalTimeStamp.get();
     }
 
-    /**
-     * Generates a {@link Path} using the format "%s-%d_%d%s" with the {@code baseNameOption}, a
-     * {@link #getGlobalTimeStamp() global timestamp} , {@link #getThreadDumpId a per thread unique
-     * id} and an optional {@code extension}.
-     *
-     * @return the output file path or null if the flag is null
-     */
-    public static Path getPath(OptionValues options, OptionKey<String> baseNameOption, String extension) throws IOException {
-        return getPath(options, baseNameOption, extension, true);
-    }
-
-    /**
-     * Generate a {@link Path} using the format "%s-%d_%s" with the {@code baseNameOption}, a
-     * {@link #getGlobalTimeStamp() global timestamp} and an optional {@code extension} .
-     *
-     * @return the output file path or null if the flag is null
-     */
-    public static Path getPathGlobal(OptionValues options, OptionKey<String> baseNameOption, String extension) throws IOException {
-        return getPath(options, baseNameOption, extension, false);
-    }
-
-    private static Path getPath(OptionValues options, OptionKey<String> baseNameOption, String extension, boolean includeThreadId) throws IOException {
-        if (baseNameOption.getValue(options) == null) {
-            return null;
-        }
-        String ext = formatExtension(extension);
-        final String name = includeThreadId
-                        ? String.format("%s-%d_%s%s", baseNameOption.getValue(options), getGlobalTimeStamp(), getThreadDumpId(ext), ext)
-                        : String.format("%s-%d%s", baseNameOption.getValue(options), getGlobalTimeStamp(), ext);
-        Path result = Paths.get(name);
-        if (result.isAbsolute()) {
-            return result;
-        }
-        Path dumpDir = DebugOptions.getDumpDirectory(options);
-        return dumpDir.resolve(name).normalize();
-    }
-
     /**
      * Gets a value based on {@code name} that can be passed to {@link Paths#get(String, String...)}
      * without causing an {@link InvalidPathException}.
@@ -145,21 +61,80 @@ public class PathUtilities {
      */
     public static String sanitizeFileName(String name) {
         try {
-            Paths.get(name);
+            Path path = Paths.get(name);
+            if (path.getNameCount() == 0) {
                 return name;
+            }
         } catch (InvalidPathException e) {
             // fall through
         }
         StringBuilder buf = new StringBuilder(name.length());
         for (int i = 0; i < name.length(); i++) {
             char c = name.charAt(i);
+            if (c != File.separatorChar && c != ' ' && !Character.isISOControl(c)) {
                 try {
                     Paths.get(String.valueOf(c));
-            } catch (InvalidPathException e) {
-                buf.append('_');
-            }
                     buf.append(c);
+                    continue;
+                } catch (InvalidPathException e) {
+                }
+            }
+            buf.append('_');
         }
         return buf.toString();
     }
+
+    /**
+     * A maximum file name length supported by most file systems. There is no platform independent
+     * way to get this in Java.
+     */
+    private static final int MAX_FILE_NAME_LENGTH = 255;
+
+    private static final String ELLIPSIS = "...";
+
+    static Path createUnique(OptionValues options, OptionKey<String> baseNameOption, String id, String label, String ext, boolean createDirectory) throws IOException {
+        String uniqueTag = "";
+        int dumpCounter = 1;
+        String prefix;
+        if (id == null) {
+            prefix = baseNameOption.getValue(options);
+            int slash = prefix.lastIndexOf(File.separatorChar);
+            prefix = prefix.substring(slash + 1);
+        } else {
+            prefix = id;
+        }
+        for (;;) {
+            int fileNameLengthWithoutLabel = uniqueTag.length() + ext.length() + prefix.length() + "[]".length();
+            int labelLengthLimit = MAX_FILE_NAME_LENGTH - fileNameLengthWithoutLabel;
+            String fileName;
+            if (labelLengthLimit < ELLIPSIS.length()) {
+                // This means `id` is very long
+                String suffix = uniqueTag + ext;
+                int idLengthLimit = Math.min(MAX_FILE_NAME_LENGTH - suffix.length(), prefix.length());
+                fileName = sanitizeFileName(prefix.substring(0, idLengthLimit) + suffix);
+            } else {
+                if (label == null) {
+                    fileName = sanitizeFileName(prefix + uniqueTag + ext);
+                } else {
+                    String adjustedLabel = label;
+                    if (label.length() > labelLengthLimit) {
+                        adjustedLabel = label.substring(0, labelLengthLimit - ELLIPSIS.length()) + ELLIPSIS;
+                    }
+                    fileName = sanitizeFileName(prefix + '[' + adjustedLabel + ']' + uniqueTag + ext);
+                }
+            }
+            Path dumpDir = DebugOptions.getDumpDirectory(options);
+            Path result = Paths.get(dumpDir.toString(), fileName);
+            try {
+                if (createDirectory) {
+                    return Files.createDirectory(result);
+                } else {
+                    return Files.createFile(result);
+                }
+            } catch (FileAlreadyExistsException e) {
+                uniqueTag = "_" + dumpCounter++;
+            }
+        }
+    }
+
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph.test/src/org/graalvm/compiler/graph/test/graphio/GraphSnippetTest.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.graph.test.graphio;
+
+import java.io.File;
+import java.lang.reflect.Method;
+import static org.junit.Assert.assertTrue;
+import org.junit.Assume;
+import org.junit.Test;
+
+public class GraphSnippetTest {
+    @Test
+    public void dumpTheFile() throws Exception {
+        Class<?> snippets = null;
+        try {
+            snippets = Class.forName("org.graalvm.graphio.GraphSnippets");
+        } catch (ClassNotFoundException notFound) {
+            Assume.assumeNoException("The snippets class has to be around", notFound);
+        }
+        Method dump = null;
+        try {
+            dump = snippets.getDeclaredMethod("dump", File.class);
+            dump.setAccessible(true);
+        } catch (RuntimeException ex) {
+            Assume.assumeTrue("Only run the test, if the method is accessible", dump != null && dump.isAccessible());
+        }
+        File diamond = File.createTempFile("diamond", ".bgv");
+        dump.invoke(null, diamond);
+        assertTrue("File .bgv created: " + diamond, diamond.length() > 50);
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph.test/src/org/graalvm/compiler/graph/test/graphio/NodeEncodingTest.java

@@ -0,0 +1,311 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.graph.test.graphio;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import org.graalvm.graphio.GraphOutput;
+import org.graalvm.graphio.GraphStructure;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import org.junit.Before;
+import org.junit.Test;
+
+public final class NodeEncodingTest {
+
+    private ByteArrayOutputStream out;
+
+    @Before
+    public void initOutput() {
+        out = new ByteArrayOutputStream();
+    }
+
+    @Test
+    public void version40TheNodeIsntDumpedWithItsID() throws Exception {
+        runTheNodeIsntDumpedWithItsID(true);
+    }
+
+    @Test
+    public void defaultVersionTheNodeIsntDumpedWithItsID() throws Exception {
+        runTheNodeIsntDumpedWithItsID(false);
+    }
+
+    private void runTheNodeIsntDumpedWithItsID(boolean explicitVersion) throws Exception {
+        WritableByteChannel w = Channels.newChannel(out);
+        MockGraph graph = new MockGraph();
+        MockNodeClass clazz = new MockNodeClass("clazz");
+        MockNode node = new MockNode(clazz, 33); // random value otherwise not found in the stream
+        try (GraphOutput<MockGraph, ?> dump = explicitVersion ? GraphOutput.newBuilder(new MockStructure()).protocolVersion(4, 0).build(w) : GraphOutput.newBuilder(new MockStructure()).build(w)) {
+            dump.beginGroup(graph, "test1", "t1", null, 0, Collections.singletonMap("node", node));
+            dump.endGroup();
+        }
+
+        assertEquals("Node is always requested", 1, node.nodeRequested);
+        assertEquals("Nobody asks for id of a node in version 4.0", 0, node.idTested);
+        assertByte(false, out.toByteArray(), 33);
+        assertEquals("Node class of the node has been requested", 1, node.nodeClassRequested);
+        assertEquals("Node class template name stored", 1, clazz.nameTemplateQueried);
+        assertFalse("No to string ops", node.toStringRequested);
+    }
+
+    @Test
+    public void dumpingNodeInVersion10() throws Exception {
+        runTheNodeIsTreatedAsString(true);
+    }
+
+    private void runTheNodeIsTreatedAsString(boolean explicitVersion) throws Exception {
+        WritableByteChannel w = Channels.newChannel(out);
+        MockGraph graph = new MockGraph();
+        MockNodeClass clazz = new MockNodeClass("clazz");
+        MockNode node = new MockNode(clazz, 33); // random value otherwise not found in the stream
+        try (GraphOutput<MockGraph, ?> dump = explicitVersion ? GraphOutput.newBuilder(new MockStructure()).protocolVersion(1, 0).build(w) : GraphOutput.newBuilder(new MockStructure()).build(w)) {
+            dump.beginGroup(graph, "test1", "t1", null, 0, Collections.singletonMap("node", node));
+            dump.endGroup();
+        }
+
+        assertEquals("Node is always requested", 1, node.nodeRequested);
+        assertEquals("Nobody asks for id of a node in version 1.0", 0, node.idTested);
+        assertByte(false, out.toByteArray(), 33);
+        assertEquals("Node class was needed to find out it is not a NodeClass instance", 1, node.nodeClassRequested);
+        assertEquals("Node class template name wasn't needed however", 0, clazz.nameTemplateQueried);
+        assertTrue("Node sent as a string version 1.0", node.toStringRequested);
+    }
+
+    @Test
+    public void dumpingNodeInVersion15() throws Exception {
+        runTheNodeIsTreatedPoolEntry(true);
+    }
+
+    private void runTheNodeIsTreatedPoolEntry(boolean explicitVersion) throws Exception {
+        WritableByteChannel w = Channels.newChannel(out);
+        MockGraph graph = new MockGraph();
+        MockNodeClass clazz = new MockNodeClass("clazz");
+        MockNode node = new MockNode(clazz, 33); // random value otherwise not found in the stream
+        try (GraphOutput<MockGraph, ?> dump = explicitVersion ? GraphOutput.newBuilder(new MockStructure()).protocolVersion(5, 0).build(w) : GraphOutput.newBuilder(new MockStructure()).build(w)) {
+            dump.beginGroup(graph, "test1", "t1", null, 0, Collections.singletonMap("node", node));
+            dump.endGroup();
+        }
+
+        assertEquals("Node is always requested", 1, node.nodeRequested);
+        assertEquals("Id of our node is requested in version 5.0", 1, node.idTested);
+        assertByte(true, out.toByteArray(), 33);
+        assertTrue("Node class was needed at least once", 1 <= node.nodeClassRequested);
+        assertEquals("Node class template name sent to server", 1, clazz.nameTemplateQueried);
+        assertFalse("Node.toString() isn't needed", node.toStringRequested);
+    }
+
+    @Test
+    public void dumpingNodeTwiceInVersion4() throws Exception {
+        WritableByteChannel w = Channels.newChannel(out);
+        MockGraph graph = new MockGraph();
+        MockNodeClass clazz = new MockNodeClass("clazz");
+        MockNode node = new MockNode(clazz, 33); // random value otherwise not found in the stream
+        try (GraphOutput<MockGraph, ?> dump = GraphOutput.newBuilder(new MockStructure()).protocolVersion(4, 0).build(w)) {
+            Map<String, Object> props = new LinkedHashMap<>();
+            props.put("node1", node);
+            props.put("node2", node);
+            props.put("node3", node);
+            dump.beginGroup(graph, "test1", "t1", null, 0, props);
+            dump.endGroup();
+        }
+
+        assertEquals("Node requested three times", 3, node.nodeRequested);
+        assertEquals("Nobody asks for id of a node in version 4.0", 0, node.idTested);
+        // check there is no encoded string for object #3
+        assertByte(false, out.toByteArray(), 1, 0, 3);
+        assertEquals("Node class of the node has been requested three times", 3, node.nodeClassRequested);
+        assertEquals("Node class template name stored", 1, clazz.nameTemplateQueried);
+        assertFalse("No to string ops", node.toStringRequested);
+    }
+
+    private static void assertByte(boolean shouldBeFound, byte[] arr, int... value) {
+        boolean found = false;
+        int at = 0;
+        for (int i = 0; i < arr.length; i++) {
+            if (arr[i] == value[at]) {
+                if (++at == value.length) {
+                    found = true;
+                    break;
+                }
+            } else {
+                at = 0;
+            }
+        }
+        if (shouldBeFound == found) {
+            return;
+        }
+        if (shouldBeFound) {
+            fail("Value " + value + " not found in\n" + Arrays.toString(arr));
+        } else {
+            fail("Value " + value + " surprisingly found in\n" + Arrays.toString(arr));
+        }
+    }
+
+    private static final class MockStructure implements GraphStructure<MockGraph, MockNode, MockNodeClass, MockNodeClass> {
+
+        @Override
+        public MockGraph graph(MockGraph currentGraph, Object obj) {
+            return obj instanceof MockGraph ? (MockGraph) obj : null;
+        }
+
+        @Override
+        public Iterable<? extends MockNode> nodes(MockGraph graph) {
+            return Collections.emptyList();
+        }
+
+        @Override
+        public int nodesCount(MockGraph graph) {
+            return 0;
+        }
+
+        @Override
+        public int nodeId(MockNode node) {
+            node.idTested++;
+            return node.id;
+        }
+
+        @Override
+        public boolean nodeHasPredecessor(MockNode node) {
+            return false;
+        }
+
+        @Override
+        public void nodeProperties(MockGraph graph, MockNode node, Map<String, ? super Object> properties) {
+        }
+
+        @Override
+        public MockNode node(Object obj) {
+            if (obj instanceof MockNode) {
+                ((MockNode) obj).nodeRequested++;
+                return (MockNode) obj;
+            }
+            return null;
+        }
+
+        @Override
+        public MockNodeClass nodeClass(Object obj) {
+            if (obj instanceof MockNode) {
+                ((MockNode) obj).nodeClassRequested++;
+            }
+            return obj instanceof MockNodeClass ? (MockNodeClass) obj : null;
+        }
+
+        @Override
+        public MockNodeClass classForNode(MockNode n) {
+            n.nodeClassRequested++;
+            return n.clazz;
+        }
+
+        @Override
+        public String nameTemplate(MockNodeClass nodeClass) {
+            nodeClass.nameTemplateQueried++;
+            return "";
+        }
+
+        @Override
+        public Object nodeClassType(MockNodeClass nodeClass) {
+            return nodeClass.getClass();
+        }
+
+        @Override
+        public MockNodeClass portInputs(MockNodeClass nodeClass) {
+            return nodeClass;
+        }
+
+        @Override
+        public MockNodeClass portOutputs(MockNodeClass nodeClass) {
+            return nodeClass;
+        }
+
+        @Override
+        public int portSize(MockNodeClass port) {
+            return 0;
+        }
+
+        @Override
+        public boolean edgeDirect(MockNodeClass port, int index) {
+            return false;
+        }
+
+        @Override
+        public String edgeName(MockNodeClass port, int index) {
+            return null;
+        }
+
+        @Override
+        public Object edgeType(MockNodeClass port, int index) {
+            return null;
+        }
+
+        @Override
+        public Collection<? extends MockNode> edgeNodes(MockGraph graph, MockNode node, MockNodeClass port, int index) {
+            return null;
+        }
+    }
+
+    private static final class MockGraph {
+
+    }
+
+    private static final class MockNode {
+        final MockNodeClass clazz;
+        final int id;
+        int idTested;
+        int nodeClassRequested;
+        int nodeRequested;
+        boolean toStringRequested;
+
+        MockNode(MockNodeClass clazz, int id) {
+            this.clazz = clazz;
+            this.id = id;
+        }
+
+        @Override
+        public String toString() {
+            this.toStringRequested = true;
+            return "MockNode{" + "id=" + id + ", class=" + clazz + '}';
+        }
+    }
+
+    private static final class MockNodeClass {
+        final String name;
+        int nameTemplateQueried;
+
+        MockNodeClass(String name) {
+            this.name = name;
+        }
+
+        @Override
+        public String toString() {
+            return "MockNodeClass{" + "name=" + name + '}';
+        }
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/Graph.java

@@ -514,30 +514,61 @@ public class Graph {
         /**
          * A node was added to a graph.
          */
-        NODE_ADDED;
+        NODE_ADDED,
+
+        /**
+         * A node was removed from the graph.
+         */
+        NODE_REMOVED;
     }
 
     /**
      * Client interested in one or more node related events.
      */
-    public interface NodeEventListener {
+    public abstract static class NodeEventListener {
 
         /**
-         * Default handler for events.
+         * A method called when a change event occurs.
+         *
+         * This method dispatches the event to user-defined triggers. The methods that change the
+         * graph (typically in Graph and Node) must call this method to dispatch the event.
          *
          * @param e an event
          * @param node the node related to {@code e}
          */
-        default void event(NodeEvent e, Node node) {
+        final void event(NodeEvent e, Node node) {
+            switch (e) {
+                case INPUT_CHANGED:
+                    inputChanged(node);
+                    break;
+                case ZERO_USAGES:
+                    usagesDroppedToZero(node);
+                    break;
+                case NODE_ADDED:
+                    nodeAdded(node);
+                    break;
+                case NODE_REMOVED:
+                    nodeRemoved(node);
+                    break;
+            }
+            changed(e, node);
         }
 
         /**
-         * Notifies this listener of a change in a node's inputs.
+         * Notifies this listener about any change event in the graph.
+         *
+         * @param e an event
+         * @param node the node related to {@code e}
+         */
+        public void changed(NodeEvent e, Node node) {
+        }
+
+        /**
+         * Notifies this listener about a change in a node's inputs.
          *
          * @param node a node who has had one of its inputs changed
          */
-        default void inputChanged(Node node) {
-            event(NodeEvent.INPUT_CHANGED, node);
+        public void inputChanged(Node node) {
         }
 
         /**
@@ -545,8 +576,7 @@ public class Graph {
          *
          * @param node a node whose {@link Node#usages()} just became empty
          */
-        default void usagesDroppedToZero(Node node) {
-            event(NodeEvent.ZERO_USAGES, node);
+        public void usagesDroppedToZero(Node node) {
         }
 
         /**
@@ -554,8 +584,15 @@ public class Graph {
          *
          * @param node a node that was just added to the graph
          */
-        default void nodeAdded(Node node) {
-            event(NodeEvent.NODE_ADDED, node);
+        public void nodeAdded(Node node) {
+        }
+
+        /**
+         * Notifies this listener of a removed node.
+         *
+         * @param node
+         */
+        public void nodeRemoved(Node node) {
         }
     }
 
@@ -583,7 +620,7 @@ public class Graph {
         }
     }
 
-    private static class ChainedNodeEventListener implements NodeEventListener {
+    private static class ChainedNodeEventListener extends NodeEventListener {
 
         NodeEventListener head;
         NodeEventListener next;
@@ -595,20 +632,32 @@ public class Graph {
 
         @Override
         public void nodeAdded(Node node) {
-            head.nodeAdded(node);
-            next.nodeAdded(node);
+            head.event(NodeEvent.NODE_ADDED, node);
+            next.event(NodeEvent.NODE_ADDED, node);
         }
 
         @Override
         public void inputChanged(Node node) {
-            head.inputChanged(node);
-            next.inputChanged(node);
+            head.event(NodeEvent.INPUT_CHANGED, node);
+            next.event(NodeEvent.INPUT_CHANGED, node);
         }
 
         @Override
         public void usagesDroppedToZero(Node node) {
-            head.usagesDroppedToZero(node);
-            next.usagesDroppedToZero(node);
+            head.event(NodeEvent.ZERO_USAGES, node);
+            next.event(NodeEvent.ZERO_USAGES, node);
+        }
+
+        @Override
+        public void nodeRemoved(Node node) {
+            head.event(NodeEvent.NODE_REMOVED, node);
+            next.event(NodeEvent.NODE_REMOVED, node);
+        }
+
+        @Override
+        public void changed(NodeEvent e, Node node) {
+            head.event(e, node);
+            next.event(e, node);
         }
     }
 
@@ -1023,7 +1072,7 @@ public class Graph {
         updateNodeCaches(node);
 
         if (nodeEventListener != null) {
-            nodeEventListener.nodeAdded(node);
+            nodeEventListener.event(NodeEvent.NODE_ADDED, node);
         }
         afterRegister(node);
     }
@@ -1085,6 +1134,10 @@ public class Graph {
         nodes[node.id] = null;
         nodesDeletedSinceLastCompression++;
 
+        if (nodeEventListener != null) {
+            nodeEventListener.event(NodeEvent.NODE_ADDED, node);
+        }
+
         // nodes aren't removed from the type cache here - they will be removed during iteration
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/Node.java

@@ -752,7 +752,7 @@ public abstract class Node implements Cloneable, Formattable, NodeInterface {
             assert !graph.isFrozen();
             NodeEventListener listener = graph.nodeEventListener;
             if (listener != null) {
-                listener.inputChanged(node);
+                listener.event(Graph.NodeEvent.INPUT_CHANGED, node);
             }
         }
     }
@@ -762,7 +762,7 @@ public abstract class Node implements Cloneable, Formattable, NodeInterface {
             assert !graph.isFrozen();
             NodeEventListener listener = graph.nodeEventListener;
             if (listener != null && node.isAlive()) {
-                listener.usagesDroppedToZero(node);
+                listener.event(Graph.NodeEvent.ZERO_USAGES, node);
             }
         }
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotNodeLIRBuilder.java

@@ -183,8 +183,7 @@ public class AArch64HotSpotNodeLIRBuilder extends AArch64NodeLIRBuilder implemen
             sig[i] = node.arguments().get(i).stamp().javaType(gen.getMetaAccess());
         }
 
-        Value[] parameters = visitInvokeArguments(gen.getResult().getFrameMapBuilder().getRegisterConfig().getCallingConvention(HotSpotCallingConventionType.JavaCall, null, sig, gen),
-                        node.arguments());
+        Value[] parameters = visitInvokeArguments(gen.getRegisterConfig().getCallingConvention(HotSpotCallingConventionType.JavaCall, null, sig, gen), node.arguments());
         append(new AArch64BreakpointOp(parameters));
     }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotAddressLowering.java

@@ -32,7 +32,6 @@ import org.graalvm.compiler.core.amd64.AMD64AddressLowering;
 import org.graalvm.compiler.core.amd64.AMD64AddressNode;
 import org.graalvm.compiler.core.common.CompressEncoding;
 import org.graalvm.compiler.core.common.LIRKind;
-import org.graalvm.compiler.core.common.NumUtil;
 import org.graalvm.compiler.core.common.type.ObjectStamp;
 import org.graalvm.compiler.core.common.type.StampFactory;
 import org.graalvm.compiler.debug.CounterKey;
@@ -44,6 +43,7 @@ import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
 import org.graalvm.compiler.nodeinfo.NodeInfo;
 import org.graalvm.compiler.nodes.CompressionNode;
 import org.graalvm.compiler.nodes.CompressionNode.CompressionOp;
+import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.calc.FloatingNode;
 import org.graalvm.compiler.nodes.spi.LIRLowerable;
@@ -93,35 +93,40 @@ public class AMD64HotSpotAddressLowering extends AMD64AddressLowering {
     }
 
     @Override
-    protected boolean improve(DebugContext debug, AMD64AddressNode addr) {
-
-        boolean result = false;
-
-        while (super.improve(debug, addr)) {
-            result = true;
+    protected boolean improve(StructuredGraph graph, DebugContext debug, AMD64AddressNode addr, boolean isBaseNegated, boolean isIndexNegated) {
+        if (super.improve(graph, debug, addr, isBaseNegated, isIndexNegated)) {
+            return true;
         }
 
         if (addr.getScale() == Scale.Times1) {
             if (addr.getIndex() instanceof CompressionNode) {
-                if (improveUncompression(addr, (CompressionNode) addr.getIndex(), addr.getBase())) {
+                if (improveUncompression(addr, (CompressionNode) addr.getIndex(), addr.getBase(), isBaseNegated, isIndexNegated)) {
                     counterFoldedUncompressDuringAddressLowering.increment(debug);
                     return true;
                 }
             }
 
             if (addr.getBase() instanceof CompressionNode) {
-                if (improveUncompression(addr, (CompressionNode) addr.getBase(), addr.getIndex())) {
+                if (improveUncompression(addr, (CompressionNode) addr.getBase(), addr.getIndex(), isBaseNegated, isIndexNegated)) {
                     counterFoldedUncompressDuringAddressLowering.increment(debug);
                     return true;
                 }
             }
         }
 
-        return result;
+        return false;
+    }
+
+    @Override
+    protected boolean mightBeOptimized(ValueNode value) {
+        return super.mightBeOptimized(value) || value instanceof CompressionNode;
+    }
+
+    private boolean improveUncompression(AMD64AddressNode addr, CompressionNode compression, ValueNode other, boolean isBaseNegated, boolean isIndexNegated) {
+        if (isBaseNegated || isIndexNegated || compression.getOp() != CompressionOp.Uncompress) {
+            return false;
         }
 
-    private boolean improveUncompression(AMD64AddressNode addr, CompressionNode compression, ValueNode other) {
-        if (compression.getOp() == CompressionOp.Uncompress) {
         CompressEncoding encoding = compression.getEncoding();
         Scale scale = Scale.fromShift(encoding.getShift());
         if (scale == null) {
@@ -146,9 +151,7 @@ public class AMD64HotSpotAddressLowering extends AMD64AddressLowering {
                     return false;
                 }
             } else {
-                    long disp = addr.getDisplacement() + encoding.getBase();
-                    if (NumUtil.isInt(disp)) {
-                        addr.setDisplacement((int) disp);
+                if (updateDisplacement(addr, encoding.getBase(), isBaseNegated)) {
                     addr.setBase(other);
                 } else {
                     return false;
@@ -161,8 +164,5 @@ public class AMD64HotSpotAddressLowering extends AMD64AddressLowering {
         addr.setScale(scale);
         addr.setIndex(compression.getValue());
         return true;
-        } else {
-            return false;
-        }
     }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotLIRGenerator.java

@@ -39,7 +39,6 @@ import java.util.List;
 import org.graalvm.compiler.asm.amd64.AMD64Address.Scale;
 import org.graalvm.compiler.core.amd64.AMD64ArithmeticLIRGenerator;
 import org.graalvm.compiler.core.amd64.AMD64LIRGenerator;
-import org.graalvm.compiler.core.amd64.AMD64LIRKindTool;
 import org.graalvm.compiler.core.amd64.AMD64MoveFactoryBase.BackupSlotProvider;
 import org.graalvm.compiler.core.common.CompressEncoding;
 import org.graalvm.compiler.core.common.LIRKind;
@@ -116,7 +115,7 @@ public class AMD64HotSpotLIRGenerator extends AMD64LIRGenerator implements HotSp
     }
 
     private AMD64HotSpotLIRGenerator(HotSpotProviders providers, GraalHotSpotVMConfig config, LIRGenerationResult lirGenRes, BackupSlotProvider backupSlotProvider) {
-        this(new AMD64LIRKindTool(), new AMD64HotSpotArithmeticLIRGenerator(), new AMD64HotSpotMoveFactory(backupSlotProvider), providers, config, lirGenRes);
+        this(new AMD64HotSpotLIRKindTool(), new AMD64HotSpotArithmeticLIRGenerator(), new AMD64HotSpotMoveFactory(backupSlotProvider), providers, config, lirGenRes);
     }
 
     protected AMD64HotSpotLIRGenerator(LIRKindTool lirKindTool, AMD64ArithmeticLIRGenerator arithmeticLIRGen, MoveFactory moveFactory, HotSpotProviders providers, GraalHotSpotVMConfig config,
@@ -363,7 +362,7 @@ public class AMD64HotSpotLIRGenerator extends AMD64LIRGenerator implements HotSp
         Stub stub = getStub();
         if (destroysRegisters) {
             if (stub != null && stub.preservesRegisters()) {
-                Register[] savedRegisters = getResult().getFrameMapBuilder().getRegisterConfig().getAllocatableRegisters().toArray();
+                Register[] savedRegisters = getRegisterConfig().getAllocatableRegisters().toArray();
                 save = emitSaveAllRegisters(savedRegisters, true);
             }
         }
@@ -567,28 +566,29 @@ public class AMD64HotSpotLIRGenerator extends AMD64LIRGenerator implements HotSp
     @Override
     public Value emitCompress(Value pointer, CompressEncoding encoding, boolean nonNull) {
         LIRKind inputKind = pointer.getValueKind(LIRKind.class);
-        assert inputKind.getPlatformKind() == AMD64Kind.QWORD;
+        LIRKindTool lirKindTool = getLIRKindTool();
+        assert inputKind.getPlatformKind() == lirKindTool.getObjectKind().getPlatformKind();
         if (inputKind.isReference(0)) {
             // oop
-            Variable result = newVariable(LIRKind.reference(AMD64Kind.DWORD));
-            append(new AMD64HotSpotMove.CompressPointer(result, asAllocatable(pointer), getProviders().getRegisters().getHeapBaseRegister().asValue(), encoding, nonNull));
+            Variable result = newVariable(lirKindTool.getNarrowOopKind());
+            append(new AMD64Move.CompressPointer(result, asAllocatable(pointer), getProviders().getRegisters().getHeapBaseRegister().asValue(), encoding, nonNull, getLIRKindTool()));
             return result;
         } else {
             // metaspace pointer
-            Variable result = newVariable(LIRKind.value(AMD64Kind.DWORD));
+            Variable result = newVariable(lirKindTool.getNarrowPointerKind());
             AllocatableValue base = Value.ILLEGAL;
             OptionValues options = getResult().getLIR().getOptions();
             if (encoding.hasBase() || GeneratePIC.getValue(options)) {
                 if (GeneratePIC.getValue(options)) {
-                    Variable baseAddress = newVariable(LIRKind.value(AMD64Kind.QWORD));
+                    Variable baseAddress = newVariable(lirKindTool.getWordKind());
                     AMD64HotSpotMove.BaseMove move = new AMD64HotSpotMove.BaseMove(baseAddress, config);
                     append(move);
                     base = baseAddress;
                 } else {
-                    base = emitLoadConstant(LIRKind.value(AMD64Kind.QWORD), JavaConstant.forLong(encoding.getBase()));
+                    base = emitLoadConstant(lirKindTool.getWordKind(), JavaConstant.forLong(encoding.getBase()));
                 }
             }
-            append(new AMD64HotSpotMove.CompressPointer(result, asAllocatable(pointer), base, encoding, nonNull));
+            append(new AMD64Move.CompressPointer(result, asAllocatable(pointer), base, encoding, nonNull, getLIRKindTool()));
             return result;
         }
     }
@@ -596,35 +596,37 @@ public class AMD64HotSpotLIRGenerator extends AMD64LIRGenerator implements HotSp
     @Override
     public Value emitUncompress(Value pointer, CompressEncoding encoding, boolean nonNull) {
         LIRKind inputKind = pointer.getValueKind(LIRKind.class);
-        assert inputKind.getPlatformKind() == AMD64Kind.DWORD;
+        LIRKindTool lirKindTool = getLIRKindTool();
+        assert inputKind.getPlatformKind() == lirKindTool.getNarrowOopKind().getPlatformKind();
         if (inputKind.isReference(0)) {
             // oop
-            Variable result = newVariable(LIRKind.reference(AMD64Kind.QWORD));
-            append(new AMD64HotSpotMove.UncompressPointer(result, asAllocatable(pointer), getProviders().getRegisters().getHeapBaseRegister().asValue(), encoding, nonNull));
+            Variable result = newVariable(lirKindTool.getObjectKind());
+            append(new AMD64Move.UncompressPointer(result, asAllocatable(pointer), getProviders().getRegisters().getHeapBaseRegister().asValue(), encoding, nonNull, lirKindTool));
             return result;
         } else {
             // metaspace pointer
-            Variable result = newVariable(LIRKind.value(AMD64Kind.QWORD));
+            LIRKind uncompressedKind = lirKindTool.getWordKind();
+            Variable result = newVariable(uncompressedKind);
             AllocatableValue base = Value.ILLEGAL;
             OptionValues options = getResult().getLIR().getOptions();
             if (encoding.hasBase() || GeneratePIC.getValue(options)) {
                 if (GeneratePIC.getValue(options)) {
-                    Variable baseAddress = newVariable(LIRKind.value(AMD64Kind.QWORD));
+                    Variable baseAddress = newVariable(uncompressedKind);
                     AMD64HotSpotMove.BaseMove move = new AMD64HotSpotMove.BaseMove(baseAddress, config);
                     append(move);
                     base = baseAddress;
                 } else {
-                    base = emitLoadConstant(LIRKind.value(AMD64Kind.QWORD), JavaConstant.forLong(encoding.getBase()));
+                    base = emitLoadConstant(uncompressedKind, JavaConstant.forLong(encoding.getBase()));
                 }
             }
-            append(new AMD64HotSpotMove.UncompressPointer(result, asAllocatable(pointer), base, encoding, nonNull));
+            append(new AMD64Move.UncompressPointer(result, asAllocatable(pointer), base, encoding, nonNull, lirKindTool));
             return result;
         }
     }
 
     @Override
     public void emitNullCheck(Value address, LIRFrameState state) {
-        if (address.getValueKind().getPlatformKind() == AMD64Kind.DWORD) {
+        if (address.getValueKind().getPlatformKind() == getLIRKindTool().getNarrowOopKind().getPlatformKind()) {
             CompressEncoding encoding = config.getOopEncoding();
             Value uncompressed;
             if (encoding.getShift() <= 3) {
@@ -635,9 +637,9 @@ public class AMD64HotSpotLIRGenerator extends AMD64LIRGenerator implements HotSp
                 uncompressed = emitUncompress(address, encoding, false);
             }
             append(new AMD64Move.NullCheckOp(asAddressValue(uncompressed), state));
-        } else {
-            super.emitNullCheck(address, state);
+            return;
         }
+        super.emitNullCheck(address, state);
     }
 
     @Override

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotLIRKindTool.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.hotspot.amd64;
+
+import jdk.vm.ci.amd64.AMD64Kind;
+import org.graalvm.compiler.core.amd64.AMD64LIRKindTool;
+import org.graalvm.compiler.core.common.LIRKind;
+
+public class AMD64HotSpotLIRKindTool extends AMD64LIRKindTool {
+    @Override
+    public LIRKind getNarrowOopKind() {
+        return LIRKind.reference(AMD64Kind.DWORD);
+    }
+
+    @Override
+    public LIRKind getNarrowPointerKind() {
+        return LIRKind.value(AMD64Kind.DWORD);
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotMove.java

@@ -24,16 +24,13 @@ package org.graalvm.compiler.hotspot.amd64;
 
 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.HINT;
-import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.ILLEGAL;
 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG;
 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK;
 import static jdk.vm.ci.code.ValueUtil.asRegister;
 import static jdk.vm.ci.code.ValueUtil.isRegister;
 import static jdk.vm.ci.code.ValueUtil.isStackSlot;
 
-import org.graalvm.compiler.asm.Label;
 import org.graalvm.compiler.asm.amd64.AMD64Address;
-import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag;
 import org.graalvm.compiler.core.common.CompressEncoding;
 import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler;
 import org.graalvm.compiler.debug.GraalError;
@@ -41,10 +38,8 @@ import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
 import org.graalvm.compiler.lir.LIRInstructionClass;
 import org.graalvm.compiler.lir.StandardOp.LoadConstantOp;
 import org.graalvm.compiler.lir.amd64.AMD64LIRInstruction;
-import org.graalvm.compiler.lir.amd64.AMD64Move;
 import org.graalvm.compiler.lir.asm.CompilationResultBuilder;
 
-import jdk.vm.ci.amd64.AMD64Kind;
 import jdk.vm.ci.code.Register;
 import jdk.vm.ci.hotspot.HotSpotMetaspaceConstant;
 import jdk.vm.ci.hotspot.HotSpotObjectConstant;
@@ -180,91 +175,6 @@ public class AMD64HotSpotMove {
         }
     }
 
-    public static final class CompressPointer extends AMD64LIRInstruction {
-        public static final LIRInstructionClass<CompressPointer> TYPE = LIRInstructionClass.create(CompressPointer.class);
-
-        private final CompressEncoding encoding;
-        private final boolean nonNull;
-
-        @Def({REG, HINT}) protected AllocatableValue result;
-        @Use({REG}) protected AllocatableValue input;
-        @Alive({REG, ILLEGAL}) protected AllocatableValue baseRegister;
-
-        public CompressPointer(AllocatableValue result, AllocatableValue input, AllocatableValue baseRegister, CompressEncoding encoding, boolean nonNull) {
-            super(TYPE);
-            this.result = result;
-            this.input = input;
-            this.baseRegister = baseRegister;
-            this.encoding = encoding;
-            this.nonNull = nonNull;
-        }
-
-        @Override
-        public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
-            AMD64Move.move(AMD64Kind.QWORD, crb, masm, result, input);
-
-            Register resReg = asRegister(result);
-            if (encoding.hasBase() || GeneratePIC.getValue(crb.getOptions())) {
-                Register baseReg = asRegister(baseRegister);
-                if (!nonNull) {
-                    masm.testq(resReg, resReg);
-                    masm.cmovq(ConditionFlag.Equal, resReg, baseReg);
-                }
-                masm.subq(resReg, baseReg);
-            }
-
-            if (encoding.hasShift()) {
-                masm.shrq(resReg, encoding.getShift());
-            }
-        }
-    }
-
-    public static final class UncompressPointer extends AMD64LIRInstruction {
-        public static final LIRInstructionClass<UncompressPointer> TYPE = LIRInstructionClass.create(UncompressPointer.class);
-
-        private final CompressEncoding encoding;
-        private final boolean nonNull;
-
-        @Def({REG, HINT}) protected AllocatableValue result;
-        @Use({REG}) protected AllocatableValue input;
-        @Alive({REG, ILLEGAL}) protected AllocatableValue baseRegister;
-
-        public UncompressPointer(AllocatableValue result, AllocatableValue input, AllocatableValue baseRegister, CompressEncoding encoding, boolean nonNull) {
-            super(TYPE);
-            this.result = result;
-            this.input = input;
-            this.baseRegister = baseRegister;
-            this.encoding = encoding;
-            this.nonNull = nonNull;
-        }
-
-        @Override
-        public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
-            AMD64Move.move(AMD64Kind.DWORD, crb, masm, result, input);
-
-            Register resReg = asRegister(result);
-            if (encoding.getShift() != 0) {
-                masm.shlq(resReg, encoding.getShift());
-            }
-
-            if (encoding.hasBase() || GeneratePIC.getValue(crb.getOptions())) {
-                if (nonNull) {
-                    masm.addq(resReg, asRegister(baseRegister));
-                } else {
-                    if (!encoding.hasShift()) {
-                        // if encoding.shift != 0, the flags are already set by the shlq
-                        masm.testq(resReg, resReg);
-                    }
-
-                    Label done = new Label();
-                    masm.jccb(ConditionFlag.Equal, done);
-                    masm.addq(resReg, asRegister(baseRegister));
-                    masm.bind(done);
-                }
-            }
-        }
-    }
-
     public static void decodeKlassPointer(CompilationResultBuilder crb, AMD64MacroAssembler masm, Register register, Register scratch, AMD64Address address, GraalHotSpotVMConfig config) {
         CompressEncoding encoding = config.getKlassEncoding();
         masm.movl(register, address);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotNodeLIRBuilder.java

@@ -189,8 +189,7 @@ public class AMD64HotSpotNodeLIRBuilder extends AMD64NodeLIRBuilder implements H
             sig[i] = node.arguments().get(i).stamp().javaType(gen.getMetaAccess());
         }
 
-        Value[] parameters = visitInvokeArguments(gen.getResult().getFrameMapBuilder().getRegisterConfig().getCallingConvention(HotSpotCallingConventionType.JavaCall, null, sig, gen),
-                        node.arguments());
+        Value[] parameters = visitInvokeArguments(gen.getRegisterConfig().getCallingConvention(HotSpotCallingConventionType.JavaCall, null, sig, gen), node.arguments());
         append(new AMD64BreakpointOp(parameters));
     }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotMove.java

@@ -113,8 +113,7 @@ public class SPARCHotSpotMove {
                 } else {
                     register = asRegister(result);
                 }
-                int bytes = result.getPlatformKind().getSizeInBytes();
-                loadFromConstantTable(crb, masm, bytes, asRegister(constantTableBase), constant, register, SPARCDelayedControlTransfer.DUMMY);
+                int bytes = loadFromConstantTable(crb, masm, asRegister(constantTableBase), constant, register, SPARCDelayedControlTransfer.DUMMY);
                 if (isStack) {
                     masm.st(register, (SPARCAddress) crb.asAddress(result), bytes);
                 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotNodeLIRBuilder.java

@@ -162,8 +162,7 @@ public class SPARCHotSpotNodeLIRBuilder extends SPARCNodeLIRBuilder implements H
             sig[i] = node.arguments().get(i).stamp().javaType(gen.getMetaAccess());
         }
 
-        Value[] parameters = visitInvokeArguments(gen.getResult().getFrameMapBuilder().getRegisterConfig().getCallingConvention(HotSpotCallingConventionType.JavaCall, null, sig, gen),
-                        node.arguments());
+        Value[] parameters = visitInvokeArguments(gen.getRegisterConfig().getCallingConvention(HotSpotCallingConventionType.JavaCall, null, sig, gen), node.arguments());
         append(new SPARCBreakpointOp(parameters));
     }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotStrategySwitchOp.java

@@ -77,8 +77,7 @@ final class SPARCHotSpotStrategySwitchOp extends SPARCControlFlow.StrategySwitch
                 boolean canUseShortBranch = masm.hasFeature(CPUFeature.CBCOND) && SPARCControlFlow.isShortBranch(masm, cbCondPosition, hint, target);
 
                 Register scratchRegister = asRegister(scratch);
-                final int byteCount = constant.isCompressed() ? 4 : 8;
-                loadFromConstantTable(crb, masm, byteCount, asRegister(constantTableBase), constant, scratchRegister, SPARCDelayedControlTransfer.DUMMY);
+                loadFromConstantTable(crb, masm, asRegister(constantTableBase), constant, scratchRegister, SPARCDelayedControlTransfer.DUMMY);
 
                 if (canUseShortBranch) {
                     CBCOND.emit(masm, conditionFlag, conditionCode == CC.Xcc, keyRegister, scratchRegister, target);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/ArrayCopyIntrinsificationTest.java

@@ -160,7 +160,7 @@ public class ArrayCopyIntrinsificationTest extends GraalCompilerTest {
     }
 
     /**
-     * Tests {@link ArrayCopySnippets#checkcastArraycopyWork}.
+     * Tests {@link ArrayCopySnippets#arraycopyCheckcastSnippet}.
      */
     @Test
     public void testArrayStoreException() {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CompilationWrapperTest.java

@@ -122,7 +122,7 @@ public class CompilationWrapperTest extends GraalCompilerTest {
      * Tests compilation requested by Truffle.
      */
     @Test
-    public void testTruffleCompilation() throws IOException, InterruptedException {
+    public void testTruffleCompilation1() throws IOException, InterruptedException {
         testHelper(Collections.emptyList(),
                         Arrays.asList(
                                         "-Dgraal.CompilationFailureAction=ExitVM",
@@ -130,6 +130,22 @@ public class CompilationWrapperTest extends GraalCompilerTest {
                         "org.graalvm.compiler.truffle.test.SLTruffleGraalTestSuite", "test");
     }
 
+    /**
+     * Tests that TruffleCompilationExceptionsAreFatal works as expected.
+     */
+    @Test
+    public void testTruffleCompilation2() throws IOException, InterruptedException {
+        Probe[] probes = {
+                        new Probe("Exiting VM due to TruffleCompilationExceptionsAreFatal=true", 1),
+        };
+        testHelper(Arrays.asList(probes),
+                        Arrays.asList(
+                                        "-Dgraal.CompilationFailureAction=Silent",
+                                        "-Dgraal.TruffleCompilationExceptionsAreFatal=true",
+                                        "-Dgraal.CrashAt=root test1"),
+                        "org.graalvm.compiler.truffle.test.SLTruffleGraalTestSuite", "test");
+    }
+
     private static final boolean VERBOSE = Boolean.getBoolean(CompilationWrapperTest.class.getSimpleName() + ".verbose");
 
     private static void testHelper(List<Probe> initialProbes, List<String> extraVmArgs, String... mainClassAndArgs) throws IOException, InterruptedException {
@@ -149,7 +165,9 @@ public class CompilationWrapperTest extends GraalCompilerTest {
         }
 
         List<Probe> probes = new ArrayList<>(initialProbes);
-        Probe diagnosticProbe = new Probe("Graal diagnostic output saved in ", 1);
+        Probe diagnosticProbe = null;
+        if (!extraVmArgs.contains("-Dgraal.TruffleCompilationExceptionsAreFatal=true")) {
+            diagnosticProbe = new Probe("Graal diagnostic output saved in ", 1);
             probes.add(diagnosticProbe);
             probes.add(new Probe("Forced crash after compiling", Integer.MAX_VALUE) {
                 @Override
@@ -157,6 +175,7 @@ public class CompilationWrapperTest extends GraalCompilerTest {
                     return actualOccurrences > 0 ? null : "expected at least 1 occurrence";
                 }
             });
+        }
 
         for (String line : proc.output) {
             for (Probe probe : probes) {
@@ -171,8 +190,11 @@ public class CompilationWrapperTest extends GraalCompilerTest {
                 Assert.fail(String.format("Did not find expected occurences of '%s' in output of command: %s%n%s", probe.substring, error, proc));
             }
         }
-
-        String diagnosticOutputZip = diagnosticProbe.lastMatchingLine.substring(diagnosticProbe.substring.length()).trim();
+        if (diagnosticProbe != null) {
+            String line = diagnosticProbe.lastMatchingLine;
+            int substringStart = line.indexOf(diagnosticProbe.substring);
+            int substringLength = diagnosticProbe.substring.length();
+            String diagnosticOutputZip = line.substring(substringStart + substringLength).trim();
 
             List<String> dumpPathEntries = Arrays.asList(dumpPath.list());
 
@@ -206,3 +228,4 @@ public class CompilationWrapperTest extends GraalCompilerTest {
             }
         }
     }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/ExplicitExceptionTest.java

@@ -22,23 +22,44 @@
  */
 package org.graalvm.compiler.hotspot.test;
 
-import org.junit.Test;
-
 import org.graalvm.compiler.core.test.GraalCompilerTest;
 import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
 import org.graalvm.compiler.options.OptionValues;
+import org.junit.Assume;
+import org.junit.Test;
 
 import jdk.vm.ci.code.InstalledCode;
+import jdk.vm.ci.meta.ProfilingInfo;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
+import jdk.vm.ci.meta.TriState;
 
 public class ExplicitExceptionTest extends GraalCompilerTest {
 
     private int expectedForeignCallCount;
 
+    /**
+     * Determines if profiling info for {@code method} indicates an exception was thrown somewhere
+     * in the method. In the case of the {@code -Xcomp} VM option, interpreter execution can be
+     * skipped altogether and other execution engines (e.g., C1) may not record seen exceptions in a
+     * method profile.
+     */
+    private static boolean exceptionWasSeen(ResolvedJavaMethod method) {
+        ProfilingInfo profilingInfo = method.getProfilingInfo();
+        if (profilingInfo != null) {
+            for (int i = 0; i < profilingInfo.getCodeSize(); i++) {
+                if (profilingInfo.getExceptionSeen(i) == TriState.TRUE) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
     @Override
     protected InstalledCode getCode(ResolvedJavaMethod method, StructuredGraph graph, boolean forceCompile, boolean installAsDefault, OptionValues options) {
         InstalledCode installedCode = super.getCode(method, graph, forceCompile, installAsDefault, options);
+        Assume.assumeTrue(exceptionWasSeen(method));
         assertDeepEquals(expectedForeignCallCount, lastCompiledGraph.getNodes().filter(ForeignCallNode.class).count());
         return installedCode;
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/GraalOSRLockTest.java

@@ -24,11 +24,13 @@ package org.graalvm.compiler.hotspot.test;
 
 import java.lang.management.ManagementFactory;
 import java.lang.management.MonitorInfo;
+import java.lang.management.RuntimeMXBean;
 import java.lang.management.ThreadInfo;
 import java.lang.management.ThreadMXBean;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 
 import org.graalvm.compiler.api.directives.GraalDirectives;
 import org.graalvm.compiler.core.phases.HighTier;
@@ -44,6 +46,7 @@ import org.junit.Test;
 
 import jdk.vm.ci.meta.ResolvedJavaMethod;
 import org.junit.Assume;
+import org.junit.BeforeClass;
 
 /**
  * Test on-stack-replacement with locks.
@@ -51,13 +54,28 @@ import org.junit.Assume;
 public class GraalOSRLockTest extends GraalOSRTestBase {
 
     private static boolean TestInSeparateThread = false;
+    private static final String COMPILE_ONLY_FLAG = "-Xcomp";
 
-    public GraalOSRLockTest() {
+    @BeforeClass
+    public static void checkVMArguments() {
         try {
             Class.forName("java.lang.management.ManagementFactory");
         } catch (ClassNotFoundException ex) {
             Assume.assumeNoException("cannot check for monitors without java.management JDK9 module", ex);
         }
+        /*
+         * Note: The -Xcomp execution mode of the VM will stop most of the OSR test cases from
+         * working as every method is compiled at level3 (followed by level4 on the second
+         * invocation). The tests in this class are written in a way that they expect a method to be
+         * executed at the invocation BCI with the interpreter and then perform an OSR to an
+         * installed nmethod at a given BCI.
+         *
+         */
+        RuntimeMXBean runtimeMxBean = ManagementFactory.getRuntimeMXBean();
+        List<String> arguments = runtimeMxBean.getInputArguments();
+        for (String arg : arguments) {
+            Assume.assumeFalse(arg.equals(COMPILE_ONLY_FLAG));
+        }
     }
 
     // testing only
@@ -438,7 +456,7 @@ public class GraalOSRLockTest extends GraalOSRTestBase {
             }
             GraalDirectives.controlFlowAnchor();
             if (!GraalDirectives.inCompiledCode()) {
-                throw new Error("Must part of compiled code");
+                throw new Error("Must be part of compiled code");
             }
             return ret;
         }
@@ -449,11 +467,11 @@ public class GraalOSRLockTest extends GraalOSRTestBase {
         ReturnValue ret = ReturnValue.FAILURE;
         synchronized (lock) {
             synchronized (lock1) {
-                for (int i = 1; i < limit; i++) {
+                for (int i = 1; i < 10 * limit; i++) {
                     GraalDirectives.blackhole(i);
-                    if (i % 1001 == 0) {
+                    if (i % 33 == 0) {
                         ret = ReturnValue.SUCCESS;
-                        if (GraalDirectives.inCompiledCode() && i + 33 > (limit)) {
+                        if (GraalDirectives.inCompiledCode() && i + 33 > (10 * limit)) {
                             GraalDirectives.blackhole(ret);
                             System.gc();
                         }
@@ -462,7 +480,7 @@ public class GraalOSRLockTest extends GraalOSRTestBase {
             }
             GraalDirectives.controlFlowAnchor();
             if (!GraalDirectives.inCompiledCode()) {
-                throw new Error("Must part of compiled code");
+                throw new Error("Must be part of compiled code already hereeeeee");
             } else {
                 // lock 1 must be free
                 if (isMonitorLockHeld(lock1)) {
@@ -519,7 +537,7 @@ public class GraalOSRLockTest extends GraalOSRTestBase {
             }
             GraalDirectives.controlFlowAnchor();
             if (!GraalDirectives.inCompiledCode()) {
-                throw new Error("Must part of compiled code");
+                throw new Error("Must be part of compiled code");
             }
             return ret;
         }
@@ -543,7 +561,7 @@ public class GraalOSRLockTest extends GraalOSRTestBase {
             }
             GraalDirectives.controlFlowAnchor();
             if (!GraalDirectives.inCompiledCode()) {
-                throw new Error("Must part of compiled code");
+                throw new Error("Must be part of compiled code");
             }
             return ret;
         }
@@ -568,7 +586,7 @@ public class GraalOSRLockTest extends GraalOSRTestBase {
             }
             GraalDirectives.controlFlowAnchor();
             if (!GraalDirectives.inCompiledCode()) {
-                throw new Error("Must part of compiled code");
+                throw new Error("Must be part of compiled code");
             }
             return ret;
         }
@@ -646,7 +664,7 @@ public class GraalOSRLockTest extends GraalOSRTestBase {
         synchronized (monitor) {
             GraalDirectives.controlFlowAnchor();
             if (!GraalDirectives.inCompiledCode()) {
-                throw new Error("Must part of compiled code");
+                throw new Error("Must be part of compiled code");
             }
         }
         return ret;
@@ -670,7 +688,7 @@ public class GraalOSRLockTest extends GraalOSRTestBase {
             }
             GraalDirectives.controlFlowAnchor();
             if (!GraalDirectives.inCompiledCode()) {
-                throw new Error("Must part of compiled code");
+                throw new Error("Must be part of compiled code");
             }
             return ret;
         }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/GraalOSRTestBase.java

@@ -35,7 +35,9 @@ import org.graalvm.compiler.debug.DebugContext;
 import org.graalvm.compiler.debug.GraalError;
 import org.graalvm.compiler.debug.TTY;
 import org.graalvm.compiler.hotspot.CompilationTask;
+import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
 import org.graalvm.compiler.hotspot.HotSpotGraalCompiler;
+import org.graalvm.compiler.hotspot.HotSpotGraalRuntimeProvider;
 import org.graalvm.compiler.java.BciBlockMapping;
 import org.graalvm.compiler.java.BciBlockMapping.BciBlock;
 import org.graalvm.compiler.nodes.StructuredGraph;
@@ -76,6 +78,13 @@ public abstract class GraalOSRTestBase extends GraalCompilerTest {
         HotSpotCompilationRequest request = new HotSpotCompilationRequest((HotSpotResolvedJavaMethod) method, bci, jvmciEnv);
         HotSpotGraalCompiler compiler = (HotSpotGraalCompiler) runtime.getCompiler();
         CompilationTask task = new CompilationTask(runtime, compiler, request, true, true, debug.getOptions());
+        if (method instanceof HotSpotResolvedJavaMethod) {
+            HotSpotGraalRuntimeProvider graalRuntime = compiler.getGraalRuntime();
+            GraalHotSpotVMConfig config = graalRuntime.getVMConfig();
+            if (((HotSpotResolvedJavaMethod) method).hasCodeAtLevel(bci, config.compilationLevelFullOptimization)) {
+                return;
+            }
+        }
         HotSpotCompilationRequestResult result = task.runCompilation(debug);
         if (result.getFailure() != null) {
             throw new GraalError(result.getFailureMessage());

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/WriteBarrierVerificationTest.java

@@ -37,7 +37,6 @@ import org.graalvm.compiler.hotspot.nodes.SerialArrayRangeWriteBarrier;
 import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier;
 import org.graalvm.compiler.hotspot.phases.WriteBarrierAdditionPhase;
 import org.graalvm.compiler.hotspot.phases.WriteBarrierVerificationPhase;
-import org.graalvm.compiler.hotspot.replacements.arraycopy.UnsafeArrayCopyNode;
 import org.graalvm.compiler.nodes.AbstractBeginNode;
 import org.graalvm.compiler.nodes.AbstractMergeNode;
 import org.graalvm.compiler.nodes.FieldLocationIdentity;
@@ -629,12 +628,6 @@ public class WriteBarrierVerificationTest extends HotSpotGraalCompilerTest {
         System.arraycopy(a, 0, b, 0, a.length);
     }
 
-    @Test
-    public void test61() {
-        GraphPredicate checkForUnsafeArrayCopy = graph -> graph.getNodes().filter(UnsafeArrayCopyNode.class).count() > 0 ? 1 : 0;
-        testPredicate("test13Snippet", checkForUnsafeArrayCopy, new int[]{});
-    }
-
     private interface GraphPredicate {
         int apply(StructuredGraph graph);
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/GraalHotSpotVMConfig.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -280,14 +280,19 @@ public class GraalHotSpotVMConfig extends HotSpotVMConfigAccess {
         }
         if (offset == -1) {
             try {
-                offset = getFieldOffset(name, Integer.class, "OopHandle");
+                offset = getFieldOffset(name, Integer.class, "jobject");
                 isHandle = true;
             } catch (JVMCIError e) {
-
+                try {
+                    // JDK-8186777
+                    offset = getFieldOffset(name, Integer.class, "OopHandle");
+                    isHandle = true;
+                } catch (JVMCIError e2) {
+                }
             }
         }
         if (offset == -1) {
-            throw new JVMCIError("cannot get offset of field " + name + " with type oop or OopHandle");
+            throw new JVMCIError("cannot get offset of field " + name + " with type oop, jobject or OopHandle");
         }
         classMirrorOffset = offset;
         classMirrorIsHandle = isHandle;
@@ -648,6 +653,8 @@ public class GraalHotSpotVMConfig extends HotSpotVMConfigAccess {
     public final long heapEndAddress = getFieldValue("CompilerToVM::Data::_heap_end_addr", Long.class, "HeapWord**");
     public final long heapTopAddress = getFieldValue("CompilerToVM::Data::_heap_top_addr", Long.class, isJDK8 ? "HeapWord**" : "HeapWord* volatile*");
 
+    public final boolean cmsIncrementalMode = getFlag("CMSIncrementalMode", Boolean.class, false);
+
     public final long inlineCacheMissStub = getFieldValue("CompilerToVM::Data::SharedRuntime_ic_miss_stub", Long.class, "address");
     public final long handleWrongMethodStub = getFieldValue("CompilerToVM::Data::SharedRuntime_handle_wrong_method_stub", Long.class, "address");
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotBackend.java

@@ -266,6 +266,8 @@ public abstract class HotSpotBackend extends Backend implements FrameMap.Referen
         unsafeArraycopyStub(HotSpotBackend.UNSAFE_ARRAYCOPY, srcAddr, dstAddr, size);
     }
 
+    public static final ForeignCallDescriptor GENERIC_ARRAYCOPY = new ForeignCallDescriptor("generic_arraycopy", int.class, Word.class, int.class, Word.class, int.class, int.class);
+
     @NodeIntrinsic(ForeignCallNode.class)
     private static native void unsafeArraycopyStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word srcAddr, Word dstAddr, Word size);
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalCompiler.java

@@ -284,7 +284,7 @@ public class HotSpotGraalCompiler implements GraalJVMCICompiler {
 
     public Object mbean() {
         if (graalRuntime instanceof HotSpotGraalRuntime) {
-            return ((HotSpotGraalRuntime)graalRuntime).mbean();
+            return ((HotSpotGraalRuntime) graalRuntime).getMBean();
         }
         return null;
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalCompilerFactory.java

@@ -26,6 +26,8 @@ import static jdk.vm.ci.common.InitTimer.timer;
 import static org.graalvm.compiler.hotspot.HotSpotGraalOptionValues.GRAAL_OPTION_PROPERTY_PREFIX;
 
 import java.io.PrintStream;
+import java.util.Map;
+import java.util.Collections;
 
 import org.graalvm.compiler.debug.MethodFilter;
 import org.graalvm.compiler.options.Option;
@@ -190,4 +192,11 @@ public final class HotSpotGraalCompilerFactory extends HotSpotJVMCICompilerFacto
         }
         return level;
     }
+
+    public Map<String, Object> mbeans() {
+        HotSpotGraalCompiler compiler = createCompiler(HotSpotJVMCIRuntime.runtime());
+        String name = "org.graalvm.compiler.hotspot:type=Options";
+        Object bean = ((HotSpotGraalRuntime) compiler.getGraalRuntime()).getMBean();
+        return Collections.singletonMap(name, bean);
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalMBean.java

@@ -283,11 +283,9 @@ public final class HotSpotGraalMBean implements javax.management.DynamicMBean {
     @Override
     public javax.management.MBeanInfo getMBeanInfo() {
         List<javax.management.MBeanAttributeInfo> attrs = new ArrayList<>();
-        if (registered != null) {
         for (OptionDescriptor descr : allOptionDescriptors()) {
             attrs.add(new javax.management.MBeanAttributeInfo(descr.getName(), descr.getType().getName(), descr.getHelp(), true, true, false));
         }
-        }
         javax.management.MBeanOperationInfo[] ops = {
                         new javax.management.MBeanOperationInfo("dumpMethod", "Enable IGV dumps for provided method", new javax.management.MBeanParameterInfo[]{
                                         new javax.management.MBeanParameterInfo("className", "java.lang.String", "Class to observe"),

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalRuntime.java

@@ -317,7 +317,7 @@ public final class HotSpotGraalRuntime implements HotSpotGraalRuntimeProvider {
         return compilationProblemsPerAction;
     }
 
-    final Object mbean() {
+    Object getMBean() {
         return mBean;
     }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java

@@ -28,8 +28,8 @@ import static org.graalvm.compiler.core.common.GraalOptions.InlineVTableStubs;
 import static org.graalvm.compiler.core.common.GraalOptions.OmitHotExceptionStacktrace;
 import static org.graalvm.compiler.hotspot.meta.HotSpotForeignCallsProviderImpl.OSR_MIGRATION_END;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_KLASS_LOCATION;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_MIRROR_LOCATION;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_MIRROR_HANDLE_LOCATION;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.CLASS_MIRROR_LOCATION;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.COMPRESSED_HUB_LOCATION;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.DISPLACED_MARK_WORD_LOCATION;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HUB_LOCATION;
@@ -41,6 +41,7 @@ import static org.graalvm.word.LocationIdentity.any;
 import java.lang.ref.Reference;
 
 import org.graalvm.compiler.api.directives.GraalDirectives;
+import org.graalvm.compiler.core.common.CompressEncoding;
 import org.graalvm.compiler.core.common.GraalOptions;
 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
 import org.graalvm.compiler.core.common.spi.ForeignCallsProvider;
@@ -90,10 +91,8 @@ import org.graalvm.compiler.hotspot.replacements.UnsafeLoadSnippets;
 import org.graalvm.compiler.hotspot.replacements.WriteBarrierSnippets;
 import org.graalvm.compiler.hotspot.replacements.aot.ResolveConstantSnippets;
 import org.graalvm.compiler.hotspot.replacements.arraycopy.ArrayCopyNode;
-import org.graalvm.compiler.hotspot.replacements.arraycopy.ArrayCopySlowPathNode;
+import org.graalvm.compiler.hotspot.replacements.arraycopy.ArrayCopyWithSlowPathNode;
 import org.graalvm.compiler.hotspot.replacements.arraycopy.ArrayCopySnippets;
-import org.graalvm.compiler.hotspot.replacements.arraycopy.ArrayCopyUnrollNode;
-import org.graalvm.compiler.hotspot.replacements.arraycopy.UnsafeArrayCopySnippets;
 import org.graalvm.compiler.hotspot.replacements.profiling.ProfileSnippets;
 import org.graalvm.compiler.hotspot.word.KlassPointer;
 import org.graalvm.compiler.nodes.AbstractBeginNode;
@@ -193,7 +192,7 @@ public class DefaultHotSpotLoweringProvider extends DefaultJavaLoweringProvider
 
     public DefaultHotSpotLoweringProvider(HotSpotGraalRuntimeProvider runtime, MetaAccessProvider metaAccess, ForeignCallsProvider foreignCalls, HotSpotRegistersProvider registers,
                     HotSpotConstantReflectionProvider constantReflection, TargetDescription target) {
-        super(metaAccess, foreignCalls, target);
+        super(metaAccess, foreignCalls, target, runtime.getVMConfig().useCompressedOops);
         this.runtime = runtime;
         this.registers = registers;
         this.constantReflection = constantReflection;
@@ -216,7 +215,6 @@ public class DefaultHotSpotLoweringProvider extends DefaultJavaLoweringProvider
         hashCodeSnippets = new HashCodeSnippets.Templates(options, factories, providers, target);
         resolveConstantSnippets = new ResolveConstantSnippets.Templates(options, factories, providers, target);
         profileSnippets = new ProfileSnippets.Templates(options, factories, providers, target);
-        providers.getReplacements().registerSnippetTemplateCache(new UnsafeArrayCopySnippets.Templates(options, factories, providers, target));
     }
 
     public MonitorSnippets.Templates getMonitorSnippets() {
@@ -315,10 +313,8 @@ public class DefaultHotSpotLoweringProvider extends DefaultJavaLoweringProvider
             }
         } else if (n instanceof ArrayCopyNode) {
             arraycopySnippets.lower((ArrayCopyNode) n, tool);
-        } else if (n instanceof ArrayCopySlowPathNode) {
-            arraycopySnippets.lower((ArrayCopySlowPathNode) n, tool);
-        } else if (n instanceof ArrayCopyUnrollNode) {
-            arraycopySnippets.lower((ArrayCopyUnrollNode) n, tool);
+        } else if (n instanceof ArrayCopyWithSlowPathNode) {
+            arraycopySnippets.lower((ArrayCopyWithSlowPathNode) n, tool);
         } else if (n instanceof G1PreWriteBarrier) {
             writeBarrierSnippets.lower((G1PreWriteBarrier) n, registers, tool);
         } else if (n instanceof G1PostWriteBarrier) {
@@ -495,20 +491,18 @@ public class DefaultHotSpotLoweringProvider extends DefaultJavaLoweringProvider
         }
     }
 
-    @Override
-    protected Stamp loadStamp(Stamp stamp, JavaKind kind, boolean compressible) {
-        if (kind == JavaKind.Object && compressible && runtime.getVMConfig().useCompressedOops) {
-            return HotSpotNarrowOopStamp.compressed((ObjectStamp) stamp, runtime.getVMConfig().getOopEncoding());
-        }
-        return super.loadStamp(stamp, kind, compressible);
+    private CompressEncoding getOopEncoding() {
+        return runtime.getVMConfig().getOopEncoding();
     }
 
     @Override
-    protected ValueNode implicitLoadConvert(JavaKind kind, ValueNode value, boolean compressible) {
-        if (kind == JavaKind.Object && compressible && runtime.getVMConfig().useCompressedOops) {
-            return new HotSpotCompressionNode(CompressionOp.Uncompress, value, runtime.getVMConfig().getOopEncoding());
+    protected Stamp loadCompressedStamp(ObjectStamp stamp) {
+        return HotSpotNarrowOopStamp.compressed(stamp, getOopEncoding());
     }
-        return super.implicitLoadConvert(kind, value, compressible);
+
+    @Override
+    protected ValueNode newCompressionNode(CompressionOp op, ValueNode value) {
+        return new HotSpotCompressionNode(op, value, getOopEncoding());
     }
 
     @Override
@@ -518,14 +512,6 @@ public class DefaultHotSpotLoweringProvider extends DefaultJavaLoweringProvider
         return ConstantNode.forConstant(base, metaAccess, graph);
     }
 
-    @Override
-    protected ValueNode implicitStoreConvert(JavaKind kind, ValueNode value, boolean compressible) {
-        if (kind == JavaKind.Object && compressible && runtime.getVMConfig().useCompressedOops) {
-            return new HotSpotCompressionNode(CompressionOp.Compress, value, runtime.getVMConfig().getOopEncoding());
-        }
-        return super.implicitStoreConvert(kind, value, compressible);
-    }
-
     @Override
     protected ValueNode createReadArrayComponentHub(StructuredGraph graph, ValueNode arrayHub, FixedNode anchor) {
         /*
@@ -800,4 +786,9 @@ public class DefaultHotSpotLoweringProvider extends DefaultJavaLoweringProvider
     public int arrayLengthOffset() {
         return runtime.getVMConfig().arrayOopDescLengthOffset();
     }
+
+    @Override
+    protected final JavaKind getStorageKind(ResolvedJavaField field) {
+        return field.getJavaKind();
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java

@@ -39,6 +39,7 @@ import java.util.zip.CRC32;
 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
 import org.graalvm.compiler.bytecode.BytecodeProvider;
 import org.graalvm.compiler.core.common.spi.ForeignCallsProvider;
+import org.graalvm.compiler.core.common.type.StampFactory;
 import org.graalvm.compiler.debug.GraalError;
 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
 import org.graalvm.compiler.hotspot.nodes.CurrentJavaThreadNode;
@@ -68,6 +69,7 @@ import org.graalvm.compiler.nodes.NamedLocationIdentity;
 import org.graalvm.compiler.nodes.PiNode;
 import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.calc.AddNode;
+import org.graalvm.compiler.nodes.calc.IntegerConvertNode;
 import org.graalvm.compiler.nodes.calc.LeftShiftNode;
 import org.graalvm.compiler.nodes.graphbuilderconf.ForeignCallPlugin;
 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.Plugins;
@@ -316,8 +318,8 @@ public class HotSpotGraphBuilderPlugins {
     private static boolean readMetaspaceConstantPoolElement(GraphBuilderContext b, ValueNode constantPoolOop, ValueNode index, JavaKind elementKind, WordTypes wordTypes, GraalHotSpotVMConfig config) {
         ValueNode constants = getMetaspaceConstantPool(b, constantPoolOop, wordTypes, config);
         int shift = CodeUtil.log2(wordTypes.getWordKind().getByteCount());
-        ValueNode scaledIndex = b.add(new LeftShiftNode(index, b.add(ConstantNode.forInt(shift))));
-        ValueNode offset = b.add(new AddNode(scaledIndex, b.add(ConstantNode.forInt(config.constantPoolSize))));
+        ValueNode scaledIndex = b.add(new LeftShiftNode(IntegerConvertNode.convert(index, StampFactory.forKind(JavaKind.Long)), b.add(ConstantNode.forInt(shift))));
+        ValueNode offset = b.add(new AddNode(scaledIndex, b.add(ConstantNode.forLong(config.constantPoolSize))));
         AddressNode elementAddress = b.add(new OffsetAddressNode(constants, offset));
         boolean notCompressible = false;
         ValueNode elementValue = WordOperationPlugin.readOp(b, elementKind, elementAddress, NamedLocationIdentity.getArrayLocation(elementKind), BarrierType.NONE, notCompressible);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java

@@ -34,6 +34,7 @@ import static org.graalvm.compiler.hotspot.HotSpotBackend.DECRYPT_WITH_ORIGINAL_
 import static org.graalvm.compiler.hotspot.HotSpotBackend.ENCRYPT;
 import static org.graalvm.compiler.hotspot.HotSpotBackend.ENCRYPT_BLOCK;
 import static org.graalvm.compiler.hotspot.HotSpotBackend.EXCEPTION_HANDLER;
+import static org.graalvm.compiler.hotspot.HotSpotBackend.GENERIC_ARRAYCOPY;
 import static org.graalvm.compiler.hotspot.HotSpotBackend.IC_MISS_HANDLER;
 import static org.graalvm.compiler.hotspot.HotSpotBackend.INITIALIZE_KLASS_BY_SYMBOL;
 import static org.graalvm.compiler.hotspot.HotSpotBackend.INVOCATION_EVENT;
@@ -85,7 +86,6 @@ import static org.graalvm.compiler.hotspot.stubs.NewArrayStub.NEW_ARRAY_C;
 import static org.graalvm.compiler.hotspot.stubs.NewInstanceStub.NEW_INSTANCE_C;
 import static org.graalvm.compiler.hotspot.stubs.StubUtil.VM_MESSAGE_C;
 import static org.graalvm.compiler.hotspot.stubs.UnwindExceptionToCallerStub.EXCEPTION_HANDLER_FOR_RETURN_ADDRESS;
-import static org.graalvm.compiler.nodes.NamedLocationIdentity.any;
 import static org.graalvm.compiler.nodes.java.ForeignCallDescriptors.REGISTER_FINALIZER;
 import static org.graalvm.compiler.replacements.Log.LOG_OBJECT;
 import static org.graalvm.compiler.replacements.Log.LOG_PRIMITIVE;
@@ -97,6 +97,7 @@ import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.Una
 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.LOG10;
 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.SIN;
 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.TAN;
+import static org.graalvm.word.LocationIdentity.any;
 
 import java.util.EnumMap;
 
@@ -213,7 +214,7 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall
         // c_rarg4 - oop ckval (super_klass)
         // return: 0 = success, n = number of copied elements xor'd with -1.
         ForeignCallDescriptor desc = new ForeignCallDescriptor(name, int.class, Word.class, Word.class, Word.class, Word.class, Word.class);
-        LocationIdentity killed = NamedLocationIdentity.getArrayLocation(JavaKind.Object);
+        LocationIdentity killed = NamedLocationIdentity.any();
         registerForeignCall(desc, routine, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, NOT_REEXECUTABLE, killed);
         checkcastArraycopyDescriptors[uninit ? 1 : 0] = desc;
     }
@@ -333,6 +334,7 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall
         registerCheckcastArraycopyDescriptor(true, c.checkcastArraycopyUninit);
         registerCheckcastArraycopyDescriptor(false, c.checkcastArraycopy);
 
+        registerForeignCall(GENERIC_ARRAYCOPY, c.genericArraycopy, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, NOT_REEXECUTABLE, NamedLocationIdentity.any());
         registerForeignCall(UNSAFE_ARRAYCOPY, c.unsafeArraycopy, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, NOT_REEXECUTABLE, NamedLocationIdentity.any());
 
         if (c.useMultiplyToLenIntrinsic()) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/aot/ResolveDynamicConstantNode.java

@@ -28,6 +28,7 @@ import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_16;
 import org.graalvm.word.LocationIdentity;
 import org.graalvm.compiler.core.common.type.Stamp;
 import org.graalvm.compiler.graph.NodeClass;
+import org.graalvm.compiler.nodeinfo.InputType;
 import org.graalvm.compiler.nodeinfo.NodeInfo;
 import org.graalvm.compiler.nodes.DeoptimizingFixedWithNextNode;
 import org.graalvm.compiler.nodes.ValueNode;
@@ -35,7 +36,7 @@ import org.graalvm.compiler.nodes.memory.MemoryCheckpoint;
 import org.graalvm.compiler.nodes.spi.Lowerable;
 import org.graalvm.compiler.nodes.spi.LoweringTool;
 
-@NodeInfo(cycles = CYCLES_4, size = SIZE_16)
+@NodeInfo(cycles = CYCLES_4, size = SIZE_16, allowedUsageTypes = {InputType.Memory})
 public class ResolveDynamicConstantNode extends DeoptimizingFixedWithNextNode implements Lowerable, MemoryCheckpoint.Single {
     public static final NodeClass<ResolveDynamicConstantNode> TYPE = NodeClass.create(ResolveDynamicConstantNode.class);
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/profiling/ProfileBranchNode.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -71,6 +71,15 @@ public class ProfileBranchNode extends ProfileWithNotificationNode {
         return branchCondition != null;
     }
 
+    @Override
+    protected boolean canBeMergedWith(ProfileNode p) {
+        if (p instanceof ProfileBranchNode) {
+            ProfileBranchNode that = (ProfileBranchNode) p;
+            return this.method.equals(that.method) && this.bci == that.bci;
+        }
+        return false;
+    }
+
     /**
      * Gathers all the {@link ProfileBranchNode}s that are inputs to the
      * {@linkplain StructuredGraph#getNodes() live nodes} in a given graph.

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/profiling/ProfileInvokeNode.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -37,6 +37,15 @@ public class ProfileInvokeNode extends ProfileWithNotificationNode {
         super(TYPE, method, freqLog, probabilityLog);
     }
 
+    @Override
+    protected boolean canBeMergedWith(ProfileNode p) {
+        if (p instanceof ProfileInvokeNode) {
+            ProfileInvokeNode that = (ProfileInvokeNode) p;
+            return this.method.equals(that.method);
+        }
+        return false;
+    }
+
     /**
      * Gathers all the {@link ProfileInvokeNode}s that are inputs to the
      * {@linkplain StructuredGraph#getNodes() live nodes} in a given graph.

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/profiling/ProfileNode.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,11 +24,17 @@ package org.graalvm.compiler.hotspot.nodes.profiling;
 
 import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_IGNORED;
 import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_IGNORED;
+import static org.graalvm.compiler.nodes.util.GraphUtil.removeFixedWithUnusedInputs;
 
 import org.graalvm.compiler.core.common.type.StampFactory;
+import org.graalvm.compiler.graph.Node;
 import org.graalvm.compiler.graph.NodeClass;
 import org.graalvm.compiler.graph.iterators.NodeIterable;
+import org.graalvm.compiler.graph.spi.Simplifiable;
+import org.graalvm.compiler.graph.spi.SimplifierTool;
 import org.graalvm.compiler.nodeinfo.NodeInfo;
+import org.graalvm.compiler.nodes.AbstractMergeNode;
+import org.graalvm.compiler.nodes.ControlSplitNode;
 import org.graalvm.compiler.nodes.DeoptimizingFixedWithNextNode;
 import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.nodes.ValueNode;
@@ -41,7 +47,7 @@ import org.graalvm.compiler.options.OptionType;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
 
 @NodeInfo(cycles = CYCLES_IGNORED, cyclesRationale = "profiling should be ignored", size = SIZE_IGNORED, sizeRationale = "profiling should be ignored")
-public class ProfileNode extends DeoptimizingFixedWithNextNode implements Lowerable {
+public abstract class ProfileNode extends DeoptimizingFixedWithNextNode implements Simplifiable, Lowerable {
     public static class Options {
         @Option(help = "Control probabilistic profiling on AMD64", type = OptionType.Expert)//
         public static final OptionKey<Boolean> ProbabilisticProfiling = new OptionKey<>(true);
@@ -54,19 +60,21 @@ public class ProfileNode extends DeoptimizingFixedWithNextNode implements Lowera
     // Only used if ProbabilisticProfiling == true and may be ignored by lowerer.
     @OptionalInput protected ValueNode random;
 
-    // logarithm base 2 of the profile probability
+    // Logarithm base 2 of the profile probability.
     protected int probabilityLog;
 
+    // Step value to add to the profile counter.
+    protected int step;
+
     protected ProfileNode(NodeClass<? extends DeoptimizingFixedWithNextNode> c, ResolvedJavaMethod method, int probabilityLog) {
         super(c, StampFactory.forVoid());
         this.method = method;
         this.probabilityLog = probabilityLog;
+        this.step = 1;
     }
 
     public ProfileNode(ResolvedJavaMethod method, int probabilityLog) {
-        super(TYPE, StampFactory.forVoid());
-        this.method = method;
-        this.probabilityLog = probabilityLog;
+        this(TYPE, method, probabilityLog);
     }
 
     @Override
@@ -92,6 +100,14 @@ public class ProfileNode extends DeoptimizingFixedWithNextNode implements Lowera
         this.random = r;
     }
 
+    public int getStep() {
+        return step;
+    }
+
+    public void setStep(int s) {
+        step = s;
+    }
+
     /**
      * Get the logarithm base 2 of the profile probability.
      */
@@ -106,4 +122,25 @@ public class ProfileNode extends DeoptimizingFixedWithNextNode implements Lowera
     public static NodeIterable<ProfileNode> getProfileNodes(StructuredGraph graph) {
         return graph.getNodes().filter(ProfileNode.class);
     }
+
+    protected abstract boolean canBeMergedWith(ProfileNode p);
+
+    @Override
+    public void simplify(SimplifierTool tool) {
+        for (Node p = predecessor(); p != null; p = p.predecessor()) {
+            // Terminate search when we hit a control split or merge.
+            if (p instanceof ControlSplitNode || p instanceof AbstractMergeNode) {
+                break;
+            }
+            if (p instanceof ProfileNode) {
+                ProfileNode that = (ProfileNode) p;
+                if (this.canBeMergedWith(that)) {
+                    that.setStep(this.getStep() + that.getStep());
+                    removeFixedWithUnusedInputs(this);
+                    tool.addToWorkList(that);
+                    break;
+                }
+            }
+        }
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/profiling/ProfileWithNotificationNode.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,7 +28,7 @@ import org.graalvm.compiler.nodeinfo.NodeInfo;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
 
 @NodeInfo
-public class ProfileWithNotificationNode extends ProfileNode {
+public abstract class ProfileWithNotificationNode extends ProfileNode {
     public static final NodeClass<ProfileWithNotificationNode> TYPE = NodeClass.create(ProfileWithNotificationNode.class);
 
     protected int freqLog;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/OnStackReplacementPhase.java

@@ -22,10 +22,16 @@
  */
 package org.graalvm.compiler.hotspot.phases;
 
+import static jdk.vm.ci.meta.SpeculationLog.SpeculationReason;
 import static org.graalvm.compiler.phases.common.DeadCodeEliminationPhase.Optionality.Required;
 
+import jdk.vm.ci.meta.DeoptimizationAction;
+import jdk.vm.ci.meta.DeoptimizationReason;
+import jdk.vm.ci.meta.JavaConstant;
+import jdk.vm.ci.meta.JavaKind;
 import org.graalvm.compiler.core.common.PermanentBailoutException;
 import org.graalvm.compiler.core.common.cfg.Loop;
+import org.graalvm.compiler.core.common.type.ObjectStamp;
 import org.graalvm.compiler.core.common.type.Stamp;
 import org.graalvm.compiler.debug.CounterKey;
 import org.graalvm.compiler.debug.DebugContext;
@@ -37,13 +43,15 @@ import org.graalvm.compiler.loop.phases.LoopTransformations;
 import org.graalvm.compiler.nodeinfo.InputType;
 import org.graalvm.compiler.nodeinfo.Verbosity;
 import org.graalvm.compiler.nodes.AbstractBeginNode;
-import org.graalvm.compiler.nodes.AbstractLocalNode;
 import org.graalvm.compiler.nodes.EntryMarkerNode;
 import org.graalvm.compiler.nodes.EntryProxyNode;
+import org.graalvm.compiler.nodes.FixedGuardNode;
 import org.graalvm.compiler.nodes.FixedNode;
 import org.graalvm.compiler.nodes.FrameState;
+import org.graalvm.compiler.nodes.LogicNode;
 import org.graalvm.compiler.nodes.LoopBeginNode;
 import org.graalvm.compiler.nodes.ParameterNode;
+import org.graalvm.compiler.nodes.PiNode;
 import org.graalvm.compiler.nodes.StartNode;
 import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.nodes.ValueNode;
@@ -53,6 +61,7 @@ import org.graalvm.compiler.nodes.extended.OSRLockNode;
 import org.graalvm.compiler.nodes.extended.OSRMonitorEnterNode;
 import org.graalvm.compiler.nodes.extended.OSRStartNode;
 import org.graalvm.compiler.nodes.java.AccessMonitorNode;
+import org.graalvm.compiler.nodes.java.InstanceOfNode;
 import org.graalvm.compiler.nodes.java.MonitorEnterNode;
 import org.graalvm.compiler.nodes.java.MonitorExitNode;
 import org.graalvm.compiler.nodes.java.MonitorIdNode;
@@ -172,15 +181,32 @@ public class OnStackReplacementPhase extends Phase {
             if (value instanceof EntryProxyNode) {
                 EntryProxyNode proxy = (EntryProxyNode) value;
                 /*
-                 * we need to drop the stamp since the types we see during OSR may be too precise
-                 * (if a branch was not parsed for example).
+                 * We need to drop the stamp since the types we see during OSR may be too precise
+                 * (if a branch was not parsed for example). In cases when this is possible, we
+                 * insert a guard and narrow the OSRLocal stamp at its usages.
                  */
-                Stamp s = proxy.stamp().unrestricted();
-                AbstractLocalNode osrLocal = null;
+                Stamp narrowedStamp = proxy.value().stamp();
+                Stamp unrestrictedStamp = proxy.stamp().unrestricted();
+                ValueNode osrLocal;
                 if (i >= localsSize) {
-                    osrLocal = graph.addOrUnique(new OSRLockNode(i - localsSize, s));
+                    osrLocal = graph.addOrUnique(new OSRLockNode(i - localsSize, unrestrictedStamp));
                 } else {
-                    osrLocal = graph.addOrUnique(new OSRLocalNode(i, s));
+                    osrLocal = graph.addOrUnique(new OSRLocalNode(i, unrestrictedStamp));
+                }
+                // Speculate on the OSRLocal stamps that could be more precise.
+                OSRLocalSpeculationReason reason = new OSRLocalSpeculationReason(osrState.bci, narrowedStamp, i);
+                if (graph.getSpeculationLog().maySpeculate(reason) && osrLocal instanceof OSRLocalNode && value.getStackKind().equals(JavaKind.Object) && !narrowedStamp.isUnrestricted()) {
+                    // Add guard.
+                    LogicNode check = graph.addOrUniqueWithInputs(InstanceOfNode.createHelper((ObjectStamp) narrowedStamp, osrLocal, null, null));
+                    JavaConstant constant = graph.getSpeculationLog().speculate(reason);
+                    FixedGuardNode guard = graph.add(new FixedGuardNode(check, DeoptimizationReason.OptimizedTypeCheckViolated, DeoptimizationAction.InvalidateRecompile, constant, false));
+                    graph.addAfterFixed(osrStart, guard);
+
+                    // Replace with a more specific type at usages.
+                    // We know that we are at the root,
+                    // so we need to replace the proxy in the state.
+                    proxy.replaceAtMatchingUsages(osrLocal, n -> n == osrState);
+                    osrLocal = graph.addOrUnique(new PiNode(osrLocal, narrowedStamp, guard));
                 }
                 proxy.replaceAndDelete(osrLocal);
             } else {
@@ -268,4 +294,30 @@ public class OnStackReplacementPhase extends Phase {
     public float codeSizeIncrease() {
         return 5.0f;
     }
+
+    private static class OSRLocalSpeculationReason implements SpeculationReason {
+        private int bci;
+        private Stamp speculatedStamp;
+        private int localIndex;
+
+        OSRLocalSpeculationReason(int bci, Stamp speculatedStamp, int localIndex) {
+            this.bci = bci;
+            this.speculatedStamp = speculatedStamp;
+            this.localIndex = localIndex;
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj instanceof OSRLocalSpeculationReason) {
+                OSRLocalSpeculationReason that = (OSRLocalSpeculationReason) obj;
+                return this.bci == that.bci && this.speculatedStamp.equals(that.speculatedStamp) && this.localIndex == that.localIndex;
+            }
+            return false;
+        }
+
+        @Override
+        public int hashCode() {
+            return (bci << 16) ^ speculatedStamp.hashCode() ^ localIndex;
+        }
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotReplacementsUtil.java

@@ -669,6 +669,11 @@ public class HotSpotReplacementsUtil {
         return config.useG1GC;
     }
 
+    @Fold
+    public static boolean useCMSIncrementalMode(@InjectedParameter GraalHotSpotVMConfig config) {
+        return config.cmsIncrementalMode;
+    }
+
     @Fold
     public static boolean useCompressedOops(@InjectedParameter GraalHotSpotVMConfig config) {
         return config.useCompressedOops;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HubGetClassNode.java

@@ -71,7 +71,7 @@ public final class HubGetClassNode extends FloatingNode implements Lowerable, Ca
             return null;
         } else {
             MetaAccessProvider metaAccess = tool.getMetaAccess();
-            if (metaAccess != null && hub.isConstant() && !GraalOptions.ImmutableCode.getValue(graph().getOptions())) {
+            if (metaAccess != null && hub.isConstant() && !GraalOptions.ImmutableCode.getValue(tool.getOptions())) {
                 ResolvedJavaType exactType = tool.getConstantReflection().asJavaType(hub.asConstant());
                 if (exactType != null) {
                     return ConstantNode.forConstant(tool.getConstantReflection().asJavaClass(exactType), metaAccess);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/IdentityHashCodeNode.java

@@ -67,7 +67,7 @@ public class IdentityHashCodeNode extends FixedWithNextNode implements Canonical
         if (object.isConstant()) {
             assert object.stamp() instanceof AbstractObjectStamp;
             JavaConstant c = (JavaConstant) object.asConstant();
-            if (ImmutableCode.getValue(getOptions())) {
+            if (ImmutableCode.getValue(tool.getOptions())) {
                 return this;
             }
             JavaConstant identityHashCode = null;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/arraycopy/ArrayCopySnippets.java

@@ -28,11 +28,12 @@ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayClassElementOffset;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperElementTypePrimitiveInPlace;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHub;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.superCheckOffsetOffset;
-import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
+import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
+import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY;
+import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
 
@@ -47,10 +48,8 @@ import org.graalvm.compiler.debug.DebugHandlersFactory;
 import org.graalvm.compiler.debug.GraalError;
 import org.graalvm.compiler.graph.Node;
 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
-import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
 import org.graalvm.compiler.hotspot.word.KlassPointer;
 import org.graalvm.compiler.nodes.CallTargetNode;
-import org.graalvm.compiler.nodes.ConstantNode;
 import org.graalvm.compiler.nodes.DeoptimizeNode;
 import org.graalvm.compiler.nodes.Invoke;
 import org.graalvm.compiler.nodes.InvokeNode;
@@ -65,8 +64,10 @@ import org.graalvm.compiler.nodes.spi.LoweringTool;
 import org.graalvm.compiler.nodes.type.StampTool;
 import org.graalvm.compiler.nodes.util.GraphUtil;
 import org.graalvm.compiler.options.OptionValues;
+import org.graalvm.compiler.replacements.ReplacementsUtil;
 import org.graalvm.compiler.replacements.SnippetCounter;
 import org.graalvm.compiler.replacements.SnippetCounter.Group;
+import org.graalvm.compiler.replacements.SnippetIntegerHistogram;
 import org.graalvm.compiler.replacements.SnippetTemplate;
 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
@@ -79,16 +80,208 @@ import org.graalvm.word.LocationIdentity;
 import org.graalvm.word.WordFactory;
 
 import jdk.vm.ci.code.TargetDescription;
-import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
 import jdk.vm.ci.meta.DeoptimizationAction;
 import jdk.vm.ci.meta.DeoptimizationReason;
-import jdk.vm.ci.meta.JavaConstant;
 import jdk.vm.ci.meta.JavaKind;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
 import jdk.vm.ci.meta.ResolvedJavaType;
 
 public class ArrayCopySnippets implements Snippets {
 
+    private enum ArrayCopyTypeCheck {
+        UNDEFINED_ARRAY_TYPE_CHECK,
+        // we know that both objects are arrays and have the same type
+        NO_ARRAY_TYPE_CHECK,
+        // can be used when we know that one of the objects is a primitive array
+        HUB_BASED_ARRAY_TYPE_CHECK,
+        // must be used when we don't have sufficient information to use one of the others
+        LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK
+    }
+
+    @Snippet
+    public static void arraycopyZeroLengthSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
+                    @ConstantParameter Counters counters) {
+        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
+        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
+        checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
+        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
+        counters.zeroLengthStaticCounter.inc();
+    }
+
+    @Snippet
+    public static void arraycopyExactSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
+                    @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter,
+                    @ConstantParameter Counters counters) {
+        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
+        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
+        checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
+        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
+        incrementLengthCounter(length, counters);
+
+        elementKindCounter.inc();
+        elementKindCopiedCounter.add(length);
+        ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
+    }
+
+    @Snippet
+    public static void arraycopyUnrolledSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
+                    @ConstantParameter JavaKind elementKind, @ConstantParameter int unrolledLength, @ConstantParameter Counters counters) {
+        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
+        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
+        checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
+        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
+        incrementLengthCounter(length, counters);
+
+        unrolledArraycopyWork(nonNullSrc, srcPos, nonNullDest, destPos, unrolledLength, elementKind);
+    }
+
+    @Snippet
+    public static void arraycopyCheckcastSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
+                    @ConstantParameter Counters counters, @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) {
+        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
+        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
+        checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
+        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
+        incrementLengthCounter(length, counters);
+
+        ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind);
+    }
+
+    @Snippet
+    public static void arraycopyGenericSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, @ConstantParameter Counters counters,
+                    @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) {
+        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
+        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
+        checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
+        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
+        incrementLengthCounter(length, counters);
+
+        ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind);
+    }
+
+    @Snippet
+    public static void arraycopyNativeSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
+        // all checks are done in the native method, so no need to emit additional checks here
+        incrementLengthCounter(length, counters);
+        counters.systemArraycopyCounter.inc();
+        counters.systemArraycopyCopiedCounter.add(length);
+
+        System.arraycopy(src, srcPos, dest, destPos, length);
+    }
+
+    @Fold
+    static LocationIdentity getArrayLocation(JavaKind kind) {
+        return NamedLocationIdentity.getArrayLocation(kind);
+    }
+
+    private static void unrolledArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, JavaKind elementKind) {
+        int scale = arrayIndexScale(elementKind);
+        int arrayBaseOffset = arrayBaseOffset(elementKind);
+        LocationIdentity arrayLocation = getArrayLocation(elementKind);
+
+        long sourceOffset = arrayBaseOffset + (long) srcPos * scale;
+        long destOffset = arrayBaseOffset + (long) destPos * scale;
+        long position = 0;
+        long delta = scale;
+        if (probability(NOT_FREQUENT_PROBABILITY, nonNullSrc == nonNullDest && srcPos < destPos)) {
+            // bad aliased case so we need to copy the array from back to front
+            position = (long) (length - 1) * scale;
+            delta = -delta;
+        }
+
+        // the length was already checked before - we can emit unconditional instructions
+        ExplodeLoopNode.explodeLoop();
+        for (int iteration = 0; iteration < length; iteration++) {
+            Object value = RawLoadNode.load(nonNullSrc, sourceOffset + position, elementKind, arrayLocation);
+            RawStoreNode.storeObject(nonNullDest, destOffset + position, value, elementKind, arrayLocation, false);
+            position += delta;
+        }
+    }
+
+    @Snippet(allowPartialIntrinsicArgumentMismatch = true)
+    public static void checkcastArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
+        if (probability(FREQUENT_PROBABILITY, length > 0)) {
+            Object nonNullSrc = PiNode.asNonNullObject(src);
+            Object nonNullDest = PiNode.asNonNullObject(dest);
+            KlassPointer srcKlass = loadHub(nonNullSrc);
+            KlassPointer destKlass = loadHub(nonNullDest);
+            if (probability(LIKELY_PROBABILITY, srcKlass == destKlass)) {
+                // no storecheck required.
+                counters.objectCheckcastSameTypeCounter.inc();
+                counters.objectCheckcastSameTypeCopiedCounter.add(length);
+                ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
+            } else {
+                KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(INJECTED_VMCONFIG), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION);
+                Word superCheckOffset = WordFactory.signed(destElemKlass.readInt(superCheckOffsetOffset(INJECTED_VMCONFIG), KLASS_SUPER_CHECK_OFFSET_LOCATION));
+
+                counters.objectCheckcastDifferentTypeCounter.inc();
+                counters.objectCheckcastDifferentTypeCopiedCounter.add(length);
+
+                int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false);
+                if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) {
+                    /*
+                     * the stub doesn't throw the ArrayStoreException, but returns the number of
+                     * copied elements (xor'd with -1).
+                     */
+                    copiedElements ^= -1;
+                    System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements);
+                }
+            }
+        }
+    }
+
+    @Snippet(allowPartialIntrinsicArgumentMismatch = true)
+    public static void genericArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
+        if (probability(FREQUENT_PROBABILITY, length > 0)) {
+            counters.genericArraycopyDifferentTypeCounter.inc();
+            counters.genericArraycopyDifferentTypeCopiedCounter.add(length);
+            int copiedElements = GenericArrayCopyCallNode.genericArraycopy(src, srcPos, dest, destPos, length);
+            if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) {
+                /*
+                 * the stub doesn't throw the ArrayStoreException, but returns the number of copied
+                 * elements (xor'd with -1).
+                 */
+                copiedElements ^= -1;
+                System.arraycopy(src, srcPos + copiedElements, dest, destPos + copiedElements, length - copiedElements);
+            }
+        }
+    }
+
+    private static void incrementLengthCounter(int length, Counters counters) {
+        counters.lengthHistogram.inc(length);
+    }
+
+    private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) {
+        if (probability(SLOW_PATH_PROBABILITY, srcPos < 0) ||
+                        probability(SLOW_PATH_PROBABILITY, destPos < 0) ||
+                        probability(SLOW_PATH_PROBABILITY, length < 0) ||
+                        probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length) ||
+                        probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) {
+            counters.checkAIOOBECounter.inc();
+            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
+        }
+        counters.checkSuccessCounter.inc();
+    }
+
+    private static void checkArrayTypes(Object nonNullSrc, Object nonNullDest, ArrayCopyTypeCheck arrayTypeCheck) {
+        if (arrayTypeCheck == ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK) {
+            // nothing to do
+        } else if (arrayTypeCheck == ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK) {
+            KlassPointer srcHub = loadHub(nonNullSrc);
+            KlassPointer destHub = loadHub(nonNullDest);
+            if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) {
+                DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
+            }
+        } else if (arrayTypeCheck == ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK) {
+            KlassPointer srcHub = loadHub(nonNullSrc);
+            KlassPointer destHub = loadHub(nonNullDest);
+            checkArrayType(srcHub);
+            checkArrayType(destHub);
+        } else {
+            ReplacementsUtil.staticAssert(false, "unknown array type check");
+        }
+    }
+
     private static int checkArrayType(KlassPointer nonNullHub) {
         int layoutHelper = readLayoutHelper(nonNullHub);
         if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) {
@@ -97,528 +290,228 @@ public class ArrayCopySnippets implements Snippets {
         return layoutHelper;
     }
 
-    private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) {
-        if (probability(SLOW_PATH_PROBABILITY, srcPos < 0)) {
-            counters.checkAIOOBECounter.inc();
-            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
-        }
-        if (probability(SLOW_PATH_PROBABILITY, destPos < 0)) {
-            counters.checkAIOOBECounter.inc();
-            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
-        }
-        if (probability(SLOW_PATH_PROBABILITY, length < 0)) {
-            counters.checkAIOOBECounter.inc();
-            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
-        }
-        if (probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length)) {
-            counters.checkAIOOBECounter.inc();
-            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
-        }
-        if (probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) {
-            counters.checkAIOOBECounter.inc();
-            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
-        }
-        counters.checkSuccessCounter.inc();
-    }
-
-    @Snippet
-    public static void arraycopyZeroLengthIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
-        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
-        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
-        KlassPointer srcHub = loadHub(nonNullSrc);
-        KlassPointer destHub = loadHub(nonNullDest);
-        checkArrayType(srcHub);
-        checkArrayType(destHub);
-        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
-        counters.zeroLengthStaticCounter.inc();
-    }
-
-    @Snippet
-    public static void arraycopyExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter counter,
-                    @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) {
-        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
-        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
-        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
-        counter.inc();
-        copiedCounter.add(length);
-        ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
-        if (length == 0) {
-            counters.zeroLengthDynamicCounter.inc();
-        } else {
-            counters.nonZeroLengthDynamicCounter.inc();
-            counters.nonZeroLengthDynamicCopiedCounter.add(length);
-        }
-    }
-
-    /**
-     * This intrinsic is useful for the case where we know something statically about one of the
-     * inputs but not the other.
-     */
-    @Snippet
-    public static void arraycopyPredictedExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind,
-                    @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) {
-        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
-        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
-        KlassPointer srcHub = loadHub(nonNullSrc);
-        KlassPointer destHub = loadHub(nonNullDest);
-        if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) {
-            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
-        }
-        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
-        counter.inc();
-        copiedCounter.add(length);
-        ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
-        if (length == 0) {
-            counters.zeroLengthDynamicCounter.inc();
-        } else {
-            counters.nonZeroLengthDynamicCounter.inc();
-            counters.nonZeroLengthDynamicCopiedCounter.add(length);
-        }
-    }
-
-    @Snippet
-    public static void arraycopyPredictedObjectWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, KlassPointer objectArrayKlass,
-                    @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) {
-        if (length > 0) {
-            KlassPointer srcHub = loadHub(PiNode.asNonNullObject(nonNullSrc));
-            KlassPointer destHub = loadHub(PiNode.asNonNullObject(nonNullDest));
-            if (probability(FAST_PATH_PROBABILITY, srcHub == destHub || destHub == objectArrayKlass)) {
-                counter.inc();
-                copiedCounter.add(length);
-                counters.predictedObjectArrayCopyFastPathCounter.inc();
-                counters.predictedObjectArrayCopyFastPathCopiedCounter.add(length);
-                ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
-            } else {
-                counters.predictedObjectArrayCopySlowPathCounter.inc();
-                counters.predictedObjectArrayCopySlowPathCopiedCounter.add(length);
-                System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
-            }
-        }
-    }
-
-    /**
-     * This is the basic template for the full arraycopy checks, including a check that the
-     * underlying type is really an array type.
-     */
-    @Snippet
-    public static void arraycopySlowPathIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, KlassPointer predictedKlass, @ConstantParameter JavaKind elementKind,
-                    @ConstantParameter SnippetInfo slowPath, @ConstantParameter Object slowPathArgument, @ConstantParameter Counters counters) {
-        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
-        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
-        KlassPointer srcHub = loadHub(nonNullSrc);
-        KlassPointer destHub = loadHub(nonNullDest);
-        checkArrayType(srcHub);
-        checkArrayType(destHub);
-        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
-        if (length == 0) {
-            counters.zeroLengthDynamicCounter.inc();
-        } else {
-            counters.nonZeroLengthDynamicCounter.inc();
-            counters.nonZeroLengthDynamicCopiedCounter.add(length);
-        }
-        ArrayCopySlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, predictedKlass, elementKind, slowPath, slowPathArgument);
-    }
-
-    /**
-     * Snippet for unrolled arraycopy.
-     */
-    @Snippet
-    public static void arraycopyUnrolledIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter int unrolledLength, @ConstantParameter JavaKind elementKind,
-                    @ConstantParameter Counters counters) {
-        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
-        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
-        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
-        if (length == 0) {
-            counters.zeroLengthDynamicCounter.inc();
-        } else {
-            counters.nonZeroLengthDynamicCounter.inc();
-            counters.nonZeroLengthDynamicCopiedCounter.add(length);
-        }
-        ArrayCopyUnrollNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, unrolledLength, elementKind);
-    }
-
-    @Snippet
-    public static void checkcastArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, @ConstantParameter Counters counters) {
-        if (length > 0) {
-            KlassPointer destKlass = loadHub(nonNullDest);
-            KlassPointer srcKlass = loadHub(nonNullSrc);
-            if (probability(SLOW_PATH_PROBABILITY, srcKlass == destKlass)) {
-                // no storecheck required.
-                counters.objectCheckcastSameTypeCounter.inc();
-                counters.objectCheckcastSameTypeCopiedCounter.add(length);
-                ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
-            } else {
-                KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(INJECTED_VMCONFIG), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION);
-                Word superCheckOffset = WordFactory.signed(destElemKlass.readInt(superCheckOffsetOffset(INJECTED_VMCONFIG), KLASS_SUPER_CHECK_OFFSET_LOCATION));
-                counters.objectCheckcastCounter.inc();
-                counters.objectCheckcastCopiedCounter.add(length);
-                int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false);
-                if (copiedElements != 0) {
-                    /*
-                     * the checkcast stub doesn't throw the ArrayStoreException, but returns the
-                     * number of copied elements (xor'd with -1).
-                     */
-                    copiedElements ^= -1;
-                    System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements);
-                }
-            }
-        }
-    }
-
-    @Snippet
-    public static void arraycopyGeneric(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
-        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
-        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
-        KlassPointer srcHub = loadHub(nonNullSrc);
-        KlassPointer destHub = loadHub(nonNullDest);
-        if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, nonNullSrc != nonNullDest)) {
-            int layoutHelper = checkArrayType(srcHub);
-            final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace(INJECTED_VMCONFIG)) == 0);
-            checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
-            if (probability(FAST_PATH_PROBABILITY, isObjectArray)) {
-                counters.genericObjectExactCallCounter.inc();
-                counters.genericObjectExactCallCopiedCounter.add(length);
-                ArrayCopyCallNode.disjointArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, JavaKind.Object);
-            } else {
-                counters.genericPrimitiveCallCounter.inc();
-                counters.genericPrimitiveCallCopiedCounter.add(length);
-                UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper);
-            }
-        } else {
-            counters.systemArraycopyCounter.inc();
-            counters.systemArraycopyCopiedCounter.add(length);
-            System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
-        }
-    }
-
-    @Fold
-    static LocationIdentity getArrayLocation(JavaKind kind) {
-        return NamedLocationIdentity.getArrayLocation(kind);
-    }
-
-    @Snippet
-    public static void arraycopyUnrolledWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, @ConstantParameter int length, @ConstantParameter JavaKind elementKind) {
-        final int scale = arrayIndexScale(elementKind);
-        int arrayBaseOffset = arrayBaseOffset(elementKind);
-        LocationIdentity arrayLocation = getArrayLocation(elementKind);
-        if (nonNullSrc == nonNullDest && srcPos < destPos) { // bad aliased case
-            long start = (long) (length - 1) * scale;
-            long i = start;
-            ExplodeLoopNode.explodeLoop();
-            for (int iteration = 0; iteration < length; iteration++) {
-                if (i >= 0) {
-                    Object a = RawLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation);
-                    RawStoreNode.storeObject(nonNullDest, arrayBaseOffset + i + (long) destPos * scale, a, elementKind, arrayLocation, false);
-                    i -= scale;
-                }
-            }
-        } else {
-            long end = (long) length * scale;
-            long i = 0;
-            ExplodeLoopNode.explodeLoop();
-            for (int iteration = 0; iteration < length; iteration++) {
-                if (i < end) {
-                    Object a = RawLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation);
-                    RawStoreNode.storeObject(nonNullDest, arrayBaseOffset + i + (long) destPos * scale, a, elementKind, arrayLocation, false);
-                    i += scale;
-                }
-            }
-        }
-    }
-
     static class Counters {
         final SnippetCounter checkSuccessCounter;
         final SnippetCounter checkAIOOBECounter;
 
-        final SnippetCounter objectCheckcastCounter;
-        final SnippetCounter objectCheckcastSameTypeCounter;
-        final SnippetCounter predictedObjectArrayCopySlowPathCounter;
-        final SnippetCounter predictedObjectArrayCopyFastPathCounter;
-
-        final SnippetCounter genericPrimitiveCallCounter;
-        final SnippetCounter genericObjectExactCallCounter;
-        final SnippetCounter systemArraycopyCounter;
-
         final SnippetCounter zeroLengthStaticCounter;
-        final SnippetCounter zeroLengthDynamicCounter;
-        final SnippetCounter nonZeroLengthDynamicCounter;
+        final SnippetIntegerHistogram lengthHistogram;
 
-        final SnippetCounter nonZeroLengthDynamicCopiedCounter;
-        final SnippetCounter genericPrimitiveCallCopiedCounter;
-        final SnippetCounter genericObjectExactCallCopiedCounter;
+        final SnippetCounter systemArraycopyCounter;
         final SnippetCounter systemArraycopyCopiedCounter;
 
-        final SnippetCounter objectCheckcastCopiedCounter;
+        final SnippetCounter genericArraycopyDifferentTypeCopiedCounter;
+        final SnippetCounter genericArraycopyDifferentTypeCounter;
+
         final SnippetCounter objectCheckcastSameTypeCopiedCounter;
-        final SnippetCounter predictedObjectArrayCopySlowPathCopiedCounter;
-        final SnippetCounter predictedObjectArrayCopyFastPathCopiedCounter;
+        final SnippetCounter objectCheckcastSameTypeCounter;
+        final SnippetCounter objectCheckcastDifferentTypeCopiedCounter;
+        final SnippetCounter objectCheckcastDifferentTypeCounter;
 
         final EnumMap<JavaKind, SnippetCounter> arraycopyCallCounters = new EnumMap<>(JavaKind.class);
-        final EnumMap<JavaKind, SnippetCounter> arraycopyCounters = new EnumMap<>(JavaKind.class);
-
         final EnumMap<JavaKind, SnippetCounter> arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class);
-        final EnumMap<JavaKind, SnippetCounter> arraycopyCopiedCounters = new EnumMap<>(JavaKind.class);
 
         Counters(SnippetCounter.Group.Factory factory) {
             final Group checkCounters = factory.createSnippetCounterGroup("System.arraycopy checkInputs");
-            final Group counters = factory.createSnippetCounterGroup("System.arraycopy");
-            final Group copiedCounters = factory.createSnippetCounterGroup("System.arraycopy copied elements");
-            final Group lengthCounters = factory.createSnippetCounterGroup("System.arraycopy 0-length checks");
+            final Group callCounters = factory.createSnippetCounterGroup("System.arraycopy calls");
+            final Group copiedElementsCounters = factory.createSnippetCounterGroup("System.arraycopy copied elements");
+            final Group lengthCounters = factory.createSnippetCounterGroup("System.arraycopy with 0-length");
 
             checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess");
             checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE");
 
-            objectCheckcastCounter = new SnippetCounter(counters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays");
-            objectCheckcastSameTypeCounter = new SnippetCounter(counters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays");
-            predictedObjectArrayCopySlowPathCounter = new SnippetCounter(counters, "Object[]{slow-path}", "used System.arraycopy slow path for predicted Object[] arrays");
-            predictedObjectArrayCopyFastPathCounter = new SnippetCounter(counters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays");
-            genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays");
-            genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays");
-            systemArraycopyCounter = new SnippetCounter(counters, "genericObject", "call to System.arraycopy");
+            zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-length copy static", "calls where the length is statically 0");
+            lengthHistogram = new SnippetIntegerHistogram(lengthCounters, 2, "length", "length");
 
-            zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-lengthcopy static", "arraycopy where the length is statically 0");
-            zeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "0-lengthcopy dynamically", "arraycopy where the length is dynamically 0");
-            nonZeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero");
+            systemArraycopyCounter = new SnippetCounter(callCounters, "native System.arraycopy", "JNI-based System.arraycopy call");
+            systemArraycopyCopiedCounter = new SnippetCounter(copiedElementsCounters, "native System.arraycopy", "JNI-based System.arraycopy call");
 
-            nonZeroLengthDynamicCopiedCounter = new SnippetCounter(copiedCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero");
-            genericPrimitiveCallCopiedCounter = new SnippetCounter(copiedCounters, "genericPrimitive", "generic arraycopy snippet for primitive arrays");
-            genericObjectExactCallCopiedCounter = new SnippetCounter(copiedCounters, "genericObjectExact", "generic arraycopy snippet for special object arrays");
-            systemArraycopyCopiedCounter = new SnippetCounter(copiedCounters, "genericObject", "call to System.arraycopy");
+            genericArraycopyDifferentTypeCounter = new SnippetCounter(callCounters, "generic[] stub", "generic arraycopy stub");
+            genericArraycopyDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "generic[] stub", "generic arraycopy stub");
 
-            objectCheckcastCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays");
-            objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays");
-            predictedObjectArrayCopySlowPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{slow-path}",
-                            "used System.arraycopy slow path for predicted Object[] arrays");
-            predictedObjectArrayCopyFastPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays");
-            createArraycopyCounter(JavaKind.Byte, counters, copiedCounters);
-            createArraycopyCounter(JavaKind.Boolean, counters, copiedCounters);
-            createArraycopyCounter(JavaKind.Char, counters, copiedCounters);
-            createArraycopyCounter(JavaKind.Short, counters, copiedCounters);
-            createArraycopyCounter(JavaKind.Int, counters, copiedCounters);
-            createArraycopyCounter(JavaKind.Long, counters, copiedCounters);
-            createArraycopyCounter(JavaKind.Float, counters, copiedCounters);
-            createArraycopyCounter(JavaKind.Double, counters, copiedCounters);
-            createArraycopyCounter(JavaKind.Object, counters, copiedCounters);
+            objectCheckcastSameTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays");
+            objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays");
+            objectCheckcastDifferentTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check");
+            objectCheckcastDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check");
+
+            createArraycopyCounter(JavaKind.Byte, callCounters, copiedElementsCounters);
+            createArraycopyCounter(JavaKind.Boolean, callCounters, copiedElementsCounters);
+            createArraycopyCounter(JavaKind.Char, callCounters, copiedElementsCounters);
+            createArraycopyCounter(JavaKind.Short, callCounters, copiedElementsCounters);
+            createArraycopyCounter(JavaKind.Int, callCounters, copiedElementsCounters);
+            createArraycopyCounter(JavaKind.Long, callCounters, copiedElementsCounters);
+            createArraycopyCounter(JavaKind.Float, callCounters, copiedElementsCounters);
+            createArraycopyCounter(JavaKind.Double, callCounters, copiedElementsCounters);
+            createArraycopyCounter(JavaKind.Object, callCounters, copiedElementsCounters);
         }
 
         void createArraycopyCounter(JavaKind kind, Group counters, Group copiedCounters) {
-            arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
-            arraycopyCounters.put(kind, new SnippetCounter(counters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
-
-            arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
-            arraycopyCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
+            arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays"));
+            arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays"));
         }
     }
 
     public static class Templates extends SnippetTemplate.AbstractTemplates {
+        private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGenericSnippet");
+        private final SnippetInfo arraycopyUnrolledSnippet = snippet("arraycopyUnrolledSnippet");
+        private final SnippetInfo arraycopyExactSnippet = snippet("arraycopyExactSnippet");
+        private final SnippetInfo arraycopyZeroLengthSnippet = snippet("arraycopyZeroLengthSnippet");
+        private final SnippetInfo arraycopyCheckcastSnippet = snippet("arraycopyCheckcastSnippet");
+        private final SnippetInfo arraycopyNativeSnippet = snippet("arraycopyNativeSnippet");
+
+        private final SnippetInfo checkcastArraycopyWithSlowPathWork = snippet("checkcastArraycopyWithSlowPathWork");
+        private final SnippetInfo genericArraycopyWithSlowPathWork = snippet("genericArraycopyWithSlowPathWork");
+
+        private ResolvedJavaMethod originalArraycopy;
+        private final Counters counters;
 
         public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target) {
             super(options, factories, providers, providers.getSnippetReflection(), target);
             this.counters = new Counters(factory);
         }
 
-        private ResolvedJavaMethod originalArraycopy() throws GraalError {
-            if (originalArraycopy == null) {
-                Method method;
-                try {
-                    method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class);
-                } catch (NoSuchMethodException | SecurityException e) {
-                    throw new GraalError(e);
-                }
-                originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method);
-            }
-            return originalArraycopy;
-        }
-
-        private ResolvedJavaMethod originalArraycopy;
-
-        private final SnippetInfo checkcastArraycopyWorkSnippet = snippet("checkcastArraycopyWork");
-        private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGeneric");
-
-        private final SnippetInfo arraycopySlowPathIntrinsicSnippet = snippet("arraycopySlowPathIntrinsic");
-        private final SnippetInfo arraycopyUnrolledIntrinsicSnippet = snippet("arraycopyUnrolledIntrinsic");
-        private final SnippetInfo arraycopyExactIntrinsicSnippet = snippet("arraycopyExactIntrinsic");
-        private final SnippetInfo arraycopyZeroLengthIntrinsicSnippet = snippet("arraycopyZeroLengthIntrinsic");
-        private final SnippetInfo arraycopyPredictedExactIntrinsicSnippet = snippet("arraycopyPredictedExactIntrinsic");
-        private final SnippetInfo arraycopyPredictedObjectWorkSnippet = snippet("arraycopyPredictedObjectWork");
-
-        private final SnippetInfo arraycopyUnrolledWorkSnippet = snippet("arraycopyUnrolledWork");
-
-        private final Counters counters;
-
         protected SnippetInfo snippet(String methodName) {
             SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, LocationIdentity.any());
             info.setOriginalMethod(originalArraycopy());
             return info;
         }
 
-        public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) {
-            return selectComponentKind(arraycopy, true);
-        }
-
-        public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy, boolean exact) {
-            ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
-            ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
-
-            if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) {
-                if (!exact) {
-                    JavaKind component = getComponentKind(srcType);
-                    if (component != null) {
-                        return component;
-                    }
-                    return getComponentKind(destType);
-                }
-                return null;
-            }
-            if (exact) {
-                if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) {
-                    return null;
-                }
-                if (!arraycopy.isExact()) {
-                    return null;
-                }
-            }
-            return srcType.getComponentType().getJavaKind();
-        }
-
-        private static JavaKind getComponentKind(ResolvedJavaType type) {
-            if (type != null && type.isArray()) {
-                return type.getComponentType().getJavaKind();
-            }
-            return null;
-        }
-
-        private static boolean shouldUnroll(ValueNode length) {
-            return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0;
-        }
-
         public void lower(ArrayCopyNode arraycopy, LoweringTool tool) {
-            JavaKind componentKind = selectComponentKind(arraycopy);
-            SnippetInfo snippetInfo = null;
-            SnippetInfo slowPathSnippetInfo = null;
-            Object slowPathArgument = null;
+            JavaKind elementKind = selectComponentKind(arraycopy);
+            SnippetInfo snippetInfo;
+            ArrayCopyTypeCheck arrayTypeCheck;
 
-            if (arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) {
-                snippetInfo = arraycopyZeroLengthIntrinsicSnippet;
-            } else if (arraycopy.isExact()) {
-                snippetInfo = arraycopyExactIntrinsicSnippet;
-                if (shouldUnroll(arraycopy.getLength())) {
-                    snippetInfo = arraycopyUnrolledIntrinsicSnippet;
-                }
-            } else {
-                if (componentKind == JavaKind.Object) {
             ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
             ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
+            if (!canBeArray(srcType) || !canBeArray(destType)) {
+                // at least one of the objects is definitely not an array - use the native call
+                // right away as the copying will fail anyways
+                snippetInfo = arraycopyNativeSnippet;
+                arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
+            } else {
                 ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType();
                 ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType();
-                    if (srcComponentType != null && destComponentType != null && !srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) {
-                        snippetInfo = arraycopySlowPathIntrinsicSnippet;
-                        slowPathSnippetInfo = checkcastArraycopyWorkSnippet;
-                        slowPathArgument = LocationIdentity.any();
-                        /*
-                         * Because this snippet has to use Sysytem.arraycopy as a slow path, we must
-                         * pretend to kill any() so clear the componentKind.
-                         */
-                        componentKind = null;
-                    }
-                }
-                if (componentKind == null && snippetInfo == null) {
-                    JavaKind predictedKind = selectComponentKind(arraycopy, false);
-                    if (predictedKind != null) {
-                        /*
-                         * At least one array is of a known type requiring no store checks, so
-                         * assume the other is of the same type. Generally this is working around
-                         * deficiencies in our propagation of type information.
-                         */
-                        componentKind = predictedKind;
-                        if (predictedKind == JavaKind.Object) {
-                            snippetInfo = arraycopySlowPathIntrinsicSnippet;
-                            slowPathSnippetInfo = arraycopyPredictedObjectWorkSnippet;
-                            slowPathArgument = predictedKind;
-                            componentKind = null;
-                        } else {
-                            snippetInfo = arraycopyPredictedExactIntrinsicSnippet;
-                        }
-                    }
-                }
-                if (snippetInfo == null) {
+
+                if (arraycopy.isExact()) {
+                    // there is a sufficient type match - we don't need any additional type checks
+                    snippetInfo = arraycopyExactSnippet;
+                    arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK;
+                } else if (srcComponentType == null && destComponentType == null) {
+                    // we don't know anything about the types - use the generic copying
                     snippetInfo = arraycopyGenericSnippet;
+                    arrayTypeCheck = ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK;
+                } else if (srcComponentType != null && destComponentType != null) {
+                    if (!srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) {
+                        // it depends on the array content if the copy succeeds - we need
+                        // a type check for every store
+                        snippetInfo = arraycopyCheckcastSnippet;
+                        arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK;
+                    } else {
+                        // one object is an object array, the other one is a primitive array.
+                        // this copy will always fail - use the native call right away
+                        assert !srcComponentType.equals(destComponentType) : "must be handled by arraycopy.isExact()";
+                        snippetInfo = arraycopyNativeSnippet;
+                        arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
+                    }
+                } else {
+                    ResolvedJavaType nonNullComponentType = srcComponentType != null ? srcComponentType : destComponentType;
+                    if (nonNullComponentType.isPrimitive()) {
+                        // one involved object is a primitive array - we can safely assume that we
+                        // are copying primitive arrays
+                        snippetInfo = arraycopyExactSnippet;
+                        arrayTypeCheck = ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK;
+                        elementKind = nonNullComponentType.getJavaKind();
+                    } else {
+                        // one involved object is an object array - we can safely assume that we are
+                        // copying object arrays that might require a store check
+                        snippetInfo = arraycopyCheckcastSnippet;
+                        arrayTypeCheck = ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK;
                     }
                 }
+            }
+
+            // a few special cases that are easier to handle when all other variables already have a
+            // value
+            if (arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) {
+                snippetInfo = arraycopyZeroLengthSnippet;
+            } else if (snippetInfo == arraycopyExactSnippet && shouldUnroll(arraycopy.getLength())) {
+                snippetInfo = arraycopyUnrolledSnippet;
+            }
+
+            // create the snippet
             Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage());
             args.add("src", arraycopy.getSource());
             args.add("srcPos", arraycopy.getSourcePosition());
             args.add("dest", arraycopy.getDestination());
             args.add("destPos", arraycopy.getDestinationPosition());
             args.add("length", arraycopy.getLength());
-            if (snippetInfo == arraycopyUnrolledIntrinsicSnippet) {
-                args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt());
-                args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal);
-            } else if (snippetInfo == arraycopySlowPathIntrinsicSnippet) {
-                ValueNode predictedKlass = null;
-                if (slowPathArgument == arraycopyPredictedObjectWorkSnippet) {
-                    HotSpotResolvedObjectType arrayClass = (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(Object[].class);
-                    predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayClass.klass(), tool.getMetaAccess(), arraycopy.graph());
-                } else {
-                    predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassAlwaysNull(), JavaConstant.NULL_POINTER, tool.getMetaAccess(), arraycopy.graph());
+            if (snippetInfo != arraycopyNativeSnippet) {
+                assert arrayTypeCheck != ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
+                args.addConst("arrayTypeCheck", arrayTypeCheck);
             }
-                args.add("predictedKlass", predictedKlass);
-                args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal);
-                args.addConst("slowPath", slowPathSnippetInfo);
-                assert slowPathArgument != null;
-                args.addConst("slowPathArgument", slowPathArgument);
-            } else if (snippetInfo == arraycopyExactIntrinsicSnippet || snippetInfo == arraycopyPredictedExactIntrinsicSnippet) {
-                assert componentKind != null;
-                args.addConst("elementKind", componentKind);
-                args.addConst("counter", counters.arraycopyCallCounters.get(componentKind));
-                args.addConst("copiedCounter", counters.arraycopyCallCopiedCounters.get(componentKind));
+            if (snippetInfo == arraycopyUnrolledSnippet) {
+                args.addConst("elementKind", elementKind != null ? elementKind : JavaKind.Illegal);
+                args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt());
+            }
+            if (snippetInfo == arraycopyExactSnippet) {
+                assert elementKind != null;
+                args.addConst("elementKind", elementKind);
+                args.addConst("elementKindCounter", counters.arraycopyCallCounters.get(elementKind));
+                args.addConst("elementKindCopiedCounter", counters.arraycopyCallCopiedCounters.get(elementKind));
             }
             args.addConst("counters", counters);
+            if (snippetInfo == arraycopyCheckcastSnippet) {
+                args.addConst("workSnippet", checkcastArraycopyWithSlowPathWork);
+                args.addConst("elementKind", JavaKind.Illegal);
+            }
+            if (snippetInfo == arraycopyGenericSnippet) {
+                args.addConst("workSnippet", genericArraycopyWithSlowPathWork);
+                args.addConst("elementKind", JavaKind.Illegal);
+            }
+
             instantiate(args, arraycopy);
         }
 
-        public void lower(ArrayCopySlowPathNode arraycopy, LoweringTool tool) {
+        public void lower(ArrayCopyWithSlowPathNode arraycopy, LoweringTool tool) {
             StructuredGraph graph = arraycopy.graph();
             if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
-                // Can't be lowered yet
+                // if an arraycopy contains a slow path, we can't lower it right away
                 return;
             }
+
             SnippetInfo snippetInfo = arraycopy.getSnippet();
             Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
-            args.add("nonNullSrc", arraycopy.getSource());
+            args.add("src", arraycopy.getSource());
             args.add("srcPos", arraycopy.getSourcePosition());
-            args.add("nonNullDest", arraycopy.getDestination());
+            args.add("dest", arraycopy.getDestination());
             args.add("destPos", arraycopy.getDestinationPosition());
-            if (snippetInfo == arraycopyUnrolledWorkSnippet) {
-                args.addConst("length", ((Integer) arraycopy.getArgument()).intValue());
-                args.addConst("elementKind", arraycopy.getElementKind());
-            } else {
             args.add("length", arraycopy.getLength());
-            }
-            if (snippetInfo == arraycopyPredictedObjectWorkSnippet) {
-                args.add("objectArrayKlass", arraycopy.getPredictedKlass());
-                args.addConst("counter", counters.arraycopyCallCounters.get(JavaKind.Object));
-                args.addConst("copiedCounter", counters.arraycopyCallCopiedCounters.get(JavaKind.Object));
             args.addConst("counters", counters);
-            }
             instantiate(args, arraycopy);
         }
 
-        public void lower(ArrayCopyUnrollNode arraycopy, LoweringTool tool) {
-            StructuredGraph graph = arraycopy.graph();
-            if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
-                // Can't be lowered yet
-                return;
+        private static boolean canBeArray(ResolvedJavaType type) {
+            return type == null || type.isJavaLangObject() || type.isArray();
         }
-            SnippetInfo snippetInfo = arraycopyUnrolledWorkSnippet;
-            Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
-            args.add("nonNullSrc", arraycopy.getSource());
-            args.add("srcPos", arraycopy.getSourcePosition());
-            args.add("nonNullDest", arraycopy.getDestination());
-            args.add("destPos", arraycopy.getDestinationPosition());
-            args.addConst("length", arraycopy.getUnrollLength());
-            args.addConst("elementKind", arraycopy.getElementKind());
-            template(graph.getDebug(), args).instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args);
+
+        public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) {
+            ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
+            ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
+
+            if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) {
+                return null;
+            }
+            if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) {
+                return null;
+            }
+            if (!arraycopy.isExact()) {
+                return null;
+            }
+            return srcType.getComponentType().getJavaKind();
+        }
+
+        private static boolean shouldUnroll(ValueNode length) {
+            return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0;
         }
 
         /**
@@ -650,8 +543,8 @@ public class ArrayCopySnippets implements Snippets {
                         newInvoke.setStateAfter(arraycopy.stateAfter());
                     }
                     graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke);
-                } else if (originalNode instanceof ArrayCopySlowPathNode) {
-                    ArrayCopySlowPathNode slowPath = (ArrayCopySlowPathNode) replacements.get(originalNode);
+                } else if (originalNode instanceof ArrayCopyWithSlowPathNode) {
+                    ArrayCopyWithSlowPathNode slowPath = (ArrayCopyWithSlowPathNode) replacements.get(originalNode);
                     assert arraycopy.stateAfter() != null : arraycopy;
                     assert slowPath.stateAfter() == arraycopy.stateAfter();
                     slowPath.setBci(arraycopy.getBci());
@@ -659,5 +552,18 @@ public class ArrayCopySnippets implements Snippets {
             }
             GraphUtil.killCFG(arraycopy);
         }
+
+        private ResolvedJavaMethod originalArraycopy() throws GraalError {
+            if (originalArraycopy == null) {
+                Method method;
+                try {
+                    method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class);
+                } catch (NoSuchMethodException | SecurityException e) {
+                    throw new GraalError(e);
+                }
+                originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method);
+            }
+            return originalArraycopy;
+        }
     }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/arraycopy/ArrayCopyUnrollNode.java

@@ -1,148 +0,0 @@
-/*
- * Copyright (c) 2015, 2016, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-package org.graalvm.compiler.hotspot.replacements.arraycopy;
-
-import jdk.vm.ci.meta.JavaKind;
-
-import static org.graalvm.word.LocationIdentity.any;
-
-import org.graalvm.compiler.core.common.type.StampFactory;
-import org.graalvm.compiler.graph.NodeClass;
-import org.graalvm.compiler.nodeinfo.InputType;
-import org.graalvm.compiler.nodeinfo.NodeInfo;
-import org.graalvm.compiler.nodes.NamedLocationIdentity;
-import org.graalvm.compiler.nodes.ValueNode;
-import org.graalvm.compiler.nodes.extended.ArrayRangeWriteNode;
-import org.graalvm.compiler.nodes.memory.MemoryAccess;
-import org.graalvm.compiler.nodes.memory.MemoryCheckpoint;
-import org.graalvm.compiler.nodes.memory.MemoryNode;
-import org.graalvm.compiler.nodes.spi.Lowerable;
-import org.graalvm.compiler.nodes.spi.LoweringTool;
-import org.graalvm.word.LocationIdentity;
-
-@NodeInfo(allowedUsageTypes = InputType.Memory)
-public class ArrayCopyUnrollNode extends ArrayRangeWriteNode implements MemoryCheckpoint.Single, Lowerable, MemoryAccess {
-
-    public static final NodeClass<ArrayCopyUnrollNode> TYPE = NodeClass.create(ArrayCopyUnrollNode.class);
-
-    @Input protected ValueNode src;
-    @Input protected ValueNode srcPos;
-    @Input protected ValueNode dest;
-    @Input protected ValueNode destPos;
-    @Input protected ValueNode length;
-
-    private JavaKind elementKind;
-
-    private int unrolledLength;
-
-    @OptionalInput(InputType.Memory) private MemoryNode lastLocationAccess;
-
-    public ArrayCopyUnrollNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, int unrolledLength, JavaKind elementKind) {
-        super(TYPE, StampFactory.forKind(JavaKind.Void));
-        this.src = src;
-        this.srcPos = srcPos;
-        this.dest = dest;
-        this.destPos = destPos;
-        this.length = length;
-        this.unrolledLength = unrolledLength;
-        assert elementKind != null && elementKind != JavaKind.Illegal;
-        this.elementKind = elementKind;
-    }
-
-    public ValueNode getSource() {
-        return src;
-    }
-
-    public ValueNode getSourcePosition() {
-        return srcPos;
-    }
-
-    public ValueNode getDestination() {
-        return dest;
-    }
-
-    public ValueNode getDestinationPosition() {
-        return destPos;
-    }
-
-    @Override
-    public ValueNode getLength() {
-        return length;
-    }
-
-    @Override
-    public ValueNode getArray() {
-        return dest;
-    }
-
-    @Override
-    public ValueNode getIndex() {
-        return destPos;
-    }
-
-    @Override
-    public boolean isObjectArray() {
-        return elementKind == JavaKind.Object;
-    }
-
-    @Override
-    public boolean isInitialization() {
-        return false;
-    }
-
-    @NodeIntrinsic
-    public static native void arraycopy(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, @ConstantNodeParameter int unrolledLength,
-                    @ConstantNodeParameter JavaKind elementKind);
-
-    public int getUnrollLength() {
-        return unrolledLength;
-    }
-
-    public JavaKind getElementKind() {
-        return elementKind;
-    }
-
-    @Override
-    public LocationIdentity getLocationIdentity() {
-        if (elementKind != null) {
-            return NamedLocationIdentity.getArrayLocation(elementKind);
-        }
-        return any();
-    }
-
-    @Override
-    public void lower(LoweringTool tool) {
-        tool.getLowerer().lower(this, tool);
-    }
-
-    @Override
-    public MemoryNode getLastLocationAccess() {
-        return lastLocationAccess;
-    }
-
-    @Override
-    public void setLastLocationAccess(MemoryNode lla) {
-        updateUsagesInterface(lastLocationAccess, lla);
-        lastLocationAccess = lla;
-    }
-}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/arraycopy/ArrayCopyWithSlowPathNode.java

@@ -22,76 +22,39 @@
  */
 package org.graalvm.compiler.hotspot.replacements.arraycopy;
 
-import jdk.vm.ci.code.BytecodeFrame;
-import jdk.vm.ci.meta.JavaKind;
-
-import static org.graalvm.word.LocationIdentity.any;
-
 import org.graalvm.compiler.graph.NodeClass;
-import org.graalvm.compiler.hotspot.word.KlassPointer;
 import org.graalvm.compiler.nodeinfo.InputType;
 import org.graalvm.compiler.nodeinfo.NodeInfo;
-import org.graalvm.compiler.nodes.NamedLocationIdentity;
 import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.type.StampTool;
 import org.graalvm.compiler.replacements.SnippetTemplate;
 import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode;
-import org.graalvm.word.LocationIdentity;
+
+import jdk.vm.ci.code.BytecodeFrame;
+import jdk.vm.ci.meta.JavaKind;
 
 @NodeInfo(allowedUsageTypes = InputType.Memory)
-public final class ArrayCopySlowPathNode extends BasicArrayCopyNode {
+public final class ArrayCopyWithSlowPathNode extends BasicArrayCopyNode {
 
-    public static final NodeClass<ArrayCopySlowPathNode> TYPE = NodeClass.create(ArrayCopySlowPathNode.class);
+    public static final NodeClass<ArrayCopyWithSlowPathNode> TYPE = NodeClass.create(ArrayCopyWithSlowPathNode.class);
 
     private final SnippetTemplate.SnippetInfo snippet;
 
-    /**
-     * Extra context for the slow path snippet.
-     */
-    private final Object argument;
-
-    /**
-     * AOT compilation requires klass constants to be exposed after the first lowering to be handled
-     * automatically. Lowering for {@link ArrayCopySlowPathNode}, with snippet ==
-     * {@link ArrayCopySnippets#arraycopyPredictedObjectWork}, requires a klass of Object[]. For
-     * other snippets {@link #predictedKlass} is a null constant.
-     */
-    @Input protected ValueNode predictedKlass;
-
-    public ArrayCopySlowPathNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode predictedKlass, JavaKind elementKind,
-                    SnippetTemplate.SnippetInfo snippet, Object argument) {
+    public ArrayCopyWithSlowPathNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, SnippetTemplate.SnippetInfo snippet, JavaKind elementKind) {
         super(TYPE, src, srcPos, dest, destPos, length, elementKind, BytecodeFrame.INVALID_FRAMESTATE_BCI);
         assert StampTool.isPointerNonNull(src) && StampTool.isPointerNonNull(dest) : "must have been null checked";
         this.snippet = snippet;
-        this.argument = argument;
-        this.predictedKlass = predictedKlass;
     }
 
     @NodeIntrinsic
-    public static native void arraycopy(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, KlassPointer predictedKlass,
-                    @ConstantNodeParameter JavaKind elementKind, @ConstantNodeParameter SnippetTemplate.SnippetInfo snippet, @ConstantNodeParameter Object argument);
+    public static native void arraycopy(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, @ConstantNodeParameter SnippetTemplate.SnippetInfo snippet,
+                    @ConstantNodeParameter JavaKind elementKind);
 
     public SnippetTemplate.SnippetInfo getSnippet() {
         return snippet;
     }
 
-    public Object getArgument() {
-        return argument;
-    }
-
-    @Override
-    public LocationIdentity getLocationIdentity() {
-        if (elementKind != null) {
-            return NamedLocationIdentity.getArrayLocation(elementKind);
-        }
-        return any();
-    }
-
     public void setBci(int bci) {
         this.bci = bci;
     }
-
-    public ValueNode getPredictedKlass() {
-        return predictedKlass;
-    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/arraycopy/CheckcastArrayCopyCallNode.java

@@ -23,12 +23,13 @@
 //JaCoCo Exclude
 package org.graalvm.compiler.hotspot.replacements.arraycopy;
 
-import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_UNKNOWN;
-import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_UNKNOWN;
 import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayBaseOffset;
 import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayIndexScale;
+import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_UNKNOWN;
+import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_UNKNOWN;
 
 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
+import org.graalvm.compiler.core.common.type.PrimitiveStamp;
 import org.graalvm.compiler.core.common.type.StampFactory;
 import org.graalvm.compiler.graph.NodeClass;
 import org.graalvm.compiler.hotspot.HotSpotGraalRuntimeProvider;
@@ -114,8 +115,9 @@ public final class CheckcastArrayCopyCallNode extends AbstractMemoryCheckpoint i
         graph().addBeforeFixed(this, basePtr);
 
         int shift = CodeUtil.log2(getArrayIndexScale(JavaKind.Object));
-        ValueNode scaledIndex = graph().unique(new LeftShiftNode(pos, ConstantNode.forInt(shift, graph())));
-        ValueNode offset = graph().unique(new AddNode(scaledIndex, ConstantNode.forInt(getArrayBaseOffset(JavaKind.Object), graph())));
+        ValueNode extendedPos = IntegerConvertNode.convert(pos, StampFactory.forKind(runtime.getTarget().wordJavaKind), graph());
+        ValueNode scaledIndex = graph().unique(new LeftShiftNode(extendedPos, ConstantNode.forInt(shift, graph())));
+        ValueNode offset = graph().unique(new AddNode(scaledIndex, ConstantNode.forIntegerBits(PrimitiveStamp.getBits(scaledIndex.stamp()), getArrayBaseOffset(JavaKind.Object), graph())));
         return graph().unique(new OffsetAddressNode(basePtr, offset));
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/arraycopy/GenericArrayCopyCallNode.java

@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2014, 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+//JaCoCo Exclude
+package org.graalvm.compiler.hotspot.replacements.arraycopy;
+
+import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_UNKNOWN;
+import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_UNKNOWN;
+
+import org.graalvm.compiler.core.common.type.StampFactory;
+import org.graalvm.compiler.graph.NodeClass;
+import org.graalvm.compiler.hotspot.HotSpotBackend;
+import org.graalvm.compiler.hotspot.HotSpotGraalRuntimeProvider;
+import org.graalvm.compiler.hotspot.nodes.GetObjectAddressNode;
+import org.graalvm.compiler.nodeinfo.InputType;
+import org.graalvm.compiler.nodeinfo.NodeInfo;
+import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.nodes.ValueNode;
+import org.graalvm.compiler.nodes.calc.IntegerConvertNode;
+import org.graalvm.compiler.nodes.extended.ForeignCallNode;
+import org.graalvm.compiler.nodes.memory.AbstractMemoryCheckpoint;
+import org.graalvm.compiler.nodes.memory.MemoryCheckpoint;
+import org.graalvm.compiler.nodes.spi.Lowerable;
+import org.graalvm.compiler.nodes.spi.LoweringTool;
+import org.graalvm.word.LocationIdentity;
+
+import jdk.vm.ci.meta.JavaKind;
+
+@NodeInfo(allowedUsageTypes = {InputType.Memory, InputType.Value}, cycles = CYCLES_UNKNOWN, size = SIZE_UNKNOWN)
+public final class GenericArrayCopyCallNode extends AbstractMemoryCheckpoint implements Lowerable, MemoryCheckpoint.Single {
+
+    public static final NodeClass<GenericArrayCopyCallNode> TYPE = NodeClass.create(GenericArrayCopyCallNode.class);
+    @Input ValueNode src;
+    @Input ValueNode srcPos;
+    @Input ValueNode dest;
+    @Input ValueNode destPos;
+    @Input ValueNode length;
+
+    protected final HotSpotGraalRuntimeProvider runtime;
+
+    protected GenericArrayCopyCallNode(@InjectedNodeParameter HotSpotGraalRuntimeProvider runtime, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length) {
+        super(TYPE, StampFactory.forKind(JavaKind.Int));
+        this.src = src;
+        this.srcPos = srcPos;
+        this.dest = dest;
+        this.destPos = destPos;
+        this.length = length;
+        this.runtime = runtime;
+    }
+
+    public ValueNode getSource() {
+        return src;
+    }
+
+    public ValueNode getSourcePosition() {
+        return srcPos;
+    }
+
+    public ValueNode getDestination() {
+        return dest;
+    }
+
+    public ValueNode getDestinationPosition() {
+        return destPos;
+    }
+
+    public ValueNode getLength() {
+        return length;
+    }
+
+    @Override
+    public void lower(LoweringTool tool) {
+        if (graph().getGuardsStage().areFrameStatesAtDeopts()) {
+            StructuredGraph graph = graph();
+            ValueNode srcAddr = objectAddress(getSource());
+            ValueNode destAddr = objectAddress(getDestination());
+            ForeignCallNode call = graph.add(new ForeignCallNode(runtime.getHostBackend().getForeignCalls(), HotSpotBackend.GENERIC_ARRAYCOPY, srcAddr, srcPos, destAddr, destPos, length));
+            call.setStateAfter(stateAfter());
+            graph.replaceFixedWithFixed(this, call);
+        }
+    }
+
+    private ValueNode objectAddress(ValueNode obj) {
+        GetObjectAddressNode result = graph().add(new GetObjectAddressNode(obj));
+        graph().addBeforeFixed(this, result);
+        return result;
+    }
+
+    private ValueNode wordValue(ValueNode value) {
+        if (value.stamp().getStackKind() != runtime.getTarget().wordJavaKind) {
+            return IntegerConvertNode.convert(value, StampFactory.forKind(runtime.getTarget().wordJavaKind), graph());
+        }
+        return value;
+    }
+
+    @Override
+    public LocationIdentity getLocationIdentity() {
+        return LocationIdentity.any();
+    }
+
+    @NodeIntrinsic
+    public static native int genericArraycopy(Object src, int srcPos, Object dest, int destPos, int length);
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/arraycopy/UnsafeArrayCopyNode.java

@@ -1,153 +0,0 @@
-/*
- * Copyright (c) 2013, 2016, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-package org.graalvm.compiler.hotspot.replacements.arraycopy;
-
-import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_256;
-import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_64;
-import static org.graalvm.word.LocationIdentity.any;
-
-import org.graalvm.compiler.core.common.type.StampFactory;
-import org.graalvm.compiler.graph.NodeClass;
-import org.graalvm.compiler.nodeinfo.InputType;
-import org.graalvm.compiler.nodeinfo.NodeInfo;
-import org.graalvm.compiler.nodes.NamedLocationIdentity;
-import org.graalvm.compiler.nodes.ValueNode;
-import org.graalvm.compiler.nodes.extended.ArrayRangeWriteNode;
-import org.graalvm.compiler.nodes.memory.MemoryAccess;
-import org.graalvm.compiler.nodes.memory.MemoryCheckpoint;
-import org.graalvm.compiler.nodes.memory.MemoryNode;
-import org.graalvm.compiler.nodes.spi.Lowerable;
-import org.graalvm.compiler.nodes.spi.LoweringTool;
-import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
-import org.graalvm.word.LocationIdentity;
-
-import jdk.vm.ci.meta.JavaKind;
-
-@NodeInfo(allowedUsageTypes = {InputType.Memory}, cycles = CYCLES_256, size = SIZE_64)
-public final class UnsafeArrayCopyNode extends ArrayRangeWriteNode implements Lowerable, MemoryCheckpoint.Single, MemoryAccess {
-
-    public static final NodeClass<UnsafeArrayCopyNode> TYPE = NodeClass.create(UnsafeArrayCopyNode.class);
-    @Input ValueNode src;
-    @Input ValueNode srcPos;
-    @Input ValueNode dest;
-    @Input ValueNode destPos;
-    @Input ValueNode length;
-    @OptionalInput ValueNode layoutHelper;
-
-    @OptionalInput(InputType.Memory) MemoryNode lastLocationAccess;
-
-    protected JavaKind elementKind;
-
-    public UnsafeArrayCopyNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode layoutHelper, JavaKind elementKind) {
-        super(TYPE, StampFactory.forVoid());
-        assert layoutHelper == null || elementKind == null;
-        this.src = src;
-        this.srcPos = srcPos;
-        this.dest = dest;
-        this.destPos = destPos;
-        this.length = length;
-        this.layoutHelper = layoutHelper;
-        this.elementKind = elementKind;
-    }
-
-    public UnsafeArrayCopyNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, JavaKind elementKind) {
-        this(src, srcPos, dest, destPos, length, null, elementKind);
-    }
-
-    public UnsafeArrayCopyNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode layoutHelper) {
-        this(src, srcPos, dest, destPos, length, layoutHelper, null);
-    }
-
-    @Override
-    public ValueNode getArray() {
-        return dest;
-    }
-
-    @Override
-    public ValueNode getIndex() {
-        return destPos;
-    }
-
-    @Override
-    public ValueNode getLength() {
-        return length;
-    }
-
-    @Override
-    public boolean isObjectArray() {
-        return elementKind == JavaKind.Object;
-    }
-
-    @Override
-    public boolean isInitialization() {
-        return false;
-    }
-
-    public JavaKind getElementKind() {
-        return elementKind;
-    }
-
-    @Override
-    public void lower(LoweringTool tool) {
-        if (graph().getGuardsStage().areFrameStatesAtDeopts()) {
-            UnsafeArrayCopySnippets.Templates templates = tool.getReplacements().getSnippetTemplateCache(UnsafeArrayCopySnippets.Templates.class);
-            templates.lower(this, tool);
-        }
-    }
-
-    public void addSnippetArguments(Arguments args) {
-        args.add("src", src);
-        args.add("srcPos", srcPos);
-        args.add("dest", dest);
-        args.add("destPos", destPos);
-        args.add("length", length);
-        if (layoutHelper != null) {
-            args.add("layoutHelper", layoutHelper);
-        }
-    }
-
-    @Override
-    public LocationIdentity getLocationIdentity() {
-        if (elementKind != null) {
-            return NamedLocationIdentity.getArrayLocation(elementKind);
-        }
-        return any();
-    }
-
-    @Override
-    public MemoryNode getLastLocationAccess() {
-        return lastLocationAccess;
-    }
-
-    @Override
-    public void setLastLocationAccess(MemoryNode lla) {
-        updateUsagesInterface(lastLocationAccess, lla);
-        lastLocationAccess = lla;
-    }
-
-    @NodeIntrinsic
-    public static native void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter JavaKind elementKind);
-
-    @NodeIntrinsic
-    public static native void arraycopyPrimitive(Object src, int srcPos, Object dest, int destPos, int length, int layoutHelper);
-}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/arraycopy/UnsafeArrayCopySnippets.java

@@ -1,324 +0,0 @@
-/*
- * Copyright (c) 2013, 2016, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-package org.graalvm.compiler.hotspot.replacements.arraycopy;
-
-import static jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider.getArrayIndexScale;
-import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeMask;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperHeaderSizeShift;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeMask;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.runtime;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
-import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
-import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
-import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
-import static org.graalvm.word.LocationIdentity.any;
-
-import org.graalvm.compiler.api.replacements.Fold;
-import org.graalvm.compiler.api.replacements.Snippet;
-import org.graalvm.compiler.core.common.NumUtil;
-import org.graalvm.compiler.debug.DebugHandlersFactory;
-import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
-import org.graalvm.compiler.hotspot.phases.WriteBarrierAdditionPhase;
-import org.graalvm.compiler.nodes.NamedLocationIdentity;
-import org.graalvm.compiler.nodes.extended.RawLoadNode;
-import org.graalvm.compiler.nodes.extended.RawStoreNode;
-import org.graalvm.compiler.nodes.extended.UnsafeCopyNode;
-import org.graalvm.compiler.nodes.spi.LoweringTool;
-import org.graalvm.compiler.options.OptionValues;
-import org.graalvm.compiler.replacements.SnippetTemplate;
-import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
-import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
-import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
-import org.graalvm.compiler.replacements.Snippets;
-import org.graalvm.compiler.word.ObjectAccess;
-import org.graalvm.word.LocationIdentity;
-import org.graalvm.word.UnsignedWord;
-import org.graalvm.word.WordFactory;
-
-import jdk.vm.ci.code.TargetDescription;
-import jdk.vm.ci.meta.JavaKind;
-
-/**
- * As opposed to {@link ArrayCopySnippets}, these Snippets do <b>not</b> perform store checks.
- */
-public class UnsafeArrayCopySnippets implements Snippets {
-
-    private static final boolean supportsUnalignedMemoryAccess = runtime().getHostJVMCIBackend().getTarget().arch.supportsUnalignedMemoryAccess();
-
-    private static final JavaKind VECTOR_KIND = JavaKind.Long;
-    private static final long VECTOR_SIZE = getArrayIndexScale(VECTOR_KIND);
-
-    private static void vectorizedCopy(Object src, int srcPos, Object dest, int destPos, int length, JavaKind baseKind, LocationIdentity locationIdentity) {
-        int arrayBaseOffset = arrayBaseOffset(baseKind);
-        int elementSize = arrayIndexScale(baseKind);
-        long byteLength = (long) length * elementSize;
-        long srcOffset = (long) srcPos * elementSize;
-        long destOffset = (long) destPos * elementSize;
-
-        long preLoopBytes;
-        long mainLoopBytes;
-        long postLoopBytes;
-
-        // We can easily vectorize the loop if both offsets have the same alignment.
-        if (byteLength >= VECTOR_SIZE && (srcOffset % VECTOR_SIZE) == (destOffset % VECTOR_SIZE)) {
-            preLoopBytes = NumUtil.roundUp(arrayBaseOffset + srcOffset, VECTOR_SIZE) - (arrayBaseOffset + srcOffset);
-            postLoopBytes = (byteLength - preLoopBytes) % VECTOR_SIZE;
-            mainLoopBytes = byteLength - preLoopBytes - postLoopBytes;
-        } else {
-            // Does the architecture support unaligned memory accesses?
-            if (supportsUnalignedMemoryAccess) {
-                preLoopBytes = byteLength % VECTOR_SIZE;
-                mainLoopBytes = byteLength - preLoopBytes;
-                postLoopBytes = 0;
-            } else {
-                // No. Let's do element-wise copying.
-                preLoopBytes = byteLength;
-                mainLoopBytes = 0;
-                postLoopBytes = 0;
-            }
-        }
-
-        if (probability(NOT_FREQUENT_PROBABILITY, src == dest) && probability(NOT_FREQUENT_PROBABILITY, srcPos < destPos)) {
-            // bad aliased case
-            srcOffset += byteLength;
-            destOffset += byteLength;
-
-            // Post-loop
-            for (long i = 0; i < postLoopBytes; i += elementSize) {
-                srcOffset -= elementSize;
-                destOffset -= elementSize;
-                UnsafeCopyNode.copy(src, arrayBaseOffset + srcOffset, dest, arrayBaseOffset + destOffset, baseKind, locationIdentity);
-            }
-            // Main-loop
-            for (long i = 0; i < mainLoopBytes; i += VECTOR_SIZE) {
-                srcOffset -= VECTOR_SIZE;
-                destOffset -= VECTOR_SIZE;
-                UnsafeCopyNode.copy(src, arrayBaseOffset + srcOffset, dest, arrayBaseOffset + destOffset, VECTOR_KIND, locationIdentity);
-            }
-            // Pre-loop
-            for (long i = 0; i < preLoopBytes; i += elementSize) {
-                srcOffset -= elementSize;
-                destOffset -= elementSize;
-                UnsafeCopyNode.copy(src, arrayBaseOffset + srcOffset, dest, arrayBaseOffset + destOffset, baseKind, locationIdentity);
-            }
-        } else {
-            // Pre-loop
-            for (long i = 0; i < preLoopBytes; i += elementSize) {
-                UnsafeCopyNode.copy(src, arrayBaseOffset + srcOffset, dest, arrayBaseOffset + destOffset, baseKind, locationIdentity);
-                srcOffset += elementSize;
-                destOffset += elementSize;
-            }
-            // Main-loop
-            for (long i = 0; i < mainLoopBytes; i += VECTOR_SIZE) {
-                UnsafeCopyNode.copy(src, arrayBaseOffset + srcOffset, dest, arrayBaseOffset + destOffset, VECTOR_KIND, locationIdentity);
-                srcOffset += VECTOR_SIZE;
-                destOffset += VECTOR_SIZE;
-            }
-            // Post-loop
-            for (long i = 0; i < postLoopBytes; i += elementSize) {
-                UnsafeCopyNode.copy(src, arrayBaseOffset + srcOffset, dest, arrayBaseOffset + destOffset, baseKind, locationIdentity);
-                srcOffset += elementSize;
-                destOffset += elementSize;
-            }
-        }
-    }
-
-    @Fold
-    static LocationIdentity getArrayLocation(JavaKind kind) {
-        return NamedLocationIdentity.getArrayLocation(kind);
-    }
-
-    @Snippet
-    public static void arraycopyByte(byte[] src, int srcPos, byte[] dest, int destPos, int length) {
-        JavaKind kind = JavaKind.Byte;
-        vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind));
-    }
-
-    @Snippet
-    public static void arraycopyBoolean(boolean[] src, int srcPos, boolean[] dest, int destPos, int length) {
-        JavaKind kind = JavaKind.Boolean;
-        vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind));
-    }
-
-    @Snippet
-    public static void arraycopyChar(char[] src, int srcPos, char[] dest, int destPos, int length) {
-        JavaKind kind = JavaKind.Char;
-        vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind));
-    }
-
-    @Snippet
-    public static void arraycopyShort(short[] src, int srcPos, short[] dest, int destPos, int length) {
-        JavaKind kind = JavaKind.Short;
-        vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind));
-    }
-
-    @Snippet
-    public static void arraycopyInt(int[] src, int srcPos, int[] dest, int destPos, int length) {
-        JavaKind kind = JavaKind.Int;
-        vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind));
-    }
-
-    @Snippet
-    public static void arraycopyFloat(float[] src, int srcPos, float[] dest, int destPos, int length) {
-        JavaKind kind = JavaKind.Float;
-        vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind));
-    }
-
-    @Snippet
-    public static void arraycopyLong(long[] src, int srcPos, long[] dest, int destPos, int length) {
-        JavaKind kind = JavaKind.Long;
-        vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind));
-    }
-
-    @Snippet
-    public static void arraycopyDouble(double[] src, int srcPos, double[] dest, int destPos, int length) {
-        JavaKind kind = JavaKind.Double;
-        /*
-         * TODO atomicity problem on 32-bit architectures: The JVM spec requires double values to be
-         * copied atomically, but not long values. For example, on Intel 32-bit this code is not
-         * atomic as long as the vector kind remains Kind.Long.
-         */
-        vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind));
-    }
-
-    /**
-     * For this kind, Object, we want to avoid write barriers between writes, but instead have them
-     * at the end of the snippet. This is done by using {@link RawStoreNode}, and rely on
-     * {@link WriteBarrierAdditionPhase} to put write barriers after the {@link UnsafeArrayCopyNode}
-     * with kind Object.
-     */
-    @Snippet
-    public static void arraycopyObject(Object[] src, int srcPos, Object[] dest, int destPos, int length) {
-        JavaKind kind = JavaKind.Object;
-        final int scale = arrayIndexScale(kind);
-        int arrayBaseOffset = arrayBaseOffset(kind);
-        LocationIdentity arrayLocation = getArrayLocation(kind);
-        if (src == dest && srcPos < destPos) { // bad aliased case
-            long start = (long) (length - 1) * scale;
-            for (long i = start; i >= 0; i -= scale) {
-                Object a = RawLoadNode.load(src, arrayBaseOffset + i + (long) srcPos * scale, kind, arrayLocation);
-                RawStoreNode.storeObject(dest, arrayBaseOffset + i + (long) destPos * scale, a, kind, getArrayLocation(kind), false);
-            }
-        } else {
-            long end = (long) length * scale;
-            for (long i = 0; i < end; i += scale) {
-                Object a = RawLoadNode.load(src, arrayBaseOffset + i + (long) srcPos * scale, kind, arrayLocation);
-                RawStoreNode.storeObject(dest, arrayBaseOffset + i + (long) destPos * scale, a, kind, getArrayLocation(kind), false);
-            }
-        }
-    }
-
-    @Snippet
-    public static void arraycopyPrimitive(Object src, int srcPos, Object dest, int destPos, int length, int layoutHelper) {
-        int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG);
-        int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG);
-
-        UnsignedWord vectorSize = WordFactory.unsigned(VECTOR_SIZE);
-        UnsignedWord srcOffset = WordFactory.unsigned(srcPos).shiftLeft(log2ElementSize).add(headerSize);
-        UnsignedWord destOffset = WordFactory.unsigned(destPos).shiftLeft(log2ElementSize).add(headerSize);
-        UnsignedWord destStart = destOffset;
-        UnsignedWord destEnd = destOffset.add(WordFactory.unsigned(length).shiftLeft(log2ElementSize));
-
-        UnsignedWord destVectorEnd = null;
-        UnsignedWord nonVectorBytes = null;
-        UnsignedWord sizeInBytes = WordFactory.unsigned(length).shiftLeft(log2ElementSize);
-        if (supportsUnalignedMemoryAccess) {
-            nonVectorBytes = sizeInBytes.unsignedRemainder(vectorSize);
-            destVectorEnd = destEnd;
-        } else {
-            boolean inPhase = srcOffset.and((int) VECTOR_SIZE - 1).equal(destOffset.and((int) VECTOR_SIZE - 1));
-            boolean hasAtLeastOneVector = sizeInBytes.aboveOrEqual(vectorSize);
-            // We must have at least one full vector, otherwise we must copy each byte separately
-            if (hasAtLeastOneVector && inPhase) { // If in phase, we can vectorize
-                nonVectorBytes = vectorSize.subtract(destStart.unsignedRemainder(vectorSize));
-            } else { // fallback is byte-wise
-                nonVectorBytes = sizeInBytes;
-            }
-            destVectorEnd = destEnd.subtract(destEnd.unsignedRemainder(vectorSize));
-        }
-
-        UnsignedWord destNonVectorEnd = destStart.add(nonVectorBytes);
-        while (destOffset.belowThan(destNonVectorEnd)) {
-            ObjectAccess.writeByte(dest, destOffset, ObjectAccess.readByte(src, srcOffset, any()), any());
-            destOffset = destOffset.add(1);
-            srcOffset = srcOffset.add(1);
-        }
-        // Unsigned destVectorEnd = destEnd.subtract(destEnd.unsignedRemainder(8));
-        while (destOffset.belowThan(destVectorEnd)) {
-            ObjectAccess.writeWord(dest, destOffset, ObjectAccess.readWord(src, srcOffset, any()), any());
-            destOffset = destOffset.add(wordSize());
-            srcOffset = srcOffset.add(wordSize());
-        }
-        // Do the last bytes each when it is required to have absolute alignment.
-        while (!supportsUnalignedMemoryAccess && destOffset.belowThan(destEnd)) {
-            ObjectAccess.writeByte(dest, destOffset, ObjectAccess.readByte(src, srcOffset, any()), any());
-            destOffset = destOffset.add(1);
-            srcOffset = srcOffset.add(1);
-        }
-    }
-
-    public static class Templates extends AbstractTemplates {
-
-        private final SnippetInfo[] arraycopySnippets;
-        private final SnippetInfo genericPrimitiveSnippet;
-
-        public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, HotSpotProviders providers, TargetDescription target) {
-            super(options, factories, providers, providers.getSnippetReflection(), target);
-
-            arraycopySnippets = new SnippetInfo[JavaKind.values().length];
-            arraycopySnippets[JavaKind.Boolean.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyBoolean");
-            arraycopySnippets[JavaKind.Byte.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyByte");
-            arraycopySnippets[JavaKind.Short.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyShort");
-            arraycopySnippets[JavaKind.Char.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyChar");
-            arraycopySnippets[JavaKind.Int.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyInt");
-            arraycopySnippets[JavaKind.Long.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyLong");
-            arraycopySnippets[JavaKind.Float.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyFloat");
-            arraycopySnippets[JavaKind.Double.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyDouble");
-            arraycopySnippets[JavaKind.Object.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyObject");
-
-            genericPrimitiveSnippet = snippet(UnsafeArrayCopySnippets.class, "arraycopyPrimitive");
-        }
-
-        public void lower(UnsafeArrayCopyNode node, LoweringTool tool) {
-            JavaKind elementKind = node.getElementKind();
-            SnippetInfo snippet;
-            if (elementKind == null) {
-                // primitive array of unknown kind
-                snippet = genericPrimitiveSnippet;
-            } else {
-                snippet = arraycopySnippets[elementKind.ordinal()];
-                assert snippet != null : "arraycopy snippet for " + elementKind.name() + " not found";
-            }
-
-            Arguments args = new Arguments(snippet, node.graph().getGuardsStage(), tool.getLoweringStage());
-            node.addSnippetArguments(args);
-
-            SnippetTemplate template = template(node.getDebug(), args);
-            template.instantiate(providers.getMetaAccess(), node, DEFAULT_REPLACER, args);
-        }
-    }
-}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/profiling/ProbabilisticProfileSnippets.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -54,6 +54,7 @@ import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
 import org.graalvm.compiler.replacements.Snippets;
 
+import jdk.vm.ci.code.CodeUtil;
 import jdk.vm.ci.code.TargetDescription;
 
 public class ProbabilisticProfileSnippets implements Snippets {
@@ -64,22 +65,22 @@ public class ProbabilisticProfileSnippets implements Snippets {
     }
 
     @Snippet
-    public static int notificationMask(int freqLog, int probLog) {
-        int probabilityMask = (1 << probLog) - 1;
+    public static int notificationMask(int freqLog, int probLog, int stepLog) {
         int frequencyMask = (1 << freqLog) - 1;
-        return frequencyMask & ~probabilityMask;
+        int stepMask = (1 << (stepLog + probLog)) - 1;
+        return frequencyMask & ~stepMask;
     }
 
     @NodeIntrinsic(ForeignCallNode.class)
     public static native void methodInvocationEvent(@ConstantNodeParameter ForeignCallDescriptor descriptor, MethodCountersPointer counters);
 
     @Snippet
-    public static void profileMethodEntryWithProbability(MethodCountersPointer counters, int random, @ConstantParameter int freqLog, @ConstantParameter int probLog) {
+    public static void profileMethodEntryWithProbability(MethodCountersPointer counters, int random, int step, int stepLog, @ConstantParameter int freqLog, @ConstantParameter int probLog) {
         if (probability(1.0 / (1 << probLog), shouldProfile(probLog, random))) {
-            int counterValue = counters.readInt(config(INJECTED_VMCONFIG).invocationCounterOffset) + (config(INJECTED_VMCONFIG).invocationCounterIncrement << probLog);
+            int counterValue = counters.readInt(config(INJECTED_VMCONFIG).invocationCounterOffset) + ((config(INJECTED_VMCONFIG).invocationCounterIncrement * step) << probLog);
             counters.writeInt(config(INJECTED_VMCONFIG).invocationCounterOffset, counterValue);
             if (freqLog >= 0) {
-                int mask = notificationMask(freqLog, probLog);
+                int mask = notificationMask(freqLog, probLog, stepLog);
                 if (probability(SLOW_PATH_PROBABILITY, (counterValue & (mask << config(INJECTED_VMCONFIG).invocationCounterShift)) == 0)) {
                     methodInvocationEvent(HotSpotBackend.INVOCATION_EVENT, counters);
                 }
@@ -91,11 +92,12 @@ public class ProbabilisticProfileSnippets implements Snippets {
     public static native void methodBackedgeEvent(@ConstantNodeParameter ForeignCallDescriptor descriptor, MethodCountersPointer counters, int bci, int targetBci);
 
     @Snippet
-    public static void profileBackedgeWithProbability(MethodCountersPointer counters, int random, @ConstantParameter int freqLog, @ConstantParameter int probLog, int bci, int targetBci) {
+    public static void profileBackedgeWithProbability(MethodCountersPointer counters, int random, int step, int stepLog, @ConstantParameter int freqLog, @ConstantParameter int probLog, int bci,
+                    int targetBci) {
         if (probability(1.0 / (1 << probLog), shouldProfile(probLog, random))) {
-            int counterValue = counters.readInt(config(INJECTED_VMCONFIG).backedgeCounterOffset) + (config(INJECTED_VMCONFIG).invocationCounterIncrement << probLog);
+            int counterValue = counters.readInt(config(INJECTED_VMCONFIG).backedgeCounterOffset) + ((config(INJECTED_VMCONFIG).invocationCounterIncrement * step) << probLog);
             counters.writeInt(config(INJECTED_VMCONFIG).backedgeCounterOffset, counterValue);
-            int mask = notificationMask(freqLog, probLog);
+            int mask = notificationMask(freqLog, probLog, stepLog);
             if (probability(SLOW_PATH_PROBABILITY, (counterValue & (mask << config(INJECTED_VMCONFIG).invocationCounterShift)) == 0)) {
                 methodBackedgeEvent(HotSpotBackend.BACKEDGE_EVENT, counters, bci, targetBci);
             }
@@ -103,10 +105,11 @@ public class ProbabilisticProfileSnippets implements Snippets {
     }
 
     @Snippet
-    public static void profileConditionalBackedgeWithProbability(MethodCountersPointer counters, int random, @ConstantParameter int freqLog, @ConstantParameter int probLog, boolean branchCondition,
+    public static void profileConditionalBackedgeWithProbability(MethodCountersPointer counters, int random, int step, int stepLog, @ConstantParameter int freqLog,
+                    @ConstantParameter int probLog, boolean branchCondition,
                     int bci, int targetBci) {
         if (branchCondition) {
-            profileBackedgeWithProbability(counters, random, freqLog, probLog, bci, targetBci);
+            profileBackedgeWithProbability(counters, random, step, stepLog, freqLog, probLog, bci, targetBci);
         }
     }
 
@@ -124,6 +127,8 @@ public class ProbabilisticProfileSnippets implements Snippets {
 
             StructuredGraph graph = profileNode.graph();
             LoadMethodCountersNode counters = graph.unique(new LoadMethodCountersNode(profileNode.getProfiledMethod()));
+            ConstantNode step = ConstantNode.forInt(profileNode.getStep(), graph);
+            ConstantNode stepLog = ConstantNode.forInt(CodeUtil.log2(profileNode.getStep()), graph);
 
             if (profileNode instanceof ProfileBranchNode) {
                 // Backedge event
@@ -132,8 +137,11 @@ public class ProbabilisticProfileSnippets implements Snippets {
                 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
                 ConstantNode bci = ConstantNode.forInt(profileBranchNode.bci(), graph);
                 ConstantNode targetBci = ConstantNode.forInt(profileBranchNode.targetBci(), graph);
+
                 args.add("counters", counters);
                 args.add("random", profileBranchNode.getRandom());
+                args.add("step", step);
+                args.add("stepLog", stepLog);
                 args.addConst("freqLog", profileBranchNode.getNotificationFreqLog());
                 args.addConst("probLog", profileBranchNode.getProbabilityLog());
                 if (profileBranchNode.hasCondition()) {
@@ -148,8 +156,11 @@ public class ProbabilisticProfileSnippets implements Snippets {
                 ProfileInvokeNode profileInvokeNode = (ProfileInvokeNode) profileNode;
                 // Method invocation event
                 Arguments args = new Arguments(profileMethodEntryWithProbability, graph.getGuardsStage(), tool.getLoweringStage());
+
                 args.add("counters", counters);
                 args.add("random", profileInvokeNode.getRandom());
+                args.add("step", step);
+                args.add("stepLog", stepLog);
                 args.addConst("freqLog", profileInvokeNode.getNotificationFreqLog());
                 args.addConst("probLog", profileInvokeNode.getProbabilityLog());
                 SnippetTemplate template = template(graph.getDebug(), args);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/profiling/ProfileSnippets.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -54,6 +54,7 @@ import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
 import org.graalvm.compiler.replacements.Snippets;
 
+import jdk.vm.ci.code.CodeUtil;
 import jdk.vm.ci.code.TargetDescription;
 
 public class ProfileSnippets implements Snippets {
@@ -61,12 +62,19 @@ public class ProfileSnippets implements Snippets {
     public static native void methodInvocationEvent(@ConstantNodeParameter ForeignCallDescriptor descriptor, MethodCountersPointer counters);
 
     @Snippet
-    public static void profileMethodEntry(MethodCountersPointer counters, @ConstantParameter int freqLog) {
-        int counterValue = counters.readInt(config(INJECTED_VMCONFIG).invocationCounterOffset) + config(INJECTED_VMCONFIG).invocationCounterIncrement;
+    protected static int notificationMask(int freqLog, int stepLog) {
+        int stepMask = (1 << stepLog) - 1;
+        int frequencyMask = (1 << freqLog) - 1;
+        return frequencyMask & ~stepMask;
+    }
+
+    @Snippet
+    public static void profileMethodEntry(MethodCountersPointer counters, int step, int stepLog, @ConstantParameter int freqLog) {
+        int counterValue = counters.readInt(config(INJECTED_VMCONFIG).invocationCounterOffset) + config(INJECTED_VMCONFIG).invocationCounterIncrement * step;
         counters.writeInt(config(INJECTED_VMCONFIG).invocationCounterOffset, counterValue);
         if (freqLog >= 0) {
-            final int frequencyMask = (1 << freqLog) - 1;
-            if (probability(SLOW_PATH_PROBABILITY, (counterValue & (frequencyMask << config(INJECTED_VMCONFIG).invocationCounterShift)) == 0)) {
+            final int mask = notificationMask(freqLog, stepLog);
+            if (probability(SLOW_PATH_PROBABILITY, (counterValue & (mask << config(INJECTED_VMCONFIG).invocationCounterShift)) == 0)) {
                 methodInvocationEvent(HotSpotBackend.INVOCATION_EVENT, counters);
             }
         }
@@ -76,19 +84,19 @@ public class ProfileSnippets implements Snippets {
     public static native void methodBackedgeEvent(@ConstantNodeParameter ForeignCallDescriptor descriptor, MethodCountersPointer counters, int bci, int targetBci);
 
     @Snippet
-    public static void profileBackedge(MethodCountersPointer counters, @ConstantParameter int freqLog, int bci, int targetBci) {
-        int counterValue = counters.readInt(config(INJECTED_VMCONFIG).backedgeCounterOffset) + config(INJECTED_VMCONFIG).invocationCounterIncrement;
+    public static void profileBackedge(MethodCountersPointer counters, int step, int stepLog, @ConstantParameter int freqLog, int bci, int targetBci) {
+        int counterValue = counters.readInt(config(INJECTED_VMCONFIG).backedgeCounterOffset) + config(INJECTED_VMCONFIG).invocationCounterIncrement * step;
         counters.writeInt(config(INJECTED_VMCONFIG).backedgeCounterOffset, counterValue);
-        final int frequencyMask = (1 << freqLog) - 1;
-        if (probability(SLOW_PATH_PROBABILITY, (counterValue & (frequencyMask << config(INJECTED_VMCONFIG).invocationCounterShift)) == 0)) {
+        final int mask = notificationMask(freqLog, stepLog);
+        if (probability(SLOW_PATH_PROBABILITY, (counterValue & (mask << config(INJECTED_VMCONFIG).invocationCounterShift)) == 0)) {
             methodBackedgeEvent(HotSpotBackend.BACKEDGE_EVENT, counters, bci, targetBci);
         }
     }
 
     @Snippet
-    public static void profileConditionalBackedge(MethodCountersPointer counters, @ConstantParameter int freqLog, boolean branchCondition, int bci, int targetBci) {
+    public static void profileConditionalBackedge(MethodCountersPointer counters, int step, int stepLog, @ConstantParameter int freqLog, boolean branchCondition, int bci, int targetBci) {
         if (branchCondition) {
-            profileBackedge(counters, freqLog, bci, targetBci);
+            profileBackedge(counters, step, stepLog, freqLog, bci, targetBci);
         }
     }
 
@@ -104,6 +112,8 @@ public class ProfileSnippets implements Snippets {
         public void lower(ProfileNode profileNode, LoweringTool tool) {
             StructuredGraph graph = profileNode.graph();
             LoadMethodCountersNode counters = graph.unique(new LoadMethodCountersNode(profileNode.getProfiledMethod()));
+            ConstantNode step = ConstantNode.forInt(profileNode.getStep(), graph);
+            ConstantNode stepLog = ConstantNode.forInt(CodeUtil.log2(profileNode.getStep()), graph);
 
             if (profileNode instanceof ProfileBranchNode) {
                 // Backedge event
@@ -113,6 +123,8 @@ public class ProfileSnippets implements Snippets {
                 ConstantNode bci = ConstantNode.forInt(profileBranchNode.bci(), graph);
                 ConstantNode targetBci = ConstantNode.forInt(profileBranchNode.targetBci(), graph);
                 args.add("counters", counters);
+                args.add("step", step);
+                args.add("stepLog", stepLog);
                 args.addConst("freqLog", profileBranchNode.getNotificationFreqLog());
                 if (profileBranchNode.hasCondition()) {
                     args.add("branchCondition", profileBranchNode.branchCondition());
@@ -127,6 +139,8 @@ public class ProfileSnippets implements Snippets {
                 // Method invocation event
                 Arguments args = new Arguments(profileMethodEntry, graph.getGuardsStage(), tool.getLoweringStage());
                 args.add("counters", counters);
+                args.add("step", step);
+                args.add("stepLog", stepLog);
                 args.addConst("freqLog", profileInvokeNode.getNotificationFreqLog());
                 SnippetTemplate template = template(graph.getDebug(), args);
                 template.instantiate(providers.getMetaAccess(), profileNode, DEFAULT_REPLACER, args);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/NewArrayStub.java

@@ -33,6 +33,7 @@ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperLog2ElementSizeShift;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useCMSIncrementalMode;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
 import static org.graalvm.compiler.hotspot.replacements.NewObjectSnippets.MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH;
 import static org.graalvm.compiler.hotspot.replacements.NewObjectSnippets.formatArray;
@@ -122,7 +123,7 @@ public class NewArrayStub extends SnippetStub {
         // check that array length is small enough for fast path.
         Word thread = registerAsWord(threadRegister);
         boolean inlineContiguousAllocationSupported = GraalHotSpotVMConfigNode.inlineContiguousAllocationSupported();
-        if (inlineContiguousAllocationSupported && length >= 0 && length <= MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH) {
+        if (inlineContiguousAllocationSupported && !useCMSIncrementalMode(INJECTED_VMCONFIG) && length >= 0 && length <= MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH) {
             Word memory = refillAllocate(thread, intArrayHub, sizeInBytes, logging(options));
             if (memory.notEqual(0)) {
                 if (logging(options)) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/NewInstanceStub.java

@@ -53,6 +53,7 @@ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteLimitOffset;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabSlowAllocationsOffset;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabStats;
+import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useCMSIncrementalMode;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useG1GC;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
@@ -147,7 +148,7 @@ public class NewInstanceStub extends SnippetStub {
          */
         Word thread = registerAsWord(threadRegister);
         boolean inlineContiguousAllocationSupported = GraalHotSpotVMConfigNode.inlineContiguousAllocationSupported();
-        if (!forceSlowPath(options) && inlineContiguousAllocationSupported) {
+        if (!forceSlowPath(options) && inlineContiguousAllocationSupported && !useCMSIncrementalMode(INJECTED_VMCONFIG)) {
             if (isInstanceKlassFullyInitialized(hub)) {
                 int sizeInBytes = readLayoutHelper(hub);
                 Word memory = refillAllocate(thread, intArrayHub, sizeInBytes, logging(options));

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BytecodeParser.java

@@ -342,7 +342,7 @@ import org.graalvm.compiler.nodes.calc.AddNode;
 import org.graalvm.compiler.nodes.calc.AndNode;
 import org.graalvm.compiler.nodes.calc.CompareNode;
 import org.graalvm.compiler.nodes.calc.ConditionalNode;
-import org.graalvm.compiler.nodes.calc.DivNode;
+import org.graalvm.compiler.nodes.calc.FloatDivNode;
 import org.graalvm.compiler.nodes.calc.FloatConvertNode;
 import org.graalvm.compiler.nodes.calc.IntegerBelowNode;
 import org.graalvm.compiler.nodes.calc.IntegerEqualsNode;
@@ -374,7 +374,6 @@ import org.graalvm.compiler.nodes.extended.MembarNode;
 import org.graalvm.compiler.nodes.extended.StateSplitProxyNode;
 import org.graalvm.compiler.nodes.extended.ValueAnchorNode;
 import org.graalvm.compiler.nodes.graphbuilderconf.ClassInitializationPlugin;
-import org.graalvm.compiler.nodes.graphbuilderconf.InvokeDynamicPlugin;
 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration;
 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.BytecodeExceptionMode;
 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext;
@@ -383,6 +382,7 @@ import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin.InlineInfo
 import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext;
 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin;
 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins.InvocationPluginReceiver;
+import org.graalvm.compiler.nodes.graphbuilderconf.InvokeDynamicPlugin;
 import org.graalvm.compiler.nodes.graphbuilderconf.NodePlugin;
 import org.graalvm.compiler.nodes.graphbuilderconf.ProfilingPlugin;
 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
@@ -435,6 +435,7 @@ import jdk.vm.ci.meta.ResolvedJavaMethod;
 import jdk.vm.ci.meta.ResolvedJavaType;
 import jdk.vm.ci.meta.Signature;
 import jdk.vm.ci.meta.TriState;
+import org.graalvm.compiler.core.common.type.IntegerStamp;
 
 /**
  * The {@code GraphBuilder} class parses the bytecode of a method and builds the IR graph.
@@ -1036,7 +1037,7 @@ public class BytecodeParser implements GraphBuilderContext {
         deopt.updateNodeSourcePosition(() -> createBytecodePosition());
     }
 
-    private AbstractBeginNode handleException(ValueNode exceptionObject, int bci) {
+    private AbstractBeginNode handleException(ValueNode exceptionObject, int bci, boolean deoptimizeOnly) {
         assert bci == BytecodeFrame.BEFORE_BCI || bci == bci() : "invalid bci";
         debug.log("Creating exception dispatch edges at %d, exception object=%s, exception seen=%s", bci, exceptionObject, (profilingInfo == null ? "" : profilingInfo.getExceptionSeen(bci)));
 
@@ -1058,8 +1059,12 @@ public class BytecodeParser implements GraphBuilderContext {
         this.controlFlowSplit = true;
         FixedWithNextNode finishedDispatch = finishInstruction(dispatchBegin, dispatchState);
 
+        if (deoptimizeOnly) {
+            DeoptimizeNode deoptimizeNode = graph.add(new DeoptimizeNode(DeoptimizationAction.None, DeoptimizationReason.TransferToInterpreter));
+            dispatchBegin.setNext(BeginNode.begin(deoptimizeNode));
+        } else {
             createHandleExceptionTarget(finishedDispatch, bci, dispatchState);
-
+        }
         return dispatchBegin;
     }
 
@@ -1111,7 +1116,7 @@ public class BytecodeParser implements GraphBuilderContext {
     }
 
     protected ValueNode genFloatDiv(ValueNode x, ValueNode y) {
-        return DivNode.create(x, y);
+        return FloatDivNode.create(x, y);
     }
 
     protected ValueNode genFloatRem(ValueNode x, ValueNode y) {
@@ -1215,7 +1220,7 @@ public class BytecodeParser implements GraphBuilderContext {
         ValueNode exception = frameState.pop(JavaKind.Object);
         FixedGuardNode nullCheck = append(new FixedGuardNode(graph.addOrUniqueWithInputs(IsNullNode.create(exception)), NullCheckException, InvalidateReprofile, true));
         ValueNode nonNullException = graph.maybeAddOrUnique(PiNode.create(exception, exception.stamp().join(objectNonNull()), nullCheck));
-        lastInstr.setNext(handleException(nonNullException, bci()));
+        lastInstr.setNext(handleException(nonNullException, bci(), false));
     }
 
     protected LogicNode createInstanceOf(TypeReference type, ValueNode object) {
@@ -1275,12 +1280,12 @@ public class BytecodeParser implements GraphBuilderContext {
         }
         BytecodeExceptionNode exception = graph.add(new BytecodeExceptionNode(metaAccess, NullPointerException.class));
         AbstractBeginNode falseSucc = graph.add(new BeginNode());
-        ValueNode nonNullReceiver = graph.addOrUnique(PiNode.create(receiver, objectNonNull(), falseSucc));
+        ValueNode nonNullReceiver = graph.addOrUniqueWithInputs(PiNode.create(receiver, objectNonNull(), falseSucc));
         append(new IfNode(graph.addOrUniqueWithInputs(IsNullNode.create(receiver)), exception, falseSucc, SLOW_PATH_PROBABILITY));
         lastInstr = falseSucc;
 
         exception.setStateAfter(createFrameState(bci(), exception));
-        exception.setNext(handleException(exception, bci()));
+        exception.setNext(handleException(exception, bci(), false));
         EXPLICIT_EXCEPTIONS.increment(debug);
         return nonNullReceiver;
     }
@@ -1292,7 +1297,7 @@ public class BytecodeParser implements GraphBuilderContext {
         lastInstr = trueSucc;
 
         exception.setStateAfter(createFrameState(bci(), exception));
-        exception.setNext(handleException(exception, bci()));
+        exception.setNext(handleException(exception, bci(), false));
     }
 
     protected ValueNode genArrayLength(ValueNode x) {
@@ -1532,8 +1537,8 @@ public class BytecodeParser implements GraphBuilderContext {
     @Override
     public void handleReplacedInvoke(CallTargetNode callTarget, JavaKind resultType) {
         BytecodeParser intrinsicCallSiteParser = getNonIntrinsicAncestor();
-        boolean withExceptionEdge = intrinsicCallSiteParser == null ? !omitInvokeExceptionEdge(null) : !intrinsicCallSiteParser.omitInvokeExceptionEdge(null);
-        createNonInlinedInvoke(withExceptionEdge, bci(), callTarget, resultType);
+        ExceptionEdgeAction exceptionEdgeAction = intrinsicCallSiteParser == null ? getActionForInvokeExceptionEdge(null) : intrinsicCallSiteParser.getActionForInvokeExceptionEdge(null);
+        createNonInlinedInvoke(exceptionEdgeAction, bci(), callTarget, resultType);
     }
 
     protected Invoke appendInvoke(InvokeKind initialInvokeKind, ResolvedJavaMethod initialTargetMethod, ValueNode[] args) {
@@ -1603,7 +1608,7 @@ public class BytecodeParser implements GraphBuilderContext {
 
         int invokeBci = bci();
         JavaTypeProfile profile = getProfileForInvoke(invokeKind);
-        boolean withExceptionEdge = !omitInvokeExceptionEdge(inlineInfo);
+        ExceptionEdgeAction edgeAction = getActionForInvokeExceptionEdge(inlineInfo);
         boolean partialIntrinsicExit = false;
         if (intrinsicContext != null && intrinsicContext.isCallToOriginal(targetMethod)) {
             partialIntrinsicExit = true;
@@ -1614,7 +1619,7 @@ public class BytecodeParser implements GraphBuilderContext {
                 // must use the same context as the call to the intrinsic.
                 invokeBci = intrinsicCallSiteParser.bci();
                 profile = intrinsicCallSiteParser.getProfileForInvoke(invokeKind);
-                withExceptionEdge = !intrinsicCallSiteParser.omitInvokeExceptionEdge(inlineInfo);
+                edgeAction = intrinsicCallSiteParser.getActionForInvokeExceptionEdge(inlineInfo);
             } else {
                 // We are parsing the intrinsic for the root compilation or for inlining,
                 // This call is a partial intrinsic exit, and we do not have profile information
@@ -1624,7 +1629,7 @@ public class BytecodeParser implements GraphBuilderContext {
                 assert intrinsicContext.isPostParseInlined();
                 invokeBci = BytecodeFrame.UNKNOWN_BCI;
                 profile = null;
-                withExceptionEdge = graph.method().getAnnotation(Snippet.class) == null;
+                edgeAction = graph.method().getAnnotation(Snippet.class) == null ? ExceptionEdgeAction.INCLUDE_AND_HANDLE : ExceptionEdgeAction.OMIT;
             }
 
             if (originalMethod.isStatic()) {
@@ -1637,10 +1642,10 @@ public class BytecodeParser implements GraphBuilderContext {
             Signature sig = originalMethod.getSignature();
             returnType = sig.getReturnType(method.getDeclaringClass());
             resultType = sig.getReturnKind();
-            assert checkPartialIntrinsicExit(intrinsicCallSiteParser == null ? null : intrinsicCallSiteParser.currentInvoke.args, args);
+            assert intrinsicContext.allowPartialIntrinsicArgumentMismatch() || checkPartialIntrinsicExit(intrinsicCallSiteParser == null ? null : intrinsicCallSiteParser.currentInvoke.args, args);
             targetMethod = originalMethod;
         }
-        Invoke invoke = createNonInlinedInvoke(withExceptionEdge, invokeBci, args, targetMethod, invokeKind, resultType, returnType, profile);
+        Invoke invoke = createNonInlinedInvoke(edgeAction, invokeBci, args, targetMethod, invokeKind, resultType, returnType, profile);
         if (partialIntrinsicExit) {
             // This invoke must never be later inlined as it might select the intrinsic graph.
             // Until there is a mechanism to guarantee that any late inlining will not select
@@ -1698,14 +1703,14 @@ public class BytecodeParser implements GraphBuilderContext {
         } else {
             for (int i = 0; i < recursiveArgs.length; i++) {
                 ValueNode arg = GraphUtil.unproxify(recursiveArgs[i]);
-                assert arg instanceof ParameterNode && ((ParameterNode) arg).index() == i : String.format("argument %d of call denoting partial intrinsic exit should be a %s with index %d, not %s", i,
-                                ParameterNode.class.getSimpleName(), i, arg);
+                assert arg instanceof ParameterNode && ((ParameterNode) arg).index() == i : String.format("argument %d of call denoting partial intrinsic exit should be a %s with index %d, not %s",
+                                i, ParameterNode.class.getSimpleName(), i, arg);
             }
         }
         return true;
     }
 
-    protected Invoke createNonInlinedInvoke(boolean withExceptionEdge, int invokeBci, ValueNode[] invokeArgs, ResolvedJavaMethod targetMethod,
+    protected Invoke createNonInlinedInvoke(ExceptionEdgeAction exceptionEdge, int invokeBci, ValueNode[] invokeArgs, ResolvedJavaMethod targetMethod,
                     InvokeKind invokeKind, JavaKind resultType, JavaType returnType, JavaTypeProfile profile) {
 
         StampPair returnStamp = graphBuilderConfig.getPlugins().getOverridingStamp(this, returnType, false);
@@ -1714,7 +1719,7 @@ public class BytecodeParser implements GraphBuilderContext {
         }
 
         MethodCallTargetNode callTarget = graph.add(createMethodCallTarget(invokeKind, targetMethod, invokeArgs, returnStamp, profile));
-        Invoke invoke = createNonInlinedInvoke(withExceptionEdge, invokeBci, callTarget, resultType);
+        Invoke invoke = createNonInlinedInvoke(exceptionEdge, invokeBci, callTarget, resultType);
 
         for (InlineInvokePlugin plugin : graphBuilderConfig.getPlugins().getInlineInvokePlugins()) {
             plugin.notifyNotInlined(this, targetMethod, invoke);
@@ -1723,11 +1728,11 @@ public class BytecodeParser implements GraphBuilderContext {
         return invoke;
     }
 
-    protected Invoke createNonInlinedInvoke(boolean withExceptionEdge, int invokeBci, CallTargetNode callTarget, JavaKind resultType) {
-        if (!withExceptionEdge) {
+    protected Invoke createNonInlinedInvoke(ExceptionEdgeAction exceptionEdge, int invokeBci, CallTargetNode callTarget, JavaKind resultType) {
+        if (exceptionEdge == ExceptionEdgeAction.OMIT) {
             return createInvoke(invokeBci, callTarget, resultType);
         } else {
-            Invoke invoke = createInvokeWithException(invokeBci, callTarget, resultType);
+            Invoke invoke = createInvokeWithException(invokeBci, callTarget, resultType, exceptionEdge);
             AbstractBeginNode beginNode = graph.add(KillingBeginNode.create(LocationIdentity.any()));
             invoke.setNext(beginNode);
             lastInstr = beginNode;
@@ -1736,20 +1741,29 @@ public class BytecodeParser implements GraphBuilderContext {
     }
 
     /**
-     * If the method returns true, the invocation of the given {@link MethodCallTargetNode call
-     * target} does not need an exception edge.
+     * Describes what should be done with the exception edge of an invocation. The edge can be
+     * omitted or included. An included edge can handle the exception or transfer execution to the
+     * interpreter for handling (deoptimize).
      */
-    protected boolean omitInvokeExceptionEdge(InlineInfo lastInlineInfo) {
+    protected enum ExceptionEdgeAction {
+        OMIT,
+        INCLUDE_AND_HANDLE,
+        INCLUDE_AND_DEOPTIMIZE
+    }
+
+    protected ExceptionEdgeAction getActionForInvokeExceptionEdge(InlineInfo lastInlineInfo) {
         if (lastInlineInfo == InlineInfo.DO_NOT_INLINE_WITH_EXCEPTION) {
-            return false;
+            return ExceptionEdgeAction.INCLUDE_AND_HANDLE;
         } else if (lastInlineInfo == InlineInfo.DO_NOT_INLINE_NO_EXCEPTION) {
-            return true;
+            return ExceptionEdgeAction.OMIT;
+        } else if (lastInlineInfo == InlineInfo.DO_NOT_INLINE_DEOPTIMIZE_ON_EXCEPTION) {
+            return ExceptionEdgeAction.INCLUDE_AND_DEOPTIMIZE;
         } else if (graphBuilderConfig.getBytecodeExceptionMode() == BytecodeExceptionMode.CheckAll) {
-            return false;
+            return ExceptionEdgeAction.INCLUDE_AND_HANDLE;
         } else if (graphBuilderConfig.getBytecodeExceptionMode() == BytecodeExceptionMode.ExplicitOnly) {
-            return false;
+            return ExceptionEdgeAction.INCLUDE_AND_HANDLE;
         } else if (graphBuilderConfig.getBytecodeExceptionMode() == BytecodeExceptionMode.OmitAll) {
-            return true;
+            return ExceptionEdgeAction.OMIT;
         } else {
             assert graphBuilderConfig.getBytecodeExceptionMode() == BytecodeExceptionMode.Profile;
             // be conservative if information was not recorded (could result in endless
@@ -1759,12 +1773,12 @@ public class BytecodeParser implements GraphBuilderContext {
                     if (profilingInfo != null) {
                         TriState exceptionSeen = profilingInfo.getExceptionSeen(bci());
                         if (exceptionSeen == TriState.FALSE) {
-                            return true;
+                            return ExceptionEdgeAction.OMIT;
                         }
                     }
                 }
             }
-            return false;
+            return ExceptionEdgeAction.INCLUDE_AND_HANDLE;
         }
     }
 
@@ -1887,7 +1901,7 @@ public class BytecodeParser implements GraphBuilderContext {
                     if (newProfile != profile) {
                         if (newProfile.getTypes().length == 0) {
                             // All profiled types select the intrinsic so
-                            // emit a fixed guard instead of a if-then-else.
+                            // emit a fixed guard instead of an if-then-else.
                             lastInstr = append(new FixedGuardNode(compare, TypeCheckedInliningViolated, InvalidateReprofile, false));
                             return new IntrinsicGuard(currentLastInstr, intrinsicReceiver, mark, null, null);
                         }
@@ -1966,7 +1980,7 @@ public class BytecodeParser implements GraphBuilderContext {
                     }
 
                     lastInstr = intrinsicGuard.nonIntrinsicBranch;
-                    createNonInlinedInvoke(omitInvokeExceptionEdge(null), bci(), args, targetMethod, invokeKind, resultType, returnType, intrinsicGuard.profile);
+                    createNonInlinedInvoke(getActionForInvokeExceptionEdge(null), bci(), args, targetMethod, invokeKind, resultType, returnType, intrinsicGuard.profile);
 
                     EndNode nonIntrinsicEnd = append(new EndNode());
                     AbstractMergeNode mergeNode = graph.add(new MergeNode());
@@ -2303,7 +2317,7 @@ public class BytecodeParser implements GraphBuilderContext {
             if (calleeBeforeUnwindNode != null) {
                 ValueNode calleeUnwindValue = parser.getUnwindValue();
                 assert calleeUnwindValue != null;
-                calleeBeforeUnwindNode.setNext(handleException(calleeUnwindValue, bci()));
+                calleeBeforeUnwindNode.setNext(handleException(calleeUnwindValue, bci(), false));
             }
         }
     }
@@ -2319,7 +2333,7 @@ public class BytecodeParser implements GraphBuilderContext {
         return invoke;
     }
 
-    protected InvokeWithExceptionNode createInvokeWithException(int invokeBci, CallTargetNode callTarget, JavaKind resultType) {
+    protected InvokeWithExceptionNode createInvokeWithException(int invokeBci, CallTargetNode callTarget, JavaKind resultType, ExceptionEdgeAction exceptionEdgeAction) {
         if (currentBlock != null && stream.nextBCI() > currentBlock.endBci) {
             /*
              * Clear non-live locals early so that the exception handler entry gets the cleared
@@ -2328,7 +2342,7 @@ public class BytecodeParser implements GraphBuilderContext {
             frameState.clearNonLiveLocals(currentBlock, liveness, false);
         }
 
-        AbstractBeginNode exceptionEdge = handleException(null, bci());
+        AbstractBeginNode exceptionEdge = handleException(null, bci(), exceptionEdgeAction == ExceptionEdgeAction.INCLUDE_AND_DEOPTIMIZE);
         InvokeWithExceptionNode invoke = append(new InvokeWithExceptionNode(callTarget, exceptionEdge, invokeBci));
         frameState.pushReturn(resultType, invoke);
         invoke.setStateAfter(createFrameState(stream.nextBCI(), invoke));
@@ -2359,20 +2373,43 @@ public class BytecodeParser implements GraphBuilderContext {
             }
         }
 
+        ValueNode realReturnVal = processReturnValue(returnVal, returnKind);
+
         frameState.setRethrowException(false);
         frameState.clearStack();
-        beforeReturn(returnVal, returnKind);
+        beforeReturn(realReturnVal, returnKind);
         if (parent == null) {
-            append(new ReturnNode(returnVal));
+            append(new ReturnNode(realReturnVal));
         } else {
             if (returnDataList == null) {
                 returnDataList = new ArrayList<>();
             }
-            returnDataList.add(new ReturnToCallerData(returnVal, lastInstr));
+            returnDataList.add(new ReturnToCallerData(realReturnVal, lastInstr));
             lastInstr = null;
         }
     }
 
+    private ValueNode processReturnValue(ValueNode value, JavaKind kind) {
+        JavaKind returnKind = method.getSignature().getReturnKind();
+        if (kind != returnKind) {
+            // sub-word integer
+            assert returnKind.isNumericInteger() && returnKind.getStackKind() == JavaKind.Int;
+            IntegerStamp stamp = (IntegerStamp) value.stamp();
+
+            // the bytecode verifier doesn't check that the value is in the correct range
+            if (stamp.lowerBound() < returnKind.getMinValue() || returnKind.getMaxValue() < stamp.upperBound()) {
+                ValueNode narrow = append(genNarrow(value, returnKind.getBitCount()));
+                if (returnKind.isUnsigned()) {
+                    return append(genZeroExtend(narrow, 32));
+                } else {
+                    return append(genSignExtend(narrow, 32));
+                }
+            }
+        }
+
+        return value;
+    }
+
     private void beforeReturn(ValueNode x, JavaKind kind) {
         if (graph.method() != null && graph.method().isJavaLangObjectInit()) {
             /*
@@ -2794,6 +2831,8 @@ public class BytecodeParser implements GraphBuilderContext {
     }
 
     private void createExceptionDispatch(ExceptionDispatchBlock block) {
+        lastInstr = finishInstruction(lastInstr, frameState);
+
         assert frameState.stackSize() == 1 : frameState;
         if (block.handler.isCatchAll()) {
             assert block.getSuccessorCount() == 1;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/FrameStateBuilder.java

@@ -207,7 +207,7 @@ public final class FrameStateBuilder implements SideEffectsState {
                 receiver = new ParameterNode(javaIndex, receiverStamp);
             }
 
-            locals[javaIndex] = graph.addOrUnique(receiver);
+            locals[javaIndex] = graph.addOrUniqueWithInputs(receiver);
             javaIndex = 1;
             index = 1;
         }
@@ -241,7 +241,7 @@ public final class FrameStateBuilder implements SideEffectsState {
                 param = new ParameterNode(index, stamp);
             }
 
-            locals[javaIndex] = graph.addOrUnique(param);
+            locals[javaIndex] = graph.addOrUniqueWithInputs(param);
             javaIndex++;
             if (kind.needsTwoSlots()) {
                 locals[javaIndex] = TWO_SLOT_MARKER;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/optimize/ConditionalElimination02.java

@@ -29,6 +29,9 @@ import jdk.vm.ci.meta.DeoptimizationReason;
 import org.junit.Test;
 
 import org.graalvm.compiler.jtt.JTTTest;
+import org.graalvm.compiler.phases.OptimisticOptimizations;
+import org.graalvm.compiler.phases.OptimisticOptimizations.Optimization;
+import org.graalvm.compiler.phases.tiers.HighTierContext;
 
 public class ConditionalElimination02 extends JTTTest {
 
@@ -59,6 +62,14 @@ public class ConditionalElimination02 extends JTTTest {
         return -1;
     }
 
+    /**
+     * These tests assume all code paths are reachable so disable profile based dead code removal.
+     */
+    @Override
+    protected HighTierContext getDefaultHighTierContext() {
+        return new HighTierContext(getProviders(), getDefaultGraphBuilderSuite(), OptimisticOptimizations.ALL.remove(Optimization.RemoveNeverExecutedCode));
+    }
+
     @Test
     public void run0() throws Throwable {
         runTest(EnumSet.of(DeoptimizationReason.NullCheckException), "test", new A(5), false, false);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/optimize/Fold_Double04.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2017, 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.jtt.optimize;
+
+import org.junit.Test;
+
+import org.graalvm.compiler.jtt.JTTTest;
+
+/*
+ * Tests constant folding of double operations.
+ */
+public class Fold_Double04 extends JTTTest {
+
+    // Contrived check whether both arguments are the same kind of zero
+    public static boolean test(double x, double y) {
+        if (x == 0) {
+            if (1 / x == Double.NEGATIVE_INFINITY) {
+                return 1 / y == Double.NEGATIVE_INFINITY;
+            } else {
+                return 1 / y == Double.POSITIVE_INFINITY;
+            }
+        }
+        return false;
+    }
+
+    @Test
+    public void run0() throws Throwable {
+        runTest("test", -0d, -0d);
+    }
+
+    @Test
+    public void run1() throws Throwable {
+        runTest("test", -0d, 0d);
+    }
+
+    @Test
+    public void run2() throws Throwable {
+        runTest("test", 0d, -0d);
+    }
+
+    @Test
+    public void run3() throws Throwable {
+        runTest("test", 0d, 0d);
+    }
+
+}