8202670: Update Graal

Reviewed-by: kvn, aph

make/CompileJavaModules.gmk

@@ -444,11 +444,13 @@ jdk.internal.vm.compiler_ADD_JAVAC_FLAGS += -parameters -XDstringConcat=inline \
 
 jdk.internal.vm.compiler_EXCLUDES += \
     jdk.internal.vm.compiler.collections.test \
+    org.graalvm.compiler.processor \
     org.graalvm.compiler.core.match.processor \
     org.graalvm.compiler.nodeinfo.processor \
     org.graalvm.compiler.options.processor \
     org.graalvm.compiler.serviceprovider.processor \
-    org.graalvm.compiler.replacements.verifier \
+    org.graalvm.compiler.replacements.processor \
+    org.graalvm.compiler.replacements.jdk9.test \
     org.graalvm.compiler.api.directives.test \
     org.graalvm.compiler.api.test \
     org.graalvm.compiler.asm.aarch64.test \

make/CompileToolsHotspot.gmk

@@ -47,34 +47,8 @@ ifeq ($(INCLUDE_GRAAL), true)
   $(eval $(call SetupJavaCompilation, BUILD_VM_COMPILER_MATCH_PROCESSOR, \
       SETUP := GENERATE_OLDBYTECODE, \
       SRC := \
-          $(SRC_DIR)/jdk.internal.vm.compiler.word/src \
+          $(SRC_DIR)/org.graalvm.compiler.processor/src \
-          $(SRC_DIR)/jdk.internal.vm.compiler.collections/src \
-          $(SRC_DIR)/org.graalvm.compiler.core/src \
-          $(SRC_DIR)/org.graalvm.compiler.core.common/src \
           $(SRC_DIR)/org.graalvm.compiler.core.match.processor/src \
-          $(SRC_DIR)/org.graalvm.compiler.api.replacements/src \
-          $(SRC_DIR)/org.graalvm.compiler.asm/src \
-          $(SRC_DIR)/org.graalvm.compiler.bytecode/src \
-          $(SRC_DIR)/org.graalvm.compiler.code/src \
-          $(SRC_DIR)/org.graalvm.compiler.debug/src \
-          $(SRC_DIR)/org.graalvm.compiler.graph/src \
-          $(SRC_DIR)/org.graalvm.compiler.lir/src \
-          $(SRC_DIR)/org.graalvm.compiler.loop/src \
-          $(SRC_DIR)/org.graalvm.compiler.loop.phases/src \
-          $(SRC_DIR)/org.graalvm.compiler.nodeinfo/src \
-          $(SRC_DIR)/org.graalvm.compiler.nodes/src \
-          $(SRC_DIR)/org.graalvm.compiler.options/src \
-          $(SRC_DIR)/org.graalvm.compiler.phases/src \
-          $(SRC_DIR)/org.graalvm.compiler.phases.common/src \
-          $(SRC_DIR)/org.graalvm.compiler.serviceprovider/src \
-          $(SRC_DIR)/org.graalvm.compiler.virtual/src \
-          $(SRC_DIR)/org.graalvm.graphio/src \
-          $(SRC_DIR)/org.graalvm.util/src \
-          $(VM_CI_SRC_DIR)/jdk.vm.ci.code/src \
-          $(VM_CI_SRC_DIR)/jdk.vm.ci.common/src \
-          $(VM_CI_SRC_DIR)/jdk.vm.ci.meta/src \
-          $(VM_CI_SRC_DIR)/jdk.vm.ci.runtime/src \
-          $(VM_CI_SRC_DIR)/jdk.vm.ci.services/src \
           , \
       EXCLUDE_FILES := $(EXCLUDE_FILES), \
       BIN := $(BUILDTOOLS_OUTPUTDIR)/jdk.vm.compiler.match.processor, \
@@ -88,7 +62,7 @@ ifeq ($(INCLUDE_GRAAL), true)
   $(eval $(call SetupJavaCompilation, BUILD_VM_COMPILER_NODEINFO_PROCESSOR, \
       SETUP := GENERATE_OLDBYTECODE, \
       SRC := \
-          $(SRC_DIR)/org.graalvm.compiler.nodeinfo/src \
+          $(SRC_DIR)/org.graalvm.compiler.processor/src \
           $(SRC_DIR)/org.graalvm.compiler.nodeinfo.processor/src \
           , \
       BIN := $(BUILDTOOLS_OUTPUTDIR)/jdk.vm.compiler.nodeinfo.processor, \
@@ -102,10 +76,8 @@ ifeq ($(INCLUDE_GRAAL), true)
   $(eval $(call SetupJavaCompilation, BUILD_VM_COMPILER_OPTIONS_PROCESSOR, \
       SETUP := GENERATE_OLDBYTECODE, \
       SRC := \
-          $(SRC_DIR)/jdk.internal.vm.compiler.collections/src \
+          $(SRC_DIR)/org.graalvm.compiler.processor/src \
-          $(SRC_DIR)/org.graalvm.compiler.options/src \
           $(SRC_DIR)/org.graalvm.compiler.options.processor/src \
-          $(SRC_DIR)/org.graalvm.util/src \
           , \
       BIN := $(BUILDTOOLS_OUTPUTDIR)/jdk.vm.compiler.options.processor, \
       JAR := $(BUILDTOOLS_OUTPUTDIR)/jdk.vm.compiler.options.processor.jar, \
@@ -115,44 +87,26 @@ ifeq ($(INCLUDE_GRAAL), true)
 
   ##############################################################################
 
-  $(eval $(call SetupJavaCompilation, BUILD_VM_COMPILER_REPLACEMENTS_VERIFIER, \
+  $(eval $(call SetupJavaCompilation, BUILD_VM_COMPILER_REPLACEMENTS_PROCESSOR, \
       SETUP := GENERATE_OLDBYTECODE, \
       SRC := \
-          $(SRC_DIR)/jdk.internal.vm.compiler.word/src \
+          $(SRC_DIR)/org.graalvm.compiler.processor/src \
-          $(SRC_DIR)/jdk.internal.vm.compiler.collections/src \
+          $(SRC_DIR)/org.graalvm.compiler.replacements.processor/src \
-          $(SRC_DIR)/org.graalvm.compiler.bytecode/src \
-          $(SRC_DIR)/org.graalvm.compiler.replacements.verifier/src \
-          $(SRC_DIR)/org.graalvm.compiler.api.replacements/src \
-          $(SRC_DIR)/org.graalvm.compiler.code/src \
-          $(SRC_DIR)/org.graalvm.compiler.core.common/src \
-          $(SRC_DIR)/org.graalvm.compiler.debug/src \
-          $(SRC_DIR)/org.graalvm.compiler.graph/src \
-          $(SRC_DIR)/org.graalvm.compiler.nodeinfo/src \
-          $(SRC_DIR)/org.graalvm.compiler.options/src \
-          $(SRC_DIR)/org.graalvm.compiler.serviceprovider/src \
-          $(SRC_DIR)/org.graalvm.graphio/src \
-          $(SRC_DIR)/org.graalvm.util/src \
-          $(VM_CI_SRC_DIR)/jdk.vm.ci.code/src \
-          $(VM_CI_SRC_DIR)/jdk.vm.ci.common/src \
-          $(VM_CI_SRC_DIR)/jdk.vm.ci.meta/src \
-          $(VM_CI_SRC_DIR)/jdk.vm.ci.runtime/src \
-          $(VM_CI_SRC_DIR)/jdk.vm.ci.services/src \
           , \
       EXCLUDE_FILES := $(EXCLUDE_FILES), \
       BIN := $(BUILDTOOLS_OUTPUTDIR)/jdk.vm.compiler.replacements.verifier, \
       JAR := $(BUILDTOOLS_OUTPUTDIR)/jdk.vm.compiler.replacements.verifier.jar, \
   ))
 
-  TARGETS += $(BUILD_VM_COMPILER_REPLACEMENTS_VERIFIER)
+  TARGETS += $(BUILD_VM_COMPILER_REPLACEMENTS_PROCESSOR)
 
   ##############################################################################
 
   $(eval $(call SetupJavaCompilation, BUILD_VM_COMPILER_SERVICEPROVIDER_PROCESSOR, \
       SETUP := GENERATE_OLDBYTECODE, \
       SRC := \
-          $(SRC_DIR)/org.graalvm.compiler.serviceprovider/src \
+          $(SRC_DIR)/org.graalvm.compiler.processor/src \
           $(SRC_DIR)/org.graalvm.compiler.serviceprovider.processor/src \
-          $(VM_CI_SRC_DIR)/jdk.vm.ci.services/src \
           , \
       EXCLUDE_FILES := $(EXCLUDE_FILES), \
       BIN := $(BUILDTOOLS_OUTPUTDIR)/jdk.vm.compiler.serviceprovider.processor, \

src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/AOTBackend.java

@@ -139,7 +139,7 @@ final class AOTBackend {
             CompilationResult compilationResult = new CompilationResult(id, isImmutablePIC);
 
             return GraalCompiler.compileGraph(graph, resolvedMethod, providers, backend, graphBuilderSuite, OptimisticOptimizations.ALL, profilingInfo, getSuites(), getLirSuites(),
-                            compilationResult, CompilationResultBuilderFactory.Default);
+                            compilationResult, CompilationResultBuilderFactory.Default, true);
 
         } catch (Throwable e) {
             main.handleError(resolvedMethod, e, " (compiling graph)");

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64Assembler.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2016, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2018, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -63,11 +63,15 @@ import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FMOV
 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FMSUB;
 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FMUL;
 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FNEG;
+import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FRINTM;
+import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FRINTN;
+import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FRINTP;
 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FRINTZ;
 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FSQRT;
 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FSUB;
 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.HINT;
 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.HLT;
+import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.LDADD;
 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.LDAR;
 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.LDAXR;
 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.LDP;
@@ -480,6 +484,9 @@ public abstract class AArch64Assembler extends Assembler {
     private static final int CASAcquireOffset = 22;
     private static final int CASReleaseOffset = 15;
 
+    private static final int LDADDAcquireOffset = 23;
+    private static final int LDADDReleaseOffset = 22;
+
     /**
      * Encoding for all instructions.
      */
@@ -511,6 +518,7 @@ public abstract class AArch64Assembler extends Assembler {
         STP(0b0 << 22),
 
         CAS(0x08A07C00),
+        LDADD(0x38200000),
 
         ADR(0x00000000),
         ADRP(0x80000000),
@@ -573,6 +581,9 @@ public abstract class AArch64Assembler extends Assembler {
         FSQRT(0x00018000),
         FNEG(0x00010000),
 
+        FRINTM(0x00050000),
+        FRINTN(0x00040000),
+        FRINTP(0x00048000),
         FRINTZ(0x00058000),
 
         FADD(0x00002000),
@@ -1330,7 +1341,18 @@ public abstract class AArch64Assembler extends Assembler {
         emitInt(transferSizeEncoding | instr.encoding | rs2(rs) | rn(rn) | rt(rt));
     }
 
-    /* Compare And Swap */
+    /**
+     * Compare And Swap word or doubleword in memory. This reads a value from an address rn,
+     * compares it against a given value rs, and, if equal, stores the value rt to memory. The value
+     * read from address rn is stored in register rs.
+     *
+     * @param size size of bits read from memory. Must be 32 or 64.
+     * @param rs general purpose register to be compared and loaded. May not be null.
+     * @param rt general purpose register to be conditionally stored. May not be null.
+     * @param rn general purpose register containing the address from which to read.
+     * @param acquire boolean value signifying if the load should use acquire semantics.
+     * @param release boolean value signifying if the store should use release semantics.
+     */
     public void cas(int size, Register rs, Register rt, Register rn, boolean acquire, boolean release) {
         assert size == 32 || size == 64;
         int transferSize = NumUtil.log2Ceil(size / 8);
@@ -1344,17 +1366,42 @@ public abstract class AArch64Assembler extends Assembler {
         emitInt(transferSizeEncoding | instr.encoding | rs2(rs) | rn(rn) | rt(rt) | (acquire ? 1 : 0) << CASAcquireOffset | (release ? 1 : 0) << CASReleaseOffset);
     }
 
+    /**
+     * Atomic add. This reads a value from an address rn, stores the value in rt, and adds the value
+     * in rs to it, and stores the result back at address rn. The initial value read from memory is
+     * stored in rt.
+     *
+     * @param size size of operand to read from memory. Must be 8, 16, 32, or 64.
+     * @param rs general purpose register to be added to contents. May not be null.
+     * @param rt general purpose register to be loaded. May not be null.
+     * @param rn general purpose register or stack pointer holding an address from which to load.
+     * @param acquire boolean value signifying if the load should use acquire semantics.
+     * @param release boolean value signifying if the store should use release semantics.
+     */
+    public void ldadd(int size, Register rs, Register rt, Register rn, boolean acquire, boolean release) {
+        assert size == 8 || size == 16 || size == 32 || size == 64;
+        int transferSize = NumUtil.log2Ceil(size / 8);
+        loadAndAddInstruction(LDADD, rs, rt, rn, transferSize, acquire, release);
+    }
+
+    private void loadAndAddInstruction(Instruction instr, Register rs, Register rt, Register rn, int log2TransferSize, boolean acquire, boolean release) {
+        assert log2TransferSize >= 0 && log2TransferSize < 4;
+        assert rt.getRegisterCategory().equals(CPU) && rs.getRegisterCategory().equals(CPU) && !rs.equals(rt);
+        int transferSizeEncoding = log2TransferSize << LoadStoreTransferSizeOffset;
+        emitInt(transferSizeEncoding | instr.encoding | rs2(rs) | rn(rn) | rt(rt) | (acquire ? 1 : 0) << LDADDAcquireOffset | (release ? 1 : 0) << LDADDReleaseOffset);
+    }
+
     /* PC-relative Address Calculation (5.4.4) */
 
     /**
      * Address of page: sign extends 21-bit offset, shifts if left by 12 and adds it to the value of
-     * the PC with its bottom 12-bits cleared, writing the result to dst.
+     * the PC with its bottom 12-bits cleared, writing the result to dst. No offset is emitted; the
-     * No offset is emiited; the instruction will be patched later.
+     * instruction will be patched later.
      *
      * @param dst general purpose register. May not be null, zero-register or stackpointer.
      */
     public void adrp(Register dst) {
-        emitInt(ADRP.encoding | PcRelImmOp | rd(dst) );
+        emitInt(ADRP.encoding | PcRelImmOp | rd(dst));
     }
 
     /**
@@ -1367,14 +1414,17 @@ public abstract class AArch64Assembler extends Assembler {
         emitInt(ADR.encoding | PcRelImmOp | rd(dst) | getPcRelativeImmEncoding(imm21));
     }
 
+    /**
+     * Adds a 21-bit signed offset to the program counter and writes the result to dst.
+     *
+     * @param dst general purpose register. May not be null, zero-register or stackpointer.
+     * @param imm21 Signed 21-bit offset.
+     * @param pos the position in the code that the instruction is emitted.
+     */
     public void adr(Register dst, int imm21, int pos) {
         emitInt(ADR.encoding | PcRelImmOp | rd(dst) | getPcRelativeImmEncoding(imm21), pos);
     }
 
-    public void adrp(Register dst, int pageOffset) {
-        emitInt(ADRP.encoding | PcRelImmOp | rd(dst) | getPcRelativeImmEncoding(pageOffset));
-    }
-
     private static int getPcRelativeImmEncoding(int imm21) {
         assert NumUtil.isSignedNbit(21, imm21);
         int imm = imm21 & NumUtil.getNbitNumberInt(21);
@@ -2432,6 +2482,39 @@ public abstract class AArch64Assembler extends Assembler {
         fpDataProcessing1Source(FRINTZ, dst, src, floatFromSize(size));
     }
 
+    /**
+     * Rounds floating-point to integral. Rounds towards nearest with ties to even.
+     *
+     * @param size register size.
+     * @param dst floating point register. May not be null.
+     * @param src floating point register. May not be null.
+     */
+    public void frintn(int size, Register dst, Register src) {
+        fpDataProcessing1Source(FRINTN, dst, src, floatFromSize(size));
+    }
+
+    /**
+     * Rounds floating-point to integral. Rounds towards minus infinity.
+     *
+     * @param size register size.
+     * @param dst floating point register. May not be null.
+     * @param src floating point register. May not be null.
+     */
+    public void frintm(int size, Register dst, Register src) {
+        fpDataProcessing1Source(FRINTM, dst, src, floatFromSize(size));
+    }
+
+    /**
+     * Rounds floating-point to integral. Rounds towards plus infinity.
+     *
+     * @param size register size.
+     * @param dst floating point register. May not be null.
+     * @param src floating point register. May not be null.
+     */
+    public void frintp(int size, Register dst, Register src) {
+        fpDataProcessing1Source(FRINTP, dst, src, floatFromSize(size));
+    }
+
     /* Floating-point Arithmetic (1 source) (5.7.6) */
 
     /**

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64MacroAssembler.java

@@ -1,6 +1,5 @@
 /*
  * Copyright (c) 2013, 2016, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2018, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -38,7 +37,6 @@ import static jdk.vm.ci.aarch64.AArch64.r9;
 import static jdk.vm.ci.aarch64.AArch64.sp;
 import static jdk.vm.ci.aarch64.AArch64.zr;
 
-import org.graalvm.compiler.asm.AbstractAddress;
 import org.graalvm.compiler.asm.Label;
 import org.graalvm.compiler.core.common.NumUtil;
 import org.graalvm.compiler.debug.GraalError;
@@ -307,9 +305,10 @@ public class AArch64MacroAssembler extends AArch64Assembler {
             case EXTENDED_REGISTER_OFFSET:
                 add(64, dst, address.getBase(), address.getOffset(), address.getExtendType(), address.isScaled() ? shiftAmt : 0);
                 break;
-            case PC_LITERAL:
+            case PC_LITERAL: {
-                super.adr(dst, address.getImmediateRaw());
+                addressOf(dst);
                 break;
+            }
             case BASE_REGISTER_ONLY:
                 movx(dst, address.getBase());
                 break;
@@ -1561,7 +1560,7 @@ public class AArch64MacroAssembler extends AArch64Assembler {
     }
 
     @Override
-    public AbstractAddress getPlaceholder(int instructionStartPosition) {
+    public AArch64Address getPlaceholder(int instructionStartPosition) {
         return AArch64Address.PLACEHOLDER;
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Assembler.java

@@ -1853,9 +1853,36 @@ public class AMD64Assembler extends Assembler {
         CMP.getMIOpcode(DWORD, isByte(imm32)).emit(this, DWORD, dst, imm32);
     }
 
-    // The 32-bit cmpxchg compares the value at adr with the contents of X86.rax,
+    /**
-    // and stores reg into adr if so; otherwise, the value at adr is loaded into X86.rax,.
+     * The 8-bit cmpxchg compares the value at adr with the contents of X86.rax, and stores reg into
-    // The ZF is set if the compared values were equal, and cleared otherwise.
+     * adr if so; otherwise, the value at adr is loaded into X86.rax,. The ZF is set if the compared
+     * values were equal, and cleared otherwise.
+     */
+    public final void cmpxchgb(Register reg, AMD64Address adr) { // cmpxchg
+        prefix(adr, reg);
+        emitByte(0x0F);
+        emitByte(0xB0);
+        emitOperandHelper(reg, adr, 0);
+    }
+
+    /**
+     * The 16-bit cmpxchg compares the value at adr with the contents of X86.rax, and stores reg
+     * into adr if so; otherwise, the value at adr is loaded into X86.rax,. The ZF is set if the
+     * compared values were equal, and cleared otherwise.
+     */
+    public final void cmpxchgw(Register reg, AMD64Address adr) { // cmpxchg
+        emitByte(0x66); // Switch to 16-bit mode.
+        prefix(adr, reg);
+        emitByte(0x0F);
+        emitByte(0xB1);
+        emitOperandHelper(reg, adr, 0);
+    }
+
+    /**
+     * The 32-bit cmpxchg compares the value at adr with the contents of X86.rax, and stores reg
+     * into adr if so; otherwise, the value at adr is loaded into X86.rax,. The ZF is set if the
+     * compared values were equal, and cleared otherwise.
+     */
     public final void cmpxchgl(Register reg, AMD64Address adr) { // cmpxchg
         prefix(adr, reg);
         emitByte(0x0F);
@@ -3677,6 +3704,21 @@ public class AMD64Assembler extends Assembler {
         emitByte(imm8);
     }
 
+    public final void xaddb(AMD64Address dst, Register src) {
+        prefix(dst, src);
+        emitByte(0x0F);
+        emitByte(0xC0);
+        emitOperandHelper(src, dst, 0);
+    }
+
+    public final void xaddw(AMD64Address dst, Register src) {
+        emitByte(0x66); // Switch to 16-bit mode.
+        prefix(dst, src);
+        emitByte(0x0F);
+        emitByte(0xC1);
+        emitOperandHelper(src, dst, 0);
+    }
+
     public final void xaddl(AMD64Address dst, Register src) {
         prefix(dst, src);
         emitByte(0x0F);
@@ -3691,6 +3733,19 @@ public class AMD64Assembler extends Assembler {
         emitOperandHelper(src, dst, 0);
     }
 
+    public final void xchgb(Register dst, AMD64Address src) {
+        prefix(src, dst);
+        emitByte(0x86);
+        emitOperandHelper(dst, src, 0);
+    }
+
+    public final void xchgw(Register dst, AMD64Address src) {
+        emitByte(0x66);
+        prefix(src, dst);
+        emitByte(0x87);
+        emitOperandHelper(dst, src, 0);
+    }
+
     public final void xchgl(Register dst, AMD64Address src) {
         prefix(src, dst);
         emitByte(0x87);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.bytecode/src/org/graalvm/compiler/bytecode/Bytecodes.java

@@ -837,4 +837,27 @@ public class Bytecodes {
 
         assert !isConditionalBranch(opcode) || isBranch(opcode) : "a conditional branch must also be a branch";
     }
+
+    public static boolean isIfBytecode(int bytecode) {
+        switch (bytecode) {
+            case IFEQ:
+            case IFNE:
+            case IFLT:
+            case IFGE:
+            case IFGT:
+            case IFLE:
+            case IF_ICMPEQ:
+            case IF_ICMPNE:
+            case IF_ICMPLT:
+            case IF_ICMPGE:
+            case IF_ICMPGT:
+            case IF_ICMPLE:
+            case IF_ACMPEQ:
+            case IF_ACMPNE:
+            case IFNULL:
+            case IFNONNULL:
+                return true;
+        }
+        return false;
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64ArithmeticLIRGenerator.java

@@ -32,9 +32,9 @@ import static org.graalvm.compiler.lir.aarch64.AArch64BitManipulationOp.BitManip
 import static org.graalvm.compiler.lir.aarch64.AArch64BitManipulationOp.BitManipulationOpCode.CLZ;
 import static org.graalvm.compiler.lir.aarch64.AArch64BitManipulationOp.BitManipulationOpCode.CTZ;
 
-import org.graalvm.compiler.core.common.NumUtil;
 import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler;
 import org.graalvm.compiler.core.common.LIRKind;
+import org.graalvm.compiler.core.common.NumUtil;
 import org.graalvm.compiler.core.common.calc.FloatConvert;
 import org.graalvm.compiler.debug.GraalError;
 import org.graalvm.compiler.lir.ConstantValue;
@@ -491,4 +491,23 @@ public class AArch64ArithmeticLIRGenerator extends ArithmeticLIRGenerator implem
         throw GraalError.unimplemented();
     }
 
+    @Override
+    public Value emitRound(Value value, RoundingMode mode) {
+        AArch64ArithmeticOp op;
+        switch (mode) {
+            case NEAREST:
+                op = AArch64ArithmeticOp.FRINTN;
+                break;
+            case UP:
+                op = AArch64ArithmeticOp.FRINTP;
+                break;
+            case DOWN:
+                op = AArch64ArithmeticOp.FRINTM;
+                break;
+            default:
+                throw GraalError.shouldNotReachHere();
+        }
+
+        return emitUnary(op, value);
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64LIRGenerator.java

@@ -52,6 +52,7 @@ import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.CondMoveOp;
 import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.StrategySwitchOp;
 import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.TableSwitchOp;
 import org.graalvm.compiler.lir.aarch64.AArch64Move;
+import org.graalvm.compiler.lir.aarch64.AArch64AtomicMove.AtomicReadAndAddOp;
 import org.graalvm.compiler.lir.aarch64.AArch64AtomicMove.CompareAndSwapOp;
 import org.graalvm.compiler.lir.aarch64.AArch64Move.MembarOp;
 import org.graalvm.compiler.lir.aarch64.AArch64PauseOp;
@@ -127,7 +128,7 @@ public abstract class AArch64LIRGenerator extends LIRGenerator {
     }
 
     @Override
-    public Variable emitLogicCompareAndSwap(Value address, Value expectedValue, Value newValue, Value trueValue, Value falseValue) {
+    public Variable emitLogicCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue, Value trueValue, Value falseValue) {
         Variable prevValue = newVariable(expectedValue.getValueKind());
         Variable scratch = newVariable(LIRKind.value(AArch64Kind.DWORD));
         append(new CompareAndSwapOp(prevValue, loadReg(expectedValue), loadReg(newValue), asAllocatable(address), scratch));
@@ -138,13 +139,23 @@ public abstract class AArch64LIRGenerator extends LIRGenerator {
     }
 
     @Override
-    public Variable emitValueCompareAndSwap(Value address, Value expectedValue, Value newValue) {
+    public Variable emitValueCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue) {
         Variable result = newVariable(newValue.getValueKind());
         Variable scratch = newVariable(LIRKind.value(AArch64Kind.WORD));
         append(new CompareAndSwapOp(result, loadNonCompareConst(expectedValue), loadReg(newValue), asAllocatable(address), scratch));
         return result;
     }
 
+    @Override
+    public Value emitAtomicReadAndAdd(Value address, ValueKind<?> kind, Value delta) {
+        Variable result = newVariable(kind);
+        Variable scratch1 = newVariable(kind);
+        Variable scratch2 = newVariable(kind);
+
+        append(new AtomicReadAndAddOp((AArch64Kind) kind.getPlatformKind(), asAllocatable(result), asAllocatable(address), asAllocatable(delta), asAllocatable(scratch1), asAllocatable(scratch2)));
+        return result;
+    }
+
     @Override
     public void emitMembar(int barriers) {
         int necessaryBarriers = target().arch.requiredBarriers(barriers);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64LIRGenerator.java

@@ -23,13 +23,16 @@
 
 package org.graalvm.compiler.core.amd64;
 
+import static jdk.vm.ci.code.ValueUtil.asRegister;
 import static jdk.vm.ci.code.ValueUtil.isAllocatableValue;
+import static jdk.vm.ci.code.ValueUtil.isRegister;
 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64BinaryArithmetic.CMP;
 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.OperandSize.DWORD;
 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.OperandSize.PD;
 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.OperandSize.PS;
 import static org.graalvm.compiler.asm.amd64.AMD64Assembler.OperandSize.QWORD;
 import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
+import static org.graalvm.compiler.lir.LIRValueUtil.asConstant;
 import static org.graalvm.compiler.lir.LIRValueUtil.asConstantValue;
 import static org.graalvm.compiler.lir.LIRValueUtil.asJavaConstant;
 import static org.graalvm.compiler.lir.LIRValueUtil.isConstantValue;
@@ -190,36 +193,68 @@ public abstract class AMD64LIRGenerator extends LIRGenerator {
         }
     }
 
-    @Override
+    private AllocatableValue asAllocatable(Value value, ValueKind<?> kind) {
-    public Variable emitLogicCompareAndSwap(Value address, Value expectedValue, Value newValue, Value trueValue, Value falseValue) {
+        if (value.getValueKind().equals(kind)) {
+            return asAllocatable(value);
+        } else if (isRegister(value)) {
+            return asRegister(value).asValue(kind);
+        } else if (isConstantValue(value)) {
+            return emitLoadConstant(kind, asConstant(value));
+        } else {
+            Variable variable = newVariable(kind);
+            emitMove(variable, value);
+            return variable;
+        }
+    }
+
+    private Value emitCompareAndSwap(boolean isLogic, LIRKind accessKind, Value address, Value expectedValue, Value newValue, Value trueValue, Value falseValue) {
         ValueKind<?> kind = newValue.getValueKind();
         assert kind.equals(expectedValue.getValueKind());
-        AMD64Kind memKind = (AMD64Kind) kind.getPlatformKind();
 
         AMD64AddressValue addressValue = asAddressValue(address);
-        RegisterValue raxRes = AMD64.rax.asValue(kind);
+        LIRKind integralAccessKind = accessKind;
-        emitMove(raxRes, expectedValue);
+        Value reinterpretedExpectedValue = expectedValue;
-        append(new CompareAndSwapOp(memKind, raxRes, addressValue, raxRes, asAllocatable(newValue)));
+        Value reinterpretedNewValue = newValue;
+        boolean isXmm = ((AMD64Kind) accessKind.getPlatformKind()).isXMM();
+        if (isXmm) {
+            if (accessKind.getPlatformKind().equals(AMD64Kind.SINGLE)) {
+                integralAccessKind = LIRKind.fromJavaKind(target().arch, JavaKind.Int);
+            } else {
+                integralAccessKind = LIRKind.fromJavaKind(target().arch, JavaKind.Long);
+            }
+            reinterpretedExpectedValue = arithmeticLIRGen.emitReinterpret(integralAccessKind, expectedValue);
+            reinterpretedNewValue = arithmeticLIRGen.emitReinterpret(integralAccessKind, newValue);
+        }
+        AMD64Kind memKind = (AMD64Kind) integralAccessKind.getPlatformKind();
+        RegisterValue aRes = AMD64.rax.asValue(integralAccessKind);
+        AllocatableValue allocatableNewValue = asAllocatable(reinterpretedNewValue, integralAccessKind);
+        emitMove(aRes, reinterpretedExpectedValue);
+        append(new CompareAndSwapOp(memKind, aRes, addressValue, aRes, allocatableNewValue));
 
-        assert trueValue.getValueKind().equals(falseValue.getValueKind());
+        if (isLogic) {
-        Variable result = newVariable(trueValue.getValueKind());
+            assert trueValue.getValueKind().equals(falseValue.getValueKind());
-        append(new CondMoveOp(result, Condition.EQ, asAllocatable(trueValue), falseValue));
+            Variable result = newVariable(trueValue.getValueKind());
-        return result;
+            append(new CondMoveOp(result, Condition.EQ, asAllocatable(trueValue), falseValue));
+            return result;
+        } else {
+            if (isXmm) {
+                return arithmeticLIRGen.emitReinterpret(accessKind, aRes);
+            } else {
+                Variable result = newVariable(kind);
+                emitMove(result, aRes);
+                return result;
+            }
+        }
     }
 
     @Override
-    public Value emitValueCompareAndSwap(Value address, Value expectedValue, Value newValue) {
+    public Variable emitLogicCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue, Value trueValue, Value falseValue) {
-        ValueKind<?> kind = newValue.getValueKind();
+        return (Variable) emitCompareAndSwap(true, accessKind, address, expectedValue, newValue, trueValue, falseValue);
-        assert kind.equals(expectedValue.getValueKind());
+    }
-        AMD64Kind memKind = (AMD64Kind) kind.getPlatformKind();
 
-        AMD64AddressValue addressValue = asAddressValue(address);
+    @Override
-        RegisterValue raxRes = AMD64.rax.asValue(kind);
+    public Value emitValueCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue) {
-        emitMove(raxRes, expectedValue);
+        return emitCompareAndSwap(false, accessKind, address, expectedValue, newValue, null, null);
-        append(new CompareAndSwapOp(memKind, raxRes, addressValue, raxRes, asAllocatable(newValue)));
-        Variable result = newVariable(kind);
-        emitMove(result, raxRes);
-        return result;
     }
 
     public void emitCompareAndSwapBranch(ValueKind<?> kind, AMD64AddressValue address, Value expectedValue, Value newValue, Condition condition, LabelRef trueLabel, LabelRef falseLabel,
@@ -235,8 +270,7 @@ public abstract class AMD64LIRGenerator extends LIRGenerator {
     }
 
     @Override
-    public Value emitAtomicReadAndAdd(Value address, Value delta) {
+    public Value emitAtomicReadAndAdd(Value address, ValueKind<?> kind, Value delta) {
-        ValueKind<?> kind = delta.getValueKind();
         Variable result = newVariable(kind);
         AMD64AddressValue addressValue = asAddressValue(address);
         append(new AMD64Move.AtomicReadAndAddOp((AMD64Kind) kind.getPlatformKind(), result, addressValue, asAllocatable(delta)));
@@ -244,8 +278,7 @@ public abstract class AMD64LIRGenerator extends LIRGenerator {
     }
 
     @Override
-    public Value emitAtomicReadAndWrite(Value address, Value newValue) {
+    public Value emitAtomicReadAndWrite(Value address, ValueKind<?> kind, Value newValue) {
-        ValueKind<?> kind = newValue.getValueKind();
         Variable result = newVariable(kind);
         AMD64AddressValue addressValue = asAddressValue(address);
         append(new AMD64Move.AtomicReadAndWriteOp((AMD64Kind) kind.getPlatformKind(), result, addressValue, asAllocatable(newValue)));

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/GraalBailoutException.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.core.common;
+
+import jdk.vm.ci.code.BailoutException;
+import org.graalvm.compiler.debug.CausableByCompilerAssert;
+
+@SuppressWarnings("serial")
+public class GraalBailoutException extends BailoutException implements CausableByCompilerAssert {
+
+    public GraalBailoutException(String format, Object... args) {
+        super(format, args);
+    }
+
+    public GraalBailoutException(Throwable cause, String format, Object... args) {
+        super(cause, format, args);
+    }
+
+    public GraalBailoutException(boolean permanent, String format, Object... args) {
+        super(permanent, format, args);
+    }
+
+    @Override
+    public boolean isCausedByCompilerAssert() {
+        return false;
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/PermanentBailoutException.java

@@ -22,9 +22,7 @@
  */
 package org.graalvm.compiler.core.common;
 
-import jdk.vm.ci.code.BailoutException;
+public class PermanentBailoutException extends GraalBailoutException {
-
-public class PermanentBailoutException extends BailoutException {
 
     private static final long serialVersionUID = -2683649650135362549L;
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/RetryableBailoutException.java

@@ -22,9 +22,7 @@
  */
 package org.graalvm.compiler.core.common;
 
-import jdk.vm.ci.code.BailoutException;
+public class RetryableBailoutException extends GraalBailoutException {
-
-public class RetryableBailoutException extends BailoutException {
 
     private static final long serialVersionUID = -7145365025679144525L;
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/IntegerStamp.java

@@ -1545,6 +1545,9 @@ public final class IntegerStamp extends PrimitiveStamp {
 
                         @Override
                         public Stamp invertStamp(int inputBits, int resultBits, Stamp outStamp) {
+                            if (outStamp.isEmpty()) {
+                                return StampFactory.forInteger(inputBits).empty();
+                            }
                             IntegerStamp stamp = (IntegerStamp) outStamp;
                             long mask = CodeUtil.mask(inputBits);
                             return StampFactory.forIntegerWithMask(inputBits, stamp.lowerBound(), stamp.upperBound(), stamp.downMask() & mask, stamp.upMask() & mask);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.match.processor/src/org/graalvm/compiler/core/match/processor/MatchProcessor.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,19 +27,20 @@ import java.io.IOException;
 import java.io.PrintWriter;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import javax.annotation.processing.AbstractProcessor;
 import javax.annotation.processing.Filer;
 import javax.annotation.processing.RoundEnvironment;
 import javax.annotation.processing.SupportedAnnotationTypes;
 import javax.lang.model.SourceVersion;
 import javax.lang.model.element.AnnotationMirror;
-import javax.lang.model.element.AnnotationValue;
 import javax.lang.model.element.Element;
 import javax.lang.model.element.ElementKind;
 import javax.lang.model.element.ExecutableElement;
@@ -48,9 +49,7 @@ import javax.lang.model.element.Name;
 import javax.lang.model.element.PackageElement;
 import javax.lang.model.element.TypeElement;
 import javax.lang.model.element.VariableElement;
-import javax.lang.model.type.MirroredTypeException;
 import javax.lang.model.type.TypeMirror;
-import javax.lang.model.util.AbstractAnnotationValueVisitor7;
 import javax.lang.model.util.ElementFilter;
 import javax.lang.model.util.Types;
 import javax.tools.Diagnostic.Kind;
@@ -58,24 +57,10 @@ import javax.tools.FileObject;
 import javax.tools.JavaFileObject;
 import javax.tools.StandardLocation;
 
-import jdk.internal.vm.compiler.collections.EconomicMap;
+import org.graalvm.compiler.processor.AbstractProcessor;
-import jdk.internal.vm.compiler.collections.EconomicSet;
-import jdk.internal.vm.compiler.collections.Equivalence;
-import org.graalvm.compiler.core.gen.NodeMatchRules;
-import org.graalvm.compiler.core.match.ComplexMatchResult;
-import org.graalvm.compiler.core.match.MatchRule;
-import org.graalvm.compiler.core.match.MatchRules;
-import org.graalvm.compiler.core.match.MatchStatement;
-import org.graalvm.compiler.core.match.MatchStatementSet;
-import org.graalvm.compiler.core.match.MatchableNode;
-import org.graalvm.compiler.core.match.MatchableNodes;
-import org.graalvm.compiler.debug.GraalError;
-import org.graalvm.compiler.graph.Position;
-import org.graalvm.compiler.nodes.ValueNode;
-import org.graalvm.compiler.serviceprovider.ServiceProvider;
 
 /**
- * Processes classes annotated with {@link MatchRule}. A {@link MatchStatementSet} service is
+ * Processes classes annotated with {@code MatchRule}. A {@code MatchStatementSet} service is
  * generated for each top level class containing at least one such field. These service objects can
  * be retrieved as follows:
  *
@@ -90,6 +75,13 @@ import org.graalvm.compiler.serviceprovider.ServiceProvider;
                 "org.graalvm.compiler.core.match.MatchableNodes"})
 public class MatchProcessor extends AbstractProcessor {
 
+    private static final String VALUE_NODE_CLASS_NAME = "org.graalvm.compiler.nodes.ValueNode";
+    private static final String COMPLEX_MATCH_RESULT_CLASS_NAME = "org.graalvm.compiler.core.match.ComplexMatchResult";
+    private static final String MATCHABLE_NODES_CLASS_NAME = "org.graalvm.compiler.core.match.MatchableNodes";
+    private static final String MATCHABLE_NODE_CLASS_NAME = "org.graalvm.compiler.core.match.MatchableNode";
+    private static final String MATCH_RULE_CLASS_NAME = "org.graalvm.compiler.core.match.MatchRule";
+    private static final String MATCH_RULES_CLASS_NAME = "org.graalvm.compiler.core.match.MatchRules";
+
     public MatchProcessor() {
     }
 
@@ -98,8 +90,8 @@ public class MatchProcessor extends AbstractProcessor {
         return SourceVersion.latest();
     }
 
-    private final Set<Element> processedMatchRule = new HashSet<>();
+    private final Set<Element> processedMatchRules = new HashSet<>();
-    private final Set<Element> processedMatchableNode = new HashSet<>();
+    private final Set<Element> processedMatchableNodes = new HashSet<>();
 
     private static class RuleParseError extends RuntimeException {
         private static final long serialVersionUID = 6456128283609257490L;
@@ -170,14 +162,14 @@ public class MatchProcessor extends AbstractProcessor {
             if (peek("(").equals("(")) {
                 next();
                 MatchDescriptor descriptor = parseType(true);
-                for (int n = 0; n < descriptor.nodeType.inputs.length; n++) {
+                for (int n = 0; n < descriptor.nodeType.inputs.size(); n++) {
                     if (peek("(").equals("(")) {
                         descriptor.inputs[n] = parseExpression();
                     } else {
                         descriptor.inputs[n] = parseType(false);
                     }
                 }
-                for (int n = 0; n < descriptor.nodeType.inputs.length; n++) {
+                for (int n = 0; n < descriptor.nodeType.inputs.size(); n++) {
                     if (descriptor.inputs[n] == null) {
                         throw new RuleParseError("not enough inputs for " + descriptor.name);
                     }
@@ -233,7 +225,7 @@ public class MatchProcessor extends AbstractProcessor {
         /**
          * Recursively accumulate any required Position declarations.
          */
-        void generatePositionDeclarations(EconomicSet<String> declarations) {
+        void generatePositionDeclarations(Set<String> declarations) {
             matchDescriptor.generatePositionDeclarations(declarations);
         }
 
@@ -251,9 +243,8 @@ public class MatchProcessor extends AbstractProcessor {
     }
 
     /**
-     * Set to true to enable logging to a local file during annotation processing. There's no normal
+     * Set to true to enable logging during annotation processing. There's no normal channel for any
-     * channel for any debug messages and debugging annotation processors requires some special
+     * debug messages and debugging annotation processors requires some special setup.
-     * setup.
      */
     private static final boolean DEBUG = false;
 
@@ -265,14 +256,21 @@ public class MatchProcessor extends AbstractProcessor {
 
     private PrintWriter getLog() {
         if (log == null) {
-            try {
+            if (processingEnv.getClass().getName().contains(".javac.")) {
-                // Create the log file within the generated source directory so it's easy to find.
+                // For javac, just log to System.err
-                // /tmp isn't platform independent and java.io.tmpdir can map anywhere, particularly
+                log = new PrintWriter(System.err);
-                // on the mac.
+            } else {
-                FileObject file = processingEnv.getFiler().createResource(StandardLocation.SOURCE_OUTPUT, "", getClass().getSimpleName() + "log");
+                try {
-                log = new PrintWriter(new FileWriter(file.toUri().getPath(), true));
+                    // Create the log file within the generated source directory so it's easy to
-            } catch (IOException e) {
+                    // find.
-                // Do nothing
+                    // /tmp isn't platform independent and java.io.tmpdir can map anywhere,
+                    // particularly
+                    // on the mac.
+                    FileObject file = processingEnv.getFiler().createResource(StandardLocation.SOURCE_OUTPUT, "", getClass().getSimpleName() + "log");
+                    log = new PrintWriter(new FileWriter(file.toUri().getPath(), true));
+                } catch (IOException e) {
+                    // Do nothing
+                }
             }
         }
         return log;
@@ -309,7 +307,7 @@ public class MatchProcessor extends AbstractProcessor {
             logMessage("throw for %s:\n", element);
         }
         logException(t);
-        errorMessage(element, "Exception throw during processing: %s %s", t, Arrays.toString(Arrays.copyOf(t.getStackTrace(), 4)));
+        printError(element, "Exception throw during processing: %s %s", t, Arrays.toString(Arrays.copyOf(t.getStackTrace(), 4)));
     }
 
     static class TypeDescriptor {
@@ -321,19 +319,19 @@ public class MatchProcessor extends AbstractProcessor {
         final String shortName;
 
         /**
-         * The simple name of the {@link ValueNode} class represented by this type.
+         * The simple name of the {@code ValueNode} class represented by this type.
          */
         final String nodeClass;
 
         /**
-         * The package of {@link ValueNode} class represented by this type.
+         * The package of {@code ValueNode} class represented by this type.
          */
         final String nodePackage;
 
         /**
          * The matchable inputs of the node.
          */
-        final String[] inputs;
+        final List<String> inputs;
 
         /**
          * Should swapped variants of this match be generated. The user of the match is expected to
@@ -350,7 +348,7 @@ public class MatchProcessor extends AbstractProcessor {
 
         final Set<Element> originatingElements = new HashSet<>();
 
-        TypeDescriptor(TypeMirror mirror, String shortName, String nodeClass, String nodePackage, String[] inputs, boolean commutative, boolean shareable) {
+        TypeDescriptor(TypeMirror mirror, String shortName, String nodeClass, String nodePackage, List<String> inputs, boolean commutative, boolean shareable) {
             this.mirror = mirror;
             this.shortName = shortName;
             this.nodeClass = nodeClass;
@@ -358,26 +356,18 @@ public class MatchProcessor extends AbstractProcessor {
             this.inputs = inputs;
             this.commutative = commutative;
             this.shareable = shareable;
-            assert !commutative || inputs.length == 2;
+            assert !commutative || inputs.size() == 2;
         }
     }
 
     /**
      * The types which are know for purpose of parsing MatchRule expressions.
      */
-    EconomicMap<String, TypeDescriptor> knownTypes = EconomicMap.create(Equivalence.DEFAULT);
+    Map<String, TypeDescriptor> knownTypes = new HashMap<>();
 
     private TypeDescriptor valueType;
 
-    private TypeMirror matchRulesTypeMirror;
+    private void declareType(TypeMirror mirror, String shortName, String nodeClass, String nodePackage, List<String> inputs, boolean commutative, boolean shareable, Element element) {
-
-    private TypeMirror matchRuleTypeMirror;
-
-    private TypeMirror matchableNodeTypeMirror;
-
-    private TypeMirror matchableNodesTypeMirror;
-
-    private void declareType(TypeMirror mirror, String shortName, String nodeClass, String nodePackage, String[] inputs, boolean commutative, boolean shareable, Element element) {
         TypeDescriptor descriptor = new TypeDescriptor(mirror, shortName, nodeClass, nodePackage, inputs, commutative, shareable);
         descriptor.originatingElements.add(element);
         knownTypes.put(shortName, descriptor);
@@ -388,7 +378,7 @@ public class MatchProcessor extends AbstractProcessor {
         if (p != null) {
             return p.getQualifiedName().toString();
         }
-        throw new GraalError("can't find package for %s", type);
+        throw new InternalError("Can't find package for " + type);
     }
 
     class MatchDescriptor {
@@ -400,13 +390,13 @@ public class MatchProcessor extends AbstractProcessor {
             this.nodeType = nodeType;
             this.name = name;
             if (forExpression) {
-                this.inputs = new MatchDescriptor[nodeType.inputs.length];
+                this.inputs = new MatchDescriptor[nodeType.inputs.size()];
             } else {
                 this.inputs = new MatchDescriptor[0];
             }
         }
 
-        public void generatePositionDeclarations(EconomicSet<String> declarations) {
+        public void generatePositionDeclarations(Set<String> declarations) {
             if (inputs.length == 0) {
                 return;
             }
@@ -469,10 +459,10 @@ public class MatchProcessor extends AbstractProcessor {
 
         private String formatSuffix() {
             if (nodeType != null) {
-                if (inputs.length != nodeType.inputs.length) {
+                if (inputs.length != nodeType.inputs.size()) {
                     return ", true)";
                 } else {
-                    if (nodeType.inputs.length > 0) {
+                    if (nodeType.inputs.size() > 0) {
                         return ", " + nodeType.nodeClass + "_positions, " + !nodeType.shareable + ")";
                     }
                     if (nodeType.shareable) {
@@ -502,7 +492,7 @@ public class MatchProcessor extends AbstractProcessor {
         String pkg = ((PackageElement) info.topDeclaringType.getEnclosingElement()).getQualifiedName().toString();
         Name topDeclaringClass = info.topDeclaringType.getSimpleName();
 
-        String matchStatementClassName = topDeclaringClass + "_" + MatchStatementSet.class.getSimpleName();
+        String matchStatementClassName = topDeclaringClass + "_MatchStatementSet";
         Element[] originatingElements = info.originatingElements.toArray(new Element[info.originatingElements.size()]);
 
         Types typeUtils = typeUtils();
@@ -516,22 +506,20 @@ public class MatchProcessor extends AbstractProcessor {
             out.println("package " + pkg + ";");
             out.println("");
             out.println("import java.util.*;");
-            out.println("import " + MatchStatementSet.class.getPackage().getName() + ".*;");
+            out.println("import org.graalvm.compiler.core.match.*;");
-            out.println("import " + NodeMatchRules.class.getName() + ";");
+            out.println("import org.graalvm.compiler.core.gen.NodeMatchRules;");
-            out.println("import " + Position.class.getName() + ";");
+            out.println("import org.graalvm.compiler.graph.Position;");
-            out.println("import " + ServiceProvider.class.getName() + ";");
             for (String p : info.requiredPackages) {
                 out.println("import " + p + ".*;");
             }
             out.println("");
 
-            out.println("@" + ServiceProvider.class.getSimpleName() + "(" + MatchStatementSet.class.getSimpleName() + ".class)");
+            out.println("public class " + matchStatementClassName + " implements MatchStatementSet {");
-            out.println("public class " + matchStatementClassName + " implements " + MatchStatementSet.class.getSimpleName() + " {");
 
             out.println();
 
             // Generate declarations for the wrapper class to invoke the code generation methods.
-            for (MethodInvokerItem invoker : info.invokers.getValues()) {
+            for (MethodInvokerItem invoker : info.invokers.values()) {
                 StringBuilder args = new StringBuilder();
                 StringBuilder types = new StringBuilder();
                 int count = invoker.fields.size();
@@ -562,7 +550,7 @@ public class MatchProcessor extends AbstractProcessor {
 
             }
 
-            String desc = MatchStatement.class.getSimpleName();
+            String desc = "MatchStatement";
 
             out.println("    @Override");
             out.println("    public Class<? extends NodeMatchRules> forClass() {");
@@ -595,6 +583,7 @@ public class MatchProcessor extends AbstractProcessor {
 
             out.println("}");
         }
+        this.createProviderFile(pkg + "." + matchStatementClassName, "org.graalvm.compiler.core.match.MatchStatementSet", originatingElements);
     }
 
     protected PrintWriter createSourceFile(String pkg, String relativeName, Filer filer, Element... originatingElements) {
@@ -662,17 +651,17 @@ public class MatchProcessor extends AbstractProcessor {
 
         final TypeElement topDeclaringType;
         final List<MatchRuleItem> matchRules = new ArrayList<>();
-        private final EconomicSet<Element> originatingElements = EconomicSet.create(Equivalence.DEFAULT);
+        private final Set<Element> originatingElements = new HashSet<>();
-        public EconomicSet<String> positionDeclarations = EconomicSet.create(Equivalence.DEFAULT);
+        public Set<String> positionDeclarations = new HashSet<>();
 
         /**
          * The mapping between elements with MatchRules and the wrapper class used invoke the code
          * generation after the match.
          */
-        EconomicMap<String, MethodInvokerItem> invokers = EconomicMap.create(Equivalence.DEFAULT);
+        Map<String, MethodInvokerItem> invokers = new HashMap<>();
 
         /**
-         * The set of packages which must be imported to refer the classes mention in matchRules.
+         * The set of packages which must be imported to refer the classes mentioned in matchRules.
          */
         Set<String> requiredPackages = new HashSet<>();
 
@@ -690,14 +679,15 @@ public class MatchProcessor extends AbstractProcessor {
         return topDeclaringType(enclosing);
     }
 
-    private AnnotationMirror findAnnotationMirror(Element element, TypeMirror typeMirror) {
+    /**
-        for (AnnotationMirror mirror : element.getAnnotationMirrors()) {
+     * The element currently being processed.
-            if (typeUtils().isSameType(mirror.getAnnotationType(), typeMirror)) {
+     */
-                return mirror;
+    private Element currentElement;
-            }
+
-        }
+    /**
-        return null;
+     * The current processing round.
-    }
+     */
+    private RoundEnvironment currentRound;
 
     @Override
     public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment roundEnv) {
@@ -706,45 +696,58 @@ public class MatchProcessor extends AbstractProcessor {
         }
 
         logMessage("Starting round %s\n", roundEnv);
-        matchRulesTypeMirror = processingEnv.getElementUtils().getTypeElement(MatchRules.class.getCanonicalName()).asType();
-        matchRuleTypeMirror = processingEnv.getElementUtils().getTypeElement(MatchRule.class.getCanonicalName()).asType();
 
-        matchableNodeTypeMirror = processingEnv.getElementUtils().getTypeElement(MatchableNode.class.getCanonicalName()).asType();
+        TypeElement matchRulesTypeElement = getTypeElement(MATCH_RULES_CLASS_NAME);
-        matchableNodesTypeMirror = processingEnv.getElementUtils().getTypeElement(MatchableNodes.class.getCanonicalName()).asType();
+        TypeElement matchRuleTypeElement = getTypeElement(MATCH_RULE_CLASS_NAME);
 
-        Element currentElement = null;
+        TypeMirror matchRulesTypeMirror = matchRulesTypeElement.asType();
+        TypeMirror matchRuleTypeMirror = matchRuleTypeElement.asType();
+
+        TypeElement matchableNodeTypeElement = getTypeElement(MATCHABLE_NODE_CLASS_NAME);
+        TypeElement matchableNodesTypeElement = getTypeElement(MATCHABLE_NODES_CLASS_NAME);
+
+        currentRound = roundEnv;
         try {
-            for (Element element : roundEnv.getElementsAnnotatedWith(MatchableNode.class)) {
+            for (Element element : roundEnv.getElementsAnnotatedWith(matchableNodeTypeElement)) {
+                currentElement = element;
                 logMessage("%s\n", element);
-                processMatchableNode(element);
+                processMatchableNodes(element);
             }
-            for (Element element : roundEnv.getElementsAnnotatedWith(MatchableNodes.class)) {
+            for (Element element : roundEnv.getElementsAnnotatedWith(matchableNodesTypeElement)) {
+                currentElement = element;
                 logMessage("%s\n", element);
-                processMatchableNode(element);
+                processMatchableNodes(element);
             }
             // Define a TypeDescriptor for the generic node but don't enter it into the nodeTypes
             // table since it shouldn't be mentioned in match rules.
-            TypeMirror valueTypeMirror = processingEnv.getElementUtils().getTypeElement(ValueNode.class.getName()).asType();
+            TypeMirror valueTypeMirror = getTypeElement(VALUE_NODE_CLASS_NAME).asType();
-            valueType = new TypeDescriptor(valueTypeMirror, "Value", ValueNode.class.getSimpleName(), ValueNode.class.getPackage().getName(), new String[0], false, false);
+            valueType = new TypeDescriptor(valueTypeMirror, "Value", "ValueNode", "org.graalvm.compiler.nodes", Collections.emptyList(), false, false);
 
-            EconomicMap<TypeElement, MatchRuleDescriptor> map = EconomicMap.create(Equivalence.DEFAULT);
+            Map<TypeElement, MatchRuleDescriptor> map = new HashMap<>();
 
-            for (Element element : roundEnv.getElementsAnnotatedWith(MatchRule.class)) {
+            for (Element element : roundEnv.getElementsAnnotatedWith(matchRuleTypeElement)) {
                 currentElement = element;
-                processMatchRule(map, element, findAnnotationMirror(element, matchRuleTypeMirror));
+                AnnotationMirror matchRule = getAnnotation(element, matchRuleTypeMirror);
+                List<AnnotationMirror> matchRuleAnnotations = Collections.singletonList(matchRule);
+                processMatchRules(map, element, matchRuleAnnotations);
             }
-            for (Element element : roundEnv.getElementsAnnotatedWith(MatchRules.class)) {
+            for (Element element : roundEnv.getElementsAnnotatedWith(matchRulesTypeElement)) {
                 currentElement = element;
-                processMatchRule(map, element, findAnnotationMirror(element, matchRulesTypeMirror));
+                AnnotationMirror matchRules = getAnnotation(element, matchRulesTypeMirror);
+                List<AnnotationMirror> matchRuleAnnotations = getAnnotationValueList(matchRules, "value", AnnotationMirror.class);
+                processMatchRules(map, element, matchRuleAnnotations);
             }
 
             currentElement = null;
-            for (MatchRuleDescriptor info : map.getValues()) {
+            for (MatchRuleDescriptor info : map.values()) {
                 createFiles(info);
             }
 
         } catch (Throwable t) {
             reportExceptionThrow(currentElement, t);
+        } finally {
+            currentElement = null;
+            currentRound = null;
         }
 
         return true;
@@ -753,27 +756,27 @@ public class MatchProcessor extends AbstractProcessor {
     /**
      * Build up the type table to be used during parsing of the MatchRule.
      */
-    private void processMatchableNode(Element element) {
+    private void processMatchableNodes(Element element) {
-        if (!processedMatchableNode.contains(element)) {
+        if (!processedMatchableNodes.contains(element)) {
             try {
-                processedMatchableNode.add(element);
+                processedMatchableNodes.add(element);
 
-                AnnotationMirror mirror = findAnnotationMirror(element, matchableNodesTypeMirror);
+                List<AnnotationMirror> matchableNodeAnnotations;
-                if (mirror == null) {
+                AnnotationMirror mirror = getAnnotation(element, getType(MATCHABLE_NODES_CLASS_NAME));
-                    mirror = findAnnotationMirror(element, matchableNodeTypeMirror);
+                if (mirror != null) {
-                }
+                    matchableNodeAnnotations = getAnnotationValueList(mirror, "value", AnnotationMirror.class);
-                if (mirror == null) {
+                } else {
-                    return;
+                    mirror = getAnnotation(element, getType(MATCHABLE_NODES_CLASS_NAME));
+                    if (mirror != null) {
+                        matchableNodeAnnotations = Collections.singletonList(mirror);
+                    } else {
+                        return;
+                    }
                 }
+
                 TypeElement topDeclaringType = topDeclaringType(element);
-                List<AnnotationMirror> mirrors = null;
+                for (AnnotationMirror matchableNode : matchableNodeAnnotations) {
-                if (typeUtils().isSameType(mirror.getAnnotationType(), matchableNodesTypeMirror)) {
+                    processMatchableNode(element, topDeclaringType, matchableNode);
-                    // Unpack the mirrors for a repeatable annotation
-                    mirrors = getAnnotationValueList(AnnotationMirror.class, mirror, "value");
-                }
-                int i = 0;
-                for (MatchableNode matchableNode : element.getAnnotationsByType(MatchableNode.class)) {
-                    processMatchableNode(element, topDeclaringType, matchableNode, mirrors != null ? mirrors.get(i++) : mirror);
                 }
             } catch (Throwable t) {
                 reportExceptionThrow(element, t);
@@ -781,27 +784,22 @@ public class MatchProcessor extends AbstractProcessor {
         }
     }
 
-    private void processMatchableNode(Element element, TypeElement topDeclaringType, MatchableNode matchable, AnnotationMirror mirror) throws GraalError {
+    private void processMatchableNode(Element element, TypeElement topDeclaringType, AnnotationMirror matchable) {
         logMessage("processMatchableNode %s %s %s\n", topDeclaringType, element, matchable);
         String nodeClass;
         String nodePackage;
-        TypeMirror nodeClassMirror = null;
+        TypeMirror nodeClassMirror = getAnnotationValue(matchable, "nodeClass", TypeMirror.class);
-        try {
-            matchable.nodeClass();
-        } catch (MirroredTypeException e) {
-            nodeClassMirror = e.getTypeMirror();
-        }
         if (nodeClassMirror == null) {
-            throw new GraalError("Can't get mirror for node class %s", element);
+            throw new InternalError("Can't get mirror for node class " + element);
         }
-        if (nodeClassMirror.toString().equals(MatchableNode.class.getName())) {
+        if (nodeClassMirror.toString().equals(MATCHABLE_NODE_CLASS_NAME)) {
             nodeClass = topDeclaringType.getQualifiedName().toString();
         } else {
             nodeClass = nodeClassMirror.toString();
         }
         TypeElement typeElement = processingEnv.getElementUtils().getTypeElement(nodeClass);
         if (typeElement == null) {
-            errorMessage(element, mirror, "Class \"%s\" cannot be resolved to a type", nodeClass);
+            printError(element, matchable, "Class \"%s\" cannot be resolved to a type", nodeClass);
             return;
         }
         nodePackage = findPackage(typeElement);
@@ -812,7 +810,8 @@ public class MatchProcessor extends AbstractProcessor {
 
         Types typeUtils = processingEnv.getTypeUtils();
         TypeElement nodeClassElement = (TypeElement) typeUtils.asElement(nodeClassMirror);
-        for (String input : matchable.inputs()) {
+        List<String> inputs = getAnnotationValueList(matchable, "inputs", String.class);
+        for (String input : inputs) {
             boolean ok = false;
             TypeElement current = nodeClassElement;
             while (!ok && current != null) {
@@ -826,17 +825,19 @@ public class MatchProcessor extends AbstractProcessor {
                 current = (TypeElement) typeUtils.asElement(theSuper);
             }
             if (!ok) {
-                errorMessage(element, mirror, "Input named \"%s\" doesn't exist in %s", input, nodeClassElement.getSimpleName());
+                printError(element, matchable, "Input named \"%s\" doesn't exist in %s", input, nodeClassElement.getSimpleName());
             }
         }
 
-        declareType(nodeClassMirror, shortName, nodeClass, nodePackage, matchable.inputs(), matchable.commutative(), matchable.shareable(), element);
+        boolean commutative = getAnnotationValue(matchable, "commutative", Boolean.class);
+        boolean shareable = getAnnotationValue(matchable, "shareable", Boolean.class);
+        declareType(nodeClassMirror, shortName, nodeClass, nodePackage, inputs, commutative, shareable, element);
     }
 
-    private void processMatchRule(EconomicMap<TypeElement, MatchRuleDescriptor> map, Element element, AnnotationMirror mirror) {
+    private void processMatchRules(Map<TypeElement, MatchRuleDescriptor> map, Element element, List<AnnotationMirror> matchRules) {
-        if (!processedMatchRule.contains(element)) {
+        if (!processedMatchRules.contains(element)) {
             try {
-                processedMatchRule.add(element);
+                processedMatchRules.add(element);
 
                 // The annotation element type should ensure this is true.
                 assert element instanceof ExecutableElement;
@@ -849,14 +850,8 @@ public class MatchProcessor extends AbstractProcessor {
                     info = new MatchRuleDescriptor(topDeclaringType);
                     map.put(topDeclaringType, info);
                 }
-                List<AnnotationMirror> mirrors = null;
+                for (AnnotationMirror matchRule : matchRules) {
-                if (typeUtils().isSameType(mirror.getAnnotationType(), matchRulesTypeMirror)) {
+                    processMatchRule((ExecutableElement) element, info, matchRule);
-                    // Unpack the mirrors for a repeatable annotation
-                    mirrors = getAnnotationValueList(AnnotationMirror.class, mirror, "value");
-                }
-                int i = 0;
-                for (MatchRule matchRule : element.getAnnotationsByType(MatchRule.class)) {
-                    processMethodMatchRule((ExecutableElement) element, info, matchRule, mirrors != null ? mirrors.get(i++) : mirror);
                 }
             } catch (Throwable t) {
                 reportExceptionThrow(element, t);
@@ -871,16 +866,16 @@ public class MatchProcessor extends AbstractProcessor {
      * @param element
      */
     private void findMatchableNodes(Element element) {
-        processMatchableNode(element);
+        processMatchableNodes(element);
         Element enclosing = element.getEnclosingElement();
         while (enclosing != null) {
             if (enclosing.getKind() == ElementKind.CLASS || enclosing.getKind() == ElementKind.INTERFACE) {
                 TypeElement current = (TypeElement) enclosing;
                 while (current != null) {
-                    processMatchableNode(current);
+                    processMatchableNodes(current);
                     for (TypeMirror intf : current.getInterfaces()) {
                         Element interfaceElement = typeUtils().asElement(intf);
-                        processMatchableNode(interfaceElement);
+                        processMatchableNodes(interfaceElement);
                         // Recurse
                         findMatchableNodes(interfaceElement);
                     }
@@ -896,34 +891,34 @@ public class MatchProcessor extends AbstractProcessor {
         return processingEnv.getTypeUtils();
     }
 
-    private void processMethodMatchRule(ExecutableElement method, MatchRuleDescriptor info, MatchRule matchRule, AnnotationMirror mirror) {
+    private void processMatchRule(ExecutableElement method, MatchRuleDescriptor info, AnnotationMirror matchRule) {
-        logMessage("processMethodMatchRule %s %s\n", method, mirror);
+        logMessage("processMatchRule %s\n", method);
 
         Types typeUtils = typeUtils();
 
         if (!method.getModifiers().contains(Modifier.PUBLIC)) {
-            errorMessage(method, "MatchRule method %s must be public", method.getSimpleName());
+            printError(method, "MatchRule method %s must be public", method.getSimpleName());
             return;
         }
         if (method.getModifiers().contains(Modifier.STATIC)) {
-            errorMessage(method, "MatchRule method %s must be non-static", method.getSimpleName());
+            printError(method, "MatchRule method %s must be non-static", method.getSimpleName());
             return;
         }
 
         try {
             TypeMirror returnType = method.getReturnType();
-            if (!typeUtils.isSameType(returnType, processingEnv.getElementUtils().getTypeElement(ComplexMatchResult.class.getName()).asType())) {
+            if (!typeUtils.isSameType(returnType, processingEnv.getElementUtils().getTypeElement(COMPLEX_MATCH_RESULT_CLASS_NAME).asType())) {
-                errorMessage(method, "MatchRule method return type must be %s", ComplexMatchResult.class.getName());
+                printError(method, "MatchRule method return type must be %s", COMPLEX_MATCH_RESULT_CLASS_NAME);
                 return;
             }
 
-            String rule = matchRule.value();
+            String rule = getAnnotationValue(matchRule, "value", String.class);
             RuleParser parser = new RuleParser(rule);
             ArrayList<TypeDescriptor> expectedTypes = parser.capturedTypes();
             ArrayList<String> expectedNames = parser.capturedNames();
             List<? extends VariableElement> actualParameters = method.getParameters();
             if (expectedTypes.size() + 1 < actualParameters.size()) {
-                errorMessage(method, "Too many arguments for match method %s != %s", expectedTypes.size() + 1, actualParameters.size());
+                printError(method, "Too many arguments for match method %s != %s", expectedTypes.size() + 1, actualParameters.size());
                 return;
             }
 
@@ -934,12 +929,12 @@ public class MatchProcessor extends AbstractProcessor {
                 String name = parameter.getSimpleName().toString();
                 int nameIndex = expectedNames.indexOf(name);
                 if (nameIndex == -1) {
-                    errorMessage(method, "Argument \"%s\" isn't captured in the match rule", name);
+                    printError(method, "Argument \"%s\" isn't captured in the match rule", name);
                     return;
                 }
                 TypeMirror type = parameter.asType();
                 if (!typeUtils.isAssignable(expectedTypes.get(nameIndex).mirror, type)) {
-                    errorMessage(method, "Captured value \"%s\" of type %s is not assignable to argument of type %s", name, expectedTypes.get(nameIndex).mirror, type);
+                    printError(method, "Captured value \"%s\" of type %s is not assignable to argument of type %s", name, expectedTypes.get(nameIndex).mirror, type);
                     return;
                 }
             }
@@ -952,7 +947,7 @@ public class MatchProcessor extends AbstractProcessor {
             } else if (invoker.method != method) {
                 // This could be supported but it's easier if they are unique since the names
                 // are used in log output and snippet counters.
-                errorMessage(method, "Use unique method names for match methods: %s.%s != %s.%s", method.getReceiverType(), method.getSimpleName(), invoker.method.getReceiverType(),
+                printError(method, "Use unique method names for match methods: %s.%s != %s.%s", method.getReceiverType(), method.getSimpleName(), invoker.method.getReceiverType(),
                                 invoker.method.getSimpleName());
                 return;
             }
@@ -960,12 +955,12 @@ public class MatchProcessor extends AbstractProcessor {
             Element enclosing = method.getEnclosingElement();
             String declaringClass = "";
             String separator = "";
-            EconomicSet<Element> originatingElementsList = info.originatingElements;
+            Set<Element> originatingElementsList = info.originatingElements;
             originatingElementsList.add(method);
             while (enclosing != null) {
                 if (enclosing.getKind() == ElementKind.CLASS || enclosing.getKind() == ElementKind.INTERFACE) {
                     if (enclosing.getModifiers().contains(Modifier.PRIVATE)) {
-                        errorMessage(method, "MatchRule cannot be declared in a private %s %s", enclosing.getKind().name().toLowerCase(), enclosing);
+                        printError(method, "MatchRule cannot be declared in a private %s %s", enclosing.getKind().name().toLowerCase(), enclosing);
                         return;
                     }
                     originatingElementsList.add(enclosing);
@@ -988,144 +983,26 @@ public class MatchProcessor extends AbstractProcessor {
                 info.matchRules.add(new MatchRuleItem(match, invoker));
             }
         } catch (RuleParseError e) {
-            errorMessage(method, mirror, e.getMessage());
+            printError(method, matchRule, e.getMessage());
         }
     }
 
-    private void errorMessage(Element element, String format, Object... args) {
+    private Element elementForMessage(Element e) {
-        processingEnv.getMessager().printMessage(Kind.ERROR, String.format(format, args), element);
+        if (currentRound != null && !currentRound.getRootElements().contains(e) && currentElement != null) {
+            return currentElement;
+        }
+        return e;
     }
 
-    private void errorMessage(Element element, AnnotationMirror mirror, String format, Object... args) {
+    private void printError(Element annotatedElement, String format, Object... args) {
-        processingEnv.getMessager().printMessage(Kind.ERROR, String.format(format, args), element, mirror);
+        Element e = elementForMessage(annotatedElement);
+        String prefix = e == annotatedElement ? "" : annotatedElement + ": ";
+        processingEnv.getMessager().printMessage(Kind.ERROR, prefix + String.format(format, args), e);
     }
 
-    // TODO borrowed from com.oracle.truffle.dsl.processor.Utils
+    private void printError(Element annotatedElement, AnnotationMirror annotation, String format, Object... args) {
-    @SuppressWarnings("unchecked")
+        Element e = elementForMessage(annotatedElement);
-    private static <T> List<T> getAnnotationValueList(Class<T> expectedListType, AnnotationMirror mirror, String name) {
+        String prefix = e == annotatedElement ? "" : annotation + " on " + annotatedElement + ": ";
-        List<? extends AnnotationValue> values = getAnnotationValue(List.class, mirror, name);
+        processingEnv.getMessager().printMessage(Kind.ERROR, prefix + String.format(format, args), e, annotation);
-        List<T> result = new ArrayList<>();
-
-        if (values != null) {
-            for (AnnotationValue value : values) {
-                T annotationValue = resolveAnnotationValue(expectedListType, value);
-                if (annotationValue != null) {
-                    result.add(annotationValue);
-                }
-            }
-        }
-        return result;
-    }
-
-    private static <T> T getAnnotationValue(Class<T> expectedType, AnnotationMirror mirror, String name) {
-        return resolveAnnotationValue(expectedType, getAnnotationValue(mirror, name));
-    }
-
-    @SuppressWarnings({"unchecked"})
-    private static <T> T resolveAnnotationValue(Class<T> expectedType, AnnotationValue value) {
-        if (value == null) {
-            return null;
-        }
-
-        Object unboxedValue = value.accept(new AnnotationValueVisitorImpl(), null);
-        if (unboxedValue != null) {
-            if (expectedType == TypeMirror.class && unboxedValue instanceof String) {
-                return null;
-            }
-            if (!expectedType.isAssignableFrom(unboxedValue.getClass())) {
-                throw new ClassCastException(unboxedValue.getClass().getName() + " not assignable from " + expectedType.getName());
-            }
-        }
-        return (T) unboxedValue;
-    }
-
-    private static AnnotationValue getAnnotationValue(AnnotationMirror mirror, String name) {
-        ExecutableElement valueMethod = null;
-        for (ExecutableElement method : ElementFilter.methodsIn(mirror.getAnnotationType().asElement().getEnclosedElements())) {
-            if (method.getSimpleName().toString().equals(name)) {
-                valueMethod = method;
-                break;
-            }
-        }
-
-        if (valueMethod == null) {
-            return null;
-        }
-
-        AnnotationValue value = mirror.getElementValues().get(valueMethod);
-        if (value == null) {
-            value = valueMethod.getDefaultValue();
-        }
-
-        return value;
-    }
-
-    private static class AnnotationValueVisitorImpl extends AbstractAnnotationValueVisitor7<Object, Void> {
-
-        @Override
-        public Object visitBoolean(boolean b, Void p) {
-            return Boolean.valueOf(b);
-        }
-
-        @Override
-        public Object visitByte(byte b, Void p) {
-            return Byte.valueOf(b);
-        }
-
-        @Override
-        public Object visitChar(char c, Void p) {
-            return c;
-        }
-
-        @Override
-        public Object visitDouble(double d, Void p) {
-            return d;
-        }
-
-        @Override
-        public Object visitFloat(float f, Void p) {
-            return f;
-        }
-
-        @Override
-        public Object visitInt(int i, Void p) {
-            return i;
-        }
-
-        @Override
-        public Object visitLong(long i, Void p) {
-            return i;
-        }
-
-        @Override
-        public Object visitShort(short s, Void p) {
-            return s;
-        }
-
-        @Override
-        public Object visitString(String s, Void p) {
-            return s;
-        }
-
-        @Override
-        public Object visitType(TypeMirror t, Void p) {
-            return t;
-        }
-
-        @Override
-        public Object visitEnumConstant(VariableElement c, Void p) {
-            return c;
-        }
-
-        @Override
-        public Object visitAnnotation(AnnotationMirror a, Void p) {
-            return a;
-        }
-
-        @Override
-        public Object visitArray(List<? extends AnnotationValue> vals, Void p) {
-            return vals;
-        }
-
     }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.sparc/src/org/graalvm/compiler/core/sparc/SPARCNodeMatchRules.java

@@ -169,7 +169,7 @@ public class SPARCNodeMatchRules extends NodeMatchRules {
                 SPARCAddressValue address = (SPARCAddressValue) operand(cas.getAddress());
                 Condition condition = successIsTrue ? Condition.EQ : Condition.NE;
 
-                Value result = getLIRGeneratorTool().emitValueCompareAndSwap(address, expectedValue, newValue);
+                Value result = getLIRGeneratorTool().emitValueCompareAndSwap(kind, address, expectedValue, newValue);
                 getLIRGeneratorTool().emitCompareBranch(kind.getPlatformKind(), result, expectedValue, condition, false, trueLabel, falseLabel, trueLabelProbability);
                 return null;
             };

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ArrayLengthProviderTest.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2015, 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.core.test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.Test;
+
+public class ArrayLengthProviderTest extends GraalCompilerTest {
+
+    public static Object test0Snippet(ArrayList<?> list, boolean a) {
+        while (true) {
+            Object[] array = toArray(list);
+            if (array.length < 1) {
+                return null;
+            }
+            if (array[0] instanceof String || a) {
+                /*
+                 * This code is outside of the loop. Accessing the array reqires a ValueProxyNode.
+                 * When the simplification of the ArrayLengthNode replaces the length access with
+                 * the ArrayList.size used to create the array, then the value needs to have a
+                 * ValueProxyNode too. In addition, the two parts of the if-condition actually lead
+                 * to two separate loop exits, with two separate proxy nodes. A ValuePhiNode is
+                 * present originally for the array, and the array length simplification needs to
+                 * create a new ValuePhiNode for the two newly introduced ValueProxyNode.
+                 */
+                if (array.length < 1) {
+                    return null;
+                }
+                return array[0];
+            }
+        }
+    }
+
+    public static Object test1Snippet(ArrayList<?> list, boolean a, boolean b) {
+        while (true) {
+            Object[] array = toArray(list);
+            if (a || b) {
+                if (array.length < 1) {
+                    return null;
+                }
+                return array[0];
+            }
+        }
+    }
+
+    public static Object[] toArray(List<?> list) {
+        return new Object[list.size()];
+    }
+
+    @Test
+    public void test0() {
+        test("test0Snippet", new ArrayList<>(Arrays.asList("a", "b")), true);
+    }
+
+    @Test
+    public void test1() {
+        test("test1Snippet", new ArrayList<>(Arrays.asList("a", "b")), true, true);
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/CanonicalizedConversionTest.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.core.test;
+
+import org.graalvm.compiler.graph.Node;
+import org.graalvm.compiler.nodes.IfNode;
+import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.nodes.calc.AddNode;
+import org.graalvm.compiler.nodes.calc.FloatEqualsNode;
+import org.graalvm.compiler.nodes.calc.ReinterpretNode;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests that substitutions for {@link Double#doubleToLongBits(double)} and
+ * {@link Float#floatToIntBits(float)} produce graphs such that multiple calls to these methods with
+ * the same input are canonicalized.
+ */
+public class CanonicalizedConversionTest extends GraalCompilerTest {
+
+    @Override
+    protected boolean checkLowTierGraph(StructuredGraph graph) {
+        int reinterpretCount = 0;
+        int floatEqualsCount = 0;
+        int addCount = 0;
+        for (Node node : graph.getNodes()) {
+            if (node instanceof ReinterpretNode) {
+                reinterpretCount++;
+            } else if (node instanceof FloatEqualsNode) {
+                floatEqualsCount++;
+            } else if (node instanceof IfNode) {
+                Assert.fail("Unexpected node: " + node);
+            } else if (node instanceof AddNode) {
+                addCount++;
+            }
+        }
+        Assert.assertEquals(1, reinterpretCount);
+        Assert.assertEquals(1, floatEqualsCount);
+        Assert.assertEquals(2, addCount);
+        return true;
+    }
+
+    @Test
+    public void test4() {
+        test("snippet4", 567.890F);
+        test("snippet4", -567.890F);
+        test("snippet4", Float.NaN);
+    }
+
+    public static int snippet4(float value) {
+        return Float.floatToIntBits(value) + Float.floatToIntBits(value) + Float.floatToIntBits(value);
+    }
+
+    @Test
+    public void test5() {
+        test("snippet5", 567.890D);
+        test("snippet5", -567.890D);
+        test("snippet5", Double.NaN);
+    }
+
+    public static long snippet5(double value) {
+        return Double.doubleToLongBits(value) + Double.doubleToLongBits(value) + Double.doubleToLongBits(value);
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ConditionalEliminationTest15.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.core.test;
+
+import org.graalvm.compiler.debug.DebugContext;
+import org.graalvm.compiler.graph.Node;
+import org.graalvm.compiler.nodes.StructuredGraph;
+import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions;
+import org.graalvm.compiler.nodes.calc.IntegerLessThanNode;
+import org.graalvm.compiler.nodes.spi.LoweringTool;
+import org.graalvm.compiler.phases.common.CanonicalizerPhase;
+import org.graalvm.compiler.phases.common.IterativeConditionalEliminationPhase;
+import org.graalvm.compiler.phases.common.LoweringPhase;
+import org.graalvm.compiler.phases.tiers.PhaseContext;
+import org.graalvm.compiler.virtual.phases.ea.EarlyReadEliminationPhase;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Collection of tests for {@link org.graalvm.compiler.phases.common.ConditionalEliminationPhase}
+ * including those that triggered bugs in this phase.
+ */
+public class ConditionalEliminationTest15 extends ConditionalEliminationTestBase {
+
+    private void checkNodeCount(String methodName, Class<? extends Node> nodeClass, int count) {
+        StructuredGraph graph = parseEager(methodName, AllowAssumptions.YES);
+
+        CanonicalizerPhase canonicalizer = new CanonicalizerPhase();
+        PhaseContext context = new PhaseContext(getProviders());
+
+        new LoweringPhase(new CanonicalizerPhase(), LoweringTool.StandardLoweringStage.HIGH_TIER).apply(graph, context);
+        canonicalizer.apply(graph, context);
+
+        // Merge arr.length reads.
+        new EarlyReadEliminationPhase(canonicalizer).apply(graph, context);
+        new IterativeConditionalEliminationPhase(canonicalizer, true).apply(graph, context);
+
+        getDebugContext().dump(DebugContext.BASIC_LEVEL, graph, "After ConditionalEliminationPhase");
+
+        Assert.assertEquals(count, graph.getNodes().filter(nodeClass).count());
+    }
+
+    public static int testRedundantIntegerLessThanNode(int index, int[] arr) {
+        while (arr[index] != 42) {
+            if (index >= 0) { // redundant
+                return 1;
+            }
+        }
+        return 2;
+    }
+
+    public static int testRedundantIntegerLessThanNode2(int index, int[] arr) {
+        while (arr[index] != 42) {
+            if (index < arr.length) { // redundant
+                return 1;
+            }
+        }
+        return 2;
+    }
+
+    @Test
+    public void testRedundantSignedLessThanNode() {
+        checkNodeCount("testRedundantIntegerLessThanNode", IntegerLessThanNode.class, 0);
+        checkNodeCount("testRedundantIntegerLessThanNode2", IntegerLessThanNode.class, 0);
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ConditionalEliminationTest2.java

@@ -22,15 +22,18 @@
  */
 package org.graalvm.compiler.core.test;
 
+import org.graalvm.compiler.debug.DebugContext;
 import org.graalvm.compiler.nodes.GuardNode;
 import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions;
+import org.graalvm.compiler.nodes.java.InstanceOfNode;
 import org.graalvm.compiler.nodes.spi.LoweringTool;
 import org.graalvm.compiler.phases.common.CanonicalizerPhase;
+import org.graalvm.compiler.phases.common.ConditionalEliminationPhase;
 import org.graalvm.compiler.phases.common.FloatingReadPhase;
 import org.graalvm.compiler.phases.common.LoweringPhase;
-import org.graalvm.compiler.phases.common.ConditionalEliminationPhase;
 import org.graalvm.compiler.phases.tiers.PhaseContext;
+import org.junit.Assert;
 import org.junit.Test;
 
 /**
@@ -60,6 +63,15 @@ public class ConditionalEliminationTest2 extends ConditionalEliminationTestBase
         final Entry next;
     }
 
+    static class A {
+    }
+
+    static class B extends A {
+    }
+
+    static class C extends A {
+    }
+
     public static Entry search(Entry start, String name, Entry alternative) {
         Entry current = start;
         do {
@@ -129,4 +141,81 @@ public class ConditionalEliminationTest2 extends ConditionalEliminationTestBase
         assertDeepEquals(0, graph.getNodes().filter(GuardNode.class).count());
     }
 
+    private void checkInstanceOfCount(String methodName, int count) {
+        StructuredGraph graph = parseEager(methodName, AllowAssumptions.YES);
+
+        CanonicalizerPhase canonicalizer = new CanonicalizerPhase();
+        PhaseContext context = new PhaseContext(getProviders());
+
+        canonicalizer.apply(graph, context);
+        new ConditionalEliminationPhase(true).apply(graph, context);
+        getDebugContext().dump(DebugContext.BASIC_LEVEL, graph, "After ConditionalEliminationPhase");
+        canonicalizer.apply(graph, context);
+
+        Assert.assertEquals(count, graph.getNodes().filter(InstanceOfNode.class).count());
+    }
+
+    public static A testRedundantInstanceOfClass(Object value) {
+        if (value != null && value.getClass() == A.class) {
+            return (A) value;
+        }
+        return null;
+    }
+
+    public static Object testRedundantInstanceOfArray(Object value) {
+        if (value != null && value.getClass() == Object[].class) {
+            return ((Object[]) value)[0];
+        }
+        return null;
+    }
+
+    public static boolean testRedundantInstanceOfPrecise(Object value) {
+        if (value != null && value.getClass() == A.class) {
+            return value instanceof A;
+        }
+        return false;
+    }
+
+    public static boolean testRedundantInstanceOfImplicitNonNull(Object value) {
+        if (value.getClass() == A.class) {
+            return value instanceof A;
+        }
+        return false;
+    }
+
+    @Test
+    public void testRedundantInstanceOf() {
+        checkInstanceOfCount("testRedundantInstanceOfClass", 1);
+        checkInstanceOfCount("testRedundantInstanceOfArray", 1);
+        checkInstanceOfCount("testRedundantInstanceOfPrecise", 1);
+        checkInstanceOfCount("testRedundantInstanceOfImplicitNonNull", 1);
+    }
+
+    public static boolean testNonRedundantInstanceOfClass(Object value) {
+        if (value instanceof A) {
+            return (value != null && value.getClass() == A.class);
+        }
+        return false;
+    }
+
+    public static boolean testNonRedundantInstanceOfArray(Object value) {
+        if (value instanceof Object[]) {
+            return (value != null && value.getClass() == Object[].class);
+        }
+        return false;
+    }
+
+    public static boolean testNonRedundantInstanceOfImplicitNonNull(Object value) {
+        if (value instanceof Object[]) {
+            return value.getClass() == Object[].class;
+        }
+        return false;
+    }
+
+    @Test
+    public void testNonRedundantInstanceOf() {
+        checkInstanceOfCount("testNonRedundantInstanceOfClass", 2);
+        checkInstanceOfCount("testNonRedundantInstanceOfArray", 2);
+        checkInstanceOfCount("testNonRedundantInstanceOfImplicitNonNull", 2);
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GraalCompilerTest.java

@@ -1054,7 +1054,7 @@ public abstract class GraalCompilerTest extends GraalTest {
         try (DebugContext.Scope s = debug.scope("Compile", graphToCompile)) {
             assert options != null;
             Request<CompilationResult> request = new Request<>(graphToCompile, installedCodeOwner, getProviders(), getBackend(), getDefaultGraphBuilderSuite(), OptimisticOptimizations.ALL,
-                            graphToCompile.getProfilingInfo(), createSuites(options), createLIRSuites(options), compilationResult, CompilationResultBuilderFactory.Default);
+                            graphToCompile.getProfilingInfo(), createSuites(options), createLIRSuites(options), compilationResult, CompilationResultBuilderFactory.Default, true);
             return GraalCompiler.compile(request);
         } catch (Throwable e) {
             throw debug.handle(e);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/InfopointReasonTest.java

@@ -68,7 +68,7 @@ public class InfopointReasonTest extends GraalCompilerTest {
         final ResolvedJavaMethod method = getResolvedJavaMethod("testMethod");
         final StructuredGraph graph = parseEager(method, AllowAssumptions.YES);
         final CompilationResult cr = compileGraph(graph, graph.method(), getProviders(), getBackend(), getDefaultGraphBuilderSuite(), OptimisticOptimizations.ALL, graph.getProfilingInfo(),
-                        createSuites(graph.getOptions()), createLIRSuites(graph.getOptions()), new CompilationResult(graph.compilationId()), CompilationResultBuilderFactory.Default);
+                        createSuites(graph.getOptions()), createLIRSuites(graph.getOptions()), new CompilationResult(graph.compilationId()), CompilationResultBuilderFactory.Default, true);
         for (Infopoint sp : cr.getInfopoints()) {
             assertNotNull(sp.reason);
             if (sp instanceof Call) {
@@ -90,7 +90,7 @@ public class InfopointReasonTest extends GraalCompilerTest {
         assertTrue(graphLineSPs > 0);
         PhaseSuite<HighTierContext> graphBuilderSuite = getCustomGraphBuilderSuite(GraphBuilderConfiguration.getDefault(getDefaultGraphBuilderPlugins()).withFullInfopoints(true));
         final CompilationResult cr = compileGraph(graph, graph.method(), getProviders(), getBackend(), graphBuilderSuite, OptimisticOptimizations.ALL, graph.getProfilingInfo(),
-                        createSuites(graph.getOptions()), createLIRSuites(graph.getOptions()), new CompilationResult(graph.compilationId()), CompilationResultBuilderFactory.Default);
+                        createSuites(graph.getOptions()), createLIRSuites(graph.getOptions()), new CompilationResult(graph.compilationId()), CompilationResultBuilderFactory.Default, true);
         int lineSPs = 0;
         for (Infopoint sp : cr.getInfopoints()) {
             assertNotNull(sp.reason);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/tutorial/InvokeGraal.java

@@ -127,7 +127,7 @@ public class InvokeGraal {
             CompilationResultBuilderFactory factory = CompilationResultBuilderFactory.Default;
 
             /* Invoke the whole Graal compilation pipeline. */
-            GraalCompiler.compileGraph(graph, method, providers, backend, graphBuilderSuite, optimisticOpts, profilingInfo, suites, lirSuites, compilationResult, factory);
+            GraalCompiler.compileGraph(graph, method, providers, backend, graphBuilderSuite, optimisticOpts, profilingInfo, suites, lirSuites, compilationResult, factory, true);
 
             /*
              * Install the compilation result into the VM, i.e., copy the byte[] array that contains

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/GraalCompiler.java

@@ -106,6 +106,7 @@ public class GraalCompiler {
         public final LIRSuites lirSuites;
         public final T compilationResult;
         public final CompilationResultBuilderFactory factory;
+        public final boolean verifySourcePositions;
 
         /**
          * @param graph the graph to be compiled
@@ -122,7 +123,8 @@ public class GraalCompiler {
          * @param factory
          */
         public Request(StructuredGraph graph, ResolvedJavaMethod installedCodeOwner, Providers providers, Backend backend, PhaseSuite<HighTierContext> graphBuilderSuite,
-                        OptimisticOptimizations optimisticOpts, ProfilingInfo profilingInfo, Suites suites, LIRSuites lirSuites, T compilationResult, CompilationResultBuilderFactory factory) {
+                        OptimisticOptimizations optimisticOpts, ProfilingInfo profilingInfo, Suites suites, LIRSuites lirSuites, T compilationResult, CompilationResultBuilderFactory factory,
+                        boolean verifySourcePositions) {
             this.graph = graph;
             this.installedCodeOwner = installedCodeOwner;
             this.providers = providers;
@@ -134,6 +136,7 @@ public class GraalCompiler {
             this.lirSuites = lirSuites;
             this.compilationResult = compilationResult;
             this.factory = factory;
+            this.verifySourcePositions = verifySourcePositions;
         }
 
         /**
@@ -156,8 +159,9 @@ public class GraalCompiler {
      */
     public static <T extends CompilationResult> T compileGraph(StructuredGraph graph, ResolvedJavaMethod installedCodeOwner, Providers providers, Backend backend,
                     PhaseSuite<HighTierContext> graphBuilderSuite, OptimisticOptimizations optimisticOpts, ProfilingInfo profilingInfo, Suites suites, LIRSuites lirSuites, T compilationResult,
-                    CompilationResultBuilderFactory factory) {
+                    CompilationResultBuilderFactory factory, boolean verifySourcePositions) {
-        return compile(new Request<>(graph, installedCodeOwner, providers, backend, graphBuilderSuite, optimisticOpts, profilingInfo, suites, lirSuites, compilationResult, factory));
+        return compile(new Request<>(graph, installedCodeOwner, providers, backend, graphBuilderSuite, optimisticOpts, profilingInfo, suites, lirSuites, compilationResult, factory,
+                        verifySourcePositions));
     }
 
     /**
@@ -173,6 +177,9 @@ public class GraalCompiler {
             try (DebugContext.Scope s0 = debug.scope("GraalCompiler", r.graph, r.providers.getCodeCache()); DebugCloseable a = CompilerTimer.start(debug)) {
                 emitFrontEnd(r.providers, r.backend, r.graph, r.graphBuilderSuite, r.optimisticOpts, r.profilingInfo, r.suites);
                 emitBackEnd(r.graph, null, r.installedCodeOwner, r.backend, r.compilationResult, r.factory, null, r.lirSuites);
+                if (r.verifySourcePositions) {
+                    assert r.graph.verifySourcePositions(true);
+                }
             } catch (Throwable e) {
                 throw debug.handle(e);
             }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/NodeMatchRules.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2009, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2009, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,8 +22,6 @@
  */
 package org.graalvm.compiler.core.gen;
 
-import jdk.vm.ci.meta.Value;
-
 import org.graalvm.compiler.core.match.MatchableNode;
 import org.graalvm.compiler.graph.Node;
 import org.graalvm.compiler.lir.LIRFrameState;
@@ -62,6 +60,8 @@ import org.graalvm.compiler.nodes.memory.FloatingReadNode;
 import org.graalvm.compiler.nodes.memory.ReadNode;
 import org.graalvm.compiler.nodes.memory.WriteNode;
 
+import jdk.vm.ci.meta.Value;
+
 @MatchableNode(nodeClass = ConstantNode.class, shareable = true)
 @MatchableNode(nodeClass = FloatConvertNode.class, inputs = {"value"})
 @MatchableNode(nodeClass = FloatingReadNode.class, inputs = {"address"})

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug.test/src/org/graalvm/compiler/debug/test/DebugContextTest.java

@@ -166,6 +166,32 @@ public class DebugContextTest {
         Assert.assertEquals(expect, log);
     }
 
+    @Test
+    public void testContextScope() {
+        OptionValues options = new OptionValues(EconomicMap.create());
+        options = new OptionValues(options, DebugOptions.Log, ":5");
+        DebugContextSetup setup = new DebugContextSetup();
+        try (DebugContext debug = setup.openDebugContext(options)) {
+            try (DebugContext.Scope s0 = debug.scope("TestLogging")) {
+                try (DebugContext.Scope s1 = debug.withContext("A")) {
+                    for (Object o : debug.context()) {
+                        Assert.assertEquals(o, "A");
+                    }
+                } catch (Throwable t) {
+                    throw debug.handle(t);
+                }
+                try (DebugContext.Scope s1 = debug.withContext("B")) {
+                    for (Object o : debug.context()) {
+                        Assert.assertEquals(o, "B");
+                    }
+                } catch (Throwable t) {
+                    throw debug.handle(t);
+                }
+            }
+        }
+
+    }
+
     @Test
     public void testEnabledSandbox() {
         EconomicMap<OptionKey<?>, Object> map = EconomicMap.create();

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/CausableByCompilerAssert.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.debug;
+
+public interface CausableByCompilerAssert {
+    boolean isCausedByCompilerAssert();
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/DebugConfigImpl.java

@@ -239,8 +239,11 @@ final class DebugConfigImpl implements DebugConfig {
 
     @Override
     public RuntimeException interceptException(DebugContext debug, Throwable e) {
-        if (e instanceof BailoutException && !DebugOptions.InterceptBailout.getValue(options)) {
+        if (e instanceof BailoutException) {
-            return null;
+            final boolean causedByCompilerAssert = e instanceof CausableByCompilerAssert && ((CausableByCompilerAssert) e).isCausedByCompilerAssert();
+            if (!DebugOptions.InterceptBailout.getValue(options) && !causedByCompilerAssert) {
+                return null;
+            }
         }
 
         OptionValues interceptOptions = new OptionValues(options,

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/DebugContext.java

@@ -96,7 +96,7 @@ public final class DebugContext implements AutoCloseable {
      */
     boolean metricsEnabled;
 
-    DebugConfig currentConfig;
+    DebugConfigImpl currentConfig;
     ScopeImpl currentScope;
     CloseableCounter currentTimer;
     CloseableCounter currentMemUseTracker;
@@ -738,6 +738,19 @@ public final class DebugContext implements AutoCloseable {
         }
     }
 
+    /**
+     * Create an unnamed scope that appends some context to the current scope.
+     *
+     * @param context an object to be appended to the {@linkplain #context() current} debug context
+     */
+    public DebugContext.Scope withContext(Object context) throws Throwable {
+        if (currentScope != null) {
+            return enterScope("", null, context);
+        } else {
+            return null;
+        }
+    }
+
     /**
      * Creates and enters a new debug scope which will be disjoint from the current debug scope.
      * <p>
@@ -787,7 +800,7 @@ public final class DebugContext implements AutoCloseable {
     class DisabledScope implements DebugContext.Scope {
         final boolean savedMetricsEnabled;
         final ScopeImpl savedScope;
-        final DebugConfig savedConfig;
+        final DebugConfigImpl savedConfig;
 
         DisabledScope() {
             this.savedMetricsEnabled = metricsEnabled;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/DebugValueMap.java

@@ -1,194 +0,0 @@
-/*
- * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-package org.graalvm.compiler.debug;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * A node in a tree of values.
- */
-public class DebugValueMap {
-
-    /**
-     * The top level maps for all threads.
-     */
-    private static final List<DebugValueMap> topLevelMaps = new ArrayList<>();
-
-    private long[] values;
-    private List<DebugValueMap> children;
-    private String name;
-
-    public DebugValueMap(String name) {
-        this.name = name;
-    }
-
-    public void setCurrentValue(int index, long l) {
-        ensureSize(index);
-        values[index] = l;
-    }
-
-    public long getCurrentValue(int index) {
-        ensureSize(index);
-        return values[index];
-    }
-
-    public void clearChildren() {
-        if (children != null) {
-            children.clear();
-        }
-    }
-
-    public void reset() {
-        if (values != null) {
-            Arrays.fill(values, 0L);
-        }
-        if (children != null) {
-            for (DebugValueMap child : children) {
-                child.reset();
-            }
-        }
-    }
-
-    private void ensureSize(int index) {
-        if (values == null) {
-            values = new long[index + 1];
-        }
-        if (values.length <= index) {
-            values = Arrays.copyOf(values, index + 1);
-        }
-    }
-
-    private int capacity() {
-        return (values == null) ? 0 : values.length;
-    }
-
-    public void addChild(DebugValueMap map) {
-        if (children == null) {
-            children = new ArrayList<>(4);
-        }
-        children.add(map);
-    }
-
-    public List<DebugValueMap> getChildren() {
-        if (children == null) {
-            return Collections.emptyList();
-        } else {
-            return Collections.unmodifiableList(children);
-        }
-    }
-
-    public boolean hasChildren() {
-        return children != null && !children.isEmpty();
-    }
-
-    public String getName() {
-        return this.name;
-    }
-
-    @Override
-    public String toString() {
-        return "DebugValueMap<" + getName() + ">";
-    }
-
-    public static synchronized void registerTopLevel(DebugValueMap map) {
-        topLevelMaps.add(map);
-    }
-
-    public static synchronized List<DebugValueMap> getTopLevelMaps() {
-        return topLevelMaps;
-    }
-
-    /**
-     * The top level map for the current thread.
-     */
-    private static final ThreadLocal<DebugValueMap> topLevelMap = new ThreadLocal<>();
-
-    static DebugValueMap getTopLevelMap() {
-        DebugValueMap map = topLevelMap.get();
-        if (map == null) {
-            map = new DebugValueMap(Thread.currentThread().getName());
-            topLevelMap.set(map);
-            registerTopLevel(map);
-        }
-        return map;
-    }
-
-    public void normalize() {
-        if (hasChildren()) {
-            Map<String, DebugValueMap> occurred = new HashMap<>();
-            for (DebugValueMap map : children) {
-                String mapName = map.getName();
-                if (!occurred.containsKey(mapName)) {
-                    occurred.put(mapName, map);
-                    map.normalize();
-                } else {
-                    occurred.get(mapName).mergeWith(map);
-                    occurred.get(mapName).normalize();
-                }
-            }
-
-            if (occurred.values().size() < children.size()) {
-                // At least one duplicate was found.
-                children.clear();
-                for (DebugValueMap map : occurred.values()) {
-                    addChild(map);
-                    map.normalize();
-                }
-            }
-        }
-    }
-
-    private void mergeWith(DebugValueMap map) {
-        if (map.hasChildren()) {
-            if (hasChildren()) {
-                children.addAll(map.children);
-            } else {
-                children = map.children;
-            }
-            map.children = null;
-        }
-
-        int size = Math.max(this.capacity(), map.capacity());
-        ensureSize(size);
-        for (int i = 0; i < size; ++i) {
-            long curValue = getCurrentValue(i);
-            long otherValue = map.getCurrentValue(i);
-            setCurrentValue(i, curValue + otherValue);
-        }
-    }
-
-    public void group() {
-        if (this.hasChildren()) {
-            List<DebugValueMap> oldChildren = new ArrayList<>(this.children);
-            this.children.clear();
-            for (DebugValueMap map : oldChildren) {
-                mergeWith(map);
-            }
-        }
-    }
-}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/PathUtilities.java

@@ -72,9 +72,10 @@ public class PathUtilities {
 
     /**
      * A maximum file name length supported by most file systems. There is no platform independent
-     * way to get this in Java.
+     * way to get this in Java. Normally it is 255. But for AUFS it is 242. Refer AUFS_MAX_NAMELEN
+     * in http://aufs.sourceforge.net/aufs3/man.html.
      */
-    private static final int MAX_FILE_NAME_LENGTH = 255;
+    private static final int MAX_FILE_NAME_LENGTH = 242;
 
     private static final String ELLIPSIS = "...";
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/ScopeImpl.java

@@ -38,31 +38,35 @@ public final class ScopeImpl implements DebugContext.Scope {
         final String indent;
         final IndentImpl parentIndent;
 
+        boolean isEmitted() {
+            return emitted;
+        }
+
+        private boolean emitted;
+
         IndentImpl(IndentImpl parentIndent) {
             this.parentIndent = parentIndent;
             this.indent = (parentIndent == null ? "" : parentIndent.indent + INDENTATION_INCREMENT);
         }
 
-        private boolean logScopeName() {
-            return logScopeName;
-        }
-
         private void printScopeName(StringBuilder str, boolean isCurrent) {
-            if (logScopeName) {
+            if (!emitted) {
-                boolean parentPrinted = false;
+                boolean mustPrint = true;
                 if (parentIndent != null) {
-                    parentPrinted = parentIndent.logScopeName();
+                    if (!parentIndent.isEmitted()) {
-                    parentIndent.printScopeName(str, false);
+                        parentIndent.printScopeName(str, false);
+                        mustPrint = false;
+                    }
                 }
                 /*
-                 * Always print the current scope, scopes with context and the any scope whose
+                 * Always print the current scope, scopes with context and any scope whose parent
-                 * parent didn't print. This ensure the first new scope always shows up.
+                 * didn't print. This ensure the first new scope always shows up.
                  */
-                if (isCurrent || printContext(null) != 0 || !parentPrinted) {
+                if (isCurrent || printContext(null) != 0 || mustPrint) {
                     str.append(indent).append("[thread:").append(Thread.currentThread().getId()).append("] scope: ").append(getQualifiedName()).append(System.lineSeparator());
                 }
                 printContext(str);
-                logScopeName = false;
+                emitted = true;
             }
         }
 
@@ -116,7 +120,12 @@ public final class ScopeImpl implements DebugContext.Scope {
     private final ScopeImpl parent;
     private final boolean sandbox;
     private IndentImpl lastUsedIndent;
-    private boolean logScopeName;
+
+    private boolean isEmptyScope() {
+        return emptyScope;
+    }
+
+    private final boolean emptyScope;
 
     private final Object[] context;
 
@@ -136,23 +145,24 @@ public final class ScopeImpl implements DebugContext.Scope {
     private PrintStream output;
     private boolean interceptDisabled;
 
-    static final Object[] EMPTY_CONTEXT = new Object[0];
-
     ScopeImpl(DebugContext owner, Thread thread) {
         this(owner, thread.getName(), null, false);
     }
 
-    ScopeImpl(DebugContext owner, String unqualifiedName, ScopeImpl parent, boolean sandbox, Object... context) {
+    private ScopeImpl(DebugContext owner, String unqualifiedName, ScopeImpl parent, boolean sandbox, Object... context) {
         this.owner = owner;
         this.parent = parent;
         this.sandbox = sandbox;
         this.context = context;
         this.unqualifiedName = unqualifiedName;
         if (parent != null) {
-            logScopeName = !unqualifiedName.equals("");
+            emptyScope = unqualifiedName.equals("");
             this.interceptDisabled = parent.interceptDisabled;
         } else {
-            logScopeName = true;
+            if (unqualifiedName.isEmpty()) {
+                throw new IllegalArgumentException("root scope name must be non-empty");
+            }
+            emptyScope = false;
         }
 
         this.output = TTY.out;
@@ -169,29 +179,29 @@ public final class ScopeImpl implements DebugContext.Scope {
         return parent == null;
     }
 
-    public boolean isDumpEnabled(int dumpLevel) {
+    boolean isDumpEnabled(int dumpLevel) {
         assert dumpLevel >= 0;
         return currentDumpLevel >= dumpLevel;
     }
 
-    public boolean isVerifyEnabled() {
+    boolean isVerifyEnabled() {
         return verifyEnabled;
     }
 
-    public boolean isLogEnabled(int logLevel) {
+    boolean isLogEnabled(int logLevel) {
         assert logLevel > 0;
         return currentLogLevel >= logLevel;
     }
 
-    public boolean isCountEnabled() {
+    boolean isCountEnabled() {
         return countEnabled;
     }
 
-    public boolean isTimeEnabled() {
+    boolean isTimeEnabled() {
         return timeEnabled;
     }
 
-    public boolean isMemUseTrackingEnabled() {
+    boolean isMemUseTrackingEnabled() {
         return memUseTrackingEnabled;
     }
 
@@ -307,7 +317,7 @@ public final class ScopeImpl implements DebugContext.Scope {
         throw silenceException(RuntimeException.class, e);
     }
 
-    void updateFlags(DebugConfig config) {
+    void updateFlags(DebugConfigImpl config) {
         if (config == null) {
             countEnabled = false;
             memUseTrackingEnabled = false;
@@ -317,6 +327,14 @@ public final class ScopeImpl implements DebugContext.Scope {
             // Be pragmatic: provide a default log stream to prevent a crash if the stream is not
             // set while logging
             output = TTY.out;
+        } else if (isEmptyScope()) {
+            countEnabled = parent.countEnabled;
+            memUseTrackingEnabled = parent.memUseTrackingEnabled;
+            timeEnabled = parent.timeEnabled;
+            verifyEnabled = parent.verifyEnabled;
+            output = parent.output;
+            currentDumpLevel = parent.currentDumpLevel;
+            currentLogLevel = parent.currentLogLevel;
         } else {
             countEnabled = config.isCountEnabled(this);
             memUseTrackingEnabled = config.isMemUseTrackingEnabled(this);
@@ -404,18 +422,21 @@ public final class ScopeImpl implements DebugContext.Scope {
             if (parent == null) {
                 qualifiedName = unqualifiedName;
             } else {
-                qualifiedName = parent.getQualifiedName() + SCOPE_SEP + unqualifiedName;
+                qualifiedName = parent.getQualifiedName();
+                if (!isEmptyScope()) {
+                    qualifiedName += SCOPE_SEP + unqualifiedName;
+                }
             }
         }
         return qualifiedName;
     }
 
-    public Indent pushIndentLogger() {
+    Indent pushIndentLogger() {
         lastUsedIndent = getLastUsedIndent().indent();
         return lastUsedIndent;
     }
 
-    public IndentImpl getLastUsedIndent() {
+    private IndentImpl getLastUsedIndent() {
         if (lastUsedIndent == null) {
             if (parent != null) {
                 lastUsedIndent = new IndentImpl(parent.getLastUsedIndent());

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/Graph.java

@@ -1199,7 +1199,7 @@ public class Graph {
         return true;
     }
 
-    public boolean verifySourcePositions() {
+    public boolean verifySourcePositions(boolean performConsistencyCheck) {
         if (trackNodeSourcePosition()) {
             ResolvedJavaMethod root = null;
             for (Node node : getNodes()) {
@@ -1211,6 +1211,11 @@ public class Graph {
                         assert pos.verifyRootMethod(root) : node;
                     }
                 }
+
+                // More strict node-type-specific check
+                if (performConsistencyCheck) {
+                    node.verifySourcePosition();
+                }
             }
         }
         return true;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/Node.java

@@ -263,7 +263,7 @@ public abstract class Node implements Cloneable, Formattable, NodeInterface {
     static class NodeCreationStackTrace extends NodeStackTrace {
     }
 
-    static class NodeInsertionStackTrace extends NodeStackTrace {
+    public static class NodeInsertionStackTrace extends NodeStackTrace {
     }
 
     public Node(NodeClass<? extends Node> c) {
@@ -1063,6 +1063,10 @@ public abstract class Node implements Cloneable, Formattable, NodeInterface {
         return true;
     }
 
+    public boolean verifySourcePosition() {
+        return true;
+    }
+
     /**
      * Perform expensive verification of inputs, usages, predecessors and successors.
      *

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/NodeClass.java

@@ -51,6 +51,7 @@ import org.graalvm.compiler.debug.CounterKey;
 import org.graalvm.compiler.debug.DebugCloseable;
 import org.graalvm.compiler.debug.DebugContext;
 import org.graalvm.compiler.debug.GraalError;
+import org.graalvm.compiler.debug.TTY;
 import org.graalvm.compiler.debug.TimerKey;
 import org.graalvm.compiler.graph.Edges.Type;
 import org.graalvm.compiler.graph.Graph.DuplicationReplacement;
@@ -125,11 +126,33 @@ public final class NodeClass<T> extends FieldIntrospection<T> {
     }
 
     public static <T> NodeClass<T> get(Class<T> clazz) {
-        NodeClass<T> result = getUnchecked(clazz);
+        int numTries = 0;
-        if (result == null && clazz != NODE_CLASS) {
+        while (true) {
-            throw GraalError.shouldNotReachHere("TYPE field not initialized for class " + clazz.getTypeName());
+            boolean shouldBeInitializedBefore = UnsafeAccess.UNSAFE.shouldBeInitialized(clazz);
+
+            NodeClass<T> result = getUnchecked(clazz);
+            if (result != null || clazz == NODE_CLASS) {
+                return result;
+            }
+
+            /*
+             * GR-9537: We observed a transient problem with TYPE fields being null. Retry a couple
+             * of times and print something to the log so that we can gather more diagnostic
+             * information without failing gates.
+             */
+            numTries++;
+            boolean shouldBeInitializedAfter = UnsafeAccess.UNSAFE.shouldBeInitialized(clazz);
+            String msg = "GR-9537 Reflective field access of TYPE field returned null. This is probably a bug in HotSpot class initialization. " +
+                            " clazz: " + clazz.getTypeName() + ", numTries: " + numTries +
+                            ", shouldBeInitializedBefore: " + shouldBeInitializedBefore + ", shouldBeInitializedAfter: " + shouldBeInitializedAfter;
+            if (numTries <= 100) {
+                TTY.println(msg);
+                UnsafeAccess.UNSAFE.ensureClassInitialized(clazz);
+            } else {
+                throw GraalError.shouldNotReachHere(msg);
+            }
+            return result;
         }
-        return result;
     }
 
     private static final Class<?> NODE_CLASS = Node.class;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotBackend.java

@@ -29,6 +29,7 @@ import static jdk.vm.ci.aarch64.AArch64.sp;
 import static jdk.vm.ci.aarch64.AArch64.zr;
 import static jdk.vm.ci.code.ValueUtil.asRegister;
 import static jdk.vm.ci.hotspot.aarch64.AArch64HotSpotRegisterConfig.fp;
+import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
 import static org.graalvm.compiler.core.common.GraalOptions.ZapStackOnMethodEntry;
 
 import jdk.internal.vm.compiler.collections.EconomicSet;
@@ -41,6 +42,7 @@ import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler.ScratchRegister;
 import org.graalvm.compiler.code.CompilationResult;
 import org.graalvm.compiler.core.aarch64.AArch64NodeMatchRules;
 import org.graalvm.compiler.core.common.CompilationIdentifier;
+import org.graalvm.compiler.core.common.LIRKind;
 import org.graalvm.compiler.core.common.alloc.RegisterAllocationConfig;
 import org.graalvm.compiler.core.common.spi.ForeignCallLinkage;
 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
@@ -48,6 +50,7 @@ import org.graalvm.compiler.hotspot.HotSpotDataBuilder;
 import org.graalvm.compiler.hotspot.HotSpotGraalRuntimeProvider;
 import org.graalvm.compiler.hotspot.HotSpotHostBackend;
 import org.graalvm.compiler.hotspot.HotSpotLIRGenerationResult;
+import org.graalvm.compiler.hotspot.meta.HotSpotConstantLoadAction;
 import org.graalvm.compiler.hotspot.meta.HotSpotForeignCallsProvider;
 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
 import org.graalvm.compiler.hotspot.stubs.Stub;
@@ -66,12 +69,15 @@ import org.graalvm.compiler.lir.gen.LIRGeneratorTool;
 import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool;
 
+import jdk.vm.ci.aarch64.AArch64Kind;
 import jdk.vm.ci.code.CallingConvention;
 import jdk.vm.ci.code.Register;
 import jdk.vm.ci.code.RegisterConfig;
 import jdk.vm.ci.code.StackSlot;
 import jdk.vm.ci.hotspot.HotSpotCallingConventionType;
+import jdk.vm.ci.hotspot.HotSpotSentinelConstant;
 import jdk.vm.ci.hotspot.aarch64.AArch64HotSpotRegisterConfig;
+import jdk.vm.ci.meta.JavaKind;
 import jdk.vm.ci.meta.JavaType;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
 
@@ -269,7 +275,7 @@ public class AArch64HotSpotBackend extends HotSpotHostBackend {
             Register klass = r10;
             if (config.useCompressedClassPointers) {
                 masm.ldr(32, klass, klassAddress);
-                AArch64HotSpotMove.decodeKlassPointer(masm, klass, klass, providers.getRegisters().getHeapBaseRegister(), config.getKlassEncoding());
+                AArch64HotSpotMove.decodeKlassPointer(crb, masm, klass, klass, config.getKlassEncoding(), config);
             } else {
                 masm.ldr(64, klass, klassAddress);
             }
@@ -285,6 +291,23 @@ public class AArch64HotSpotBackend extends HotSpotHostBackend {
         crb.recordMark(config.MARKID_OSR_ENTRY);
         masm.bind(verifiedStub);
         crb.recordMark(config.MARKID_VERIFIED_ENTRY);
+
+        if (GeneratePIC.getValue(crb.getOptions())) {
+            // Check for method state
+            HotSpotFrameContext frameContext = (HotSpotFrameContext) crb.frameContext;
+            if (!frameContext.isStub) {
+                crb.recordInlineDataInCodeWithNote(new HotSpotSentinelConstant(LIRKind.value(AArch64Kind.QWORD), JavaKind.Long), HotSpotConstantLoadAction.MAKE_NOT_ENTRANT);
+                try (ScratchRegister sc = masm.getScratchRegister()) {
+                    Register scratch = sc.getRegister();
+                    masm.addressOf(scratch);
+                    masm.ldr(64, scratch, AArch64Address.createBaseRegisterOnlyAddress(scratch));
+                    Label noCall = new Label();
+                    masm.cbz(64, scratch, noCall);
+                    AArch64Call.directJmp(crb, masm, getForeignCalls().lookupForeignCall(WRONG_METHOD_HANDLER));
+                    masm.bind(noCall);
+                }
+            }
+        }
     }
 
     private static void emitCodeBody(CompilationResultBuilder crb, LIR lir, AArch64MacroAssembler masm) {
@@ -299,8 +322,10 @@ public class AArch64HotSpotBackend extends HotSpotHostBackend {
      * @see "http://mail.openjdk.java.net/pipermail/aarch64-port-dev/2013-September/000273.html"
      */
     public static void emitInvalidatePlaceholder(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
-        crb.blockComment("[nop for method invalidation]");
+        if (!GeneratePIC.getValue(crb.getOptions())) {
-        masm.nop();
+            crb.blockComment("[nop for method invalidation]");
+            masm.nop();
+        }
     }
 
     private void emitCodeSuffix(CompilationResultBuilder crb, AArch64MacroAssembler masm, FrameMap frameMap) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotBackendFactory.java

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2013, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,6 +26,10 @@ package org.graalvm.compiler.hotspot.aarch64;
 import static jdk.vm.ci.aarch64.AArch64.sp;
 import static jdk.vm.ci.common.InitTimer.timer;
 
+import jdk.vm.ci.code.Architecture;
+import jdk.vm.ci.code.Register;
+import jdk.vm.ci.code.RegisterConfig;
+import jdk.vm.ci.code.TargetDescription;
 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
 import org.graalvm.compiler.bytecode.BytecodeProvider;
 import org.graalvm.compiler.core.aarch64.AArch64AddressLoweringByUse;
@@ -64,10 +69,6 @@ import org.graalvm.compiler.serviceprovider.ServiceProvider;
 import org.graalvm.compiler.word.WordTypes;
 
 import jdk.vm.ci.aarch64.AArch64;
-import jdk.vm.ci.code.Architecture;
-import jdk.vm.ci.code.RegisterArray;
-import jdk.vm.ci.code.RegisterConfig;
-import jdk.vm.ci.code.TargetDescription;
 import jdk.vm.ci.common.InitTimer;
 import jdk.vm.ci.hotspot.HotSpotCodeCacheProvider;
 import jdk.vm.ci.hotspot.HotSpotConstantReflectionProvider;
@@ -77,6 +78,9 @@ import jdk.vm.ci.hotspot.aarch64.AArch64HotSpotRegisterConfig;
 import jdk.vm.ci.meta.Value;
 import jdk.vm.ci.runtime.JVMCIBackend;
 
+import java.util.ArrayList;
+import java.util.List;
+
 @ServiceProvider(HotSpotBackendFactory.class)
 public class AArch64HotSpotBackendFactory implements HotSpotBackendFactory {
 
@@ -200,12 +204,20 @@ public class AArch64HotSpotBackendFactory implements HotSpotBackendFactory {
     }
 
     protected static Value[] createNativeABICallerSaveRegisters(@SuppressWarnings("unused") GraalHotSpotVMConfig config, RegisterConfig regConfig) {
-        AArch64HotSpotRegisterConfig conf = (AArch64HotSpotRegisterConfig) regConfig;
+        List<Register> callerSave = new ArrayList<>(regConfig.getAllocatableRegisters().asList());
-        RegisterArray callerSavedRegisters = conf.getCallerSaveRegisters();
+        callerSave.remove(AArch64.r19);
-        int size = callerSavedRegisters.size();
+        callerSave.remove(AArch64.r20);
-        Value[] nativeABICallerSaveRegisters = new Value[size];
+        callerSave.remove(AArch64.r21);
-        for (int i = 0; i < size; i++) {
+        callerSave.remove(AArch64.r22);
-            nativeABICallerSaveRegisters[i] = callerSavedRegisters.get(i).asValue();
+        callerSave.remove(AArch64.r23);
+        callerSave.remove(AArch64.r24);
+        callerSave.remove(AArch64.r25);
+        callerSave.remove(AArch64.r26);
+        callerSave.remove(AArch64.r27);
+        callerSave.remove(AArch64.r28);
+        Value[] nativeABICallerSaveRegisters = new Value[callerSave.size()];
+        for (int i = 0; i < callerSave.size(); i++) {
+            nativeABICallerSaveRegisters[i] = callerSave.get(i).asValue();
         }
         return nativeABICallerSaveRegisters;
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotCRuntimeCallEpilogueOp.java

@@ -24,6 +24,7 @@ package org.graalvm.compiler.hotspot.aarch64;
 
 import static jdk.vm.ci.aarch64.AArch64.zr;
 
+import org.graalvm.compiler.asm.Label;
 import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler;
 import org.graalvm.compiler.lir.LIRInstructionClass;
 import org.graalvm.compiler.lir.Opcode;
@@ -37,20 +38,22 @@ public class AArch64HotSpotCRuntimeCallEpilogueOp extends AArch64LIRInstruction
     public static final LIRInstructionClass<AArch64HotSpotCRuntimeCallEpilogueOp> TYPE = LIRInstructionClass.create(AArch64HotSpotCRuntimeCallEpilogueOp.class);
 
     private final int threadLastJavaSpOffset;
-    private final int threadLastJavaFpOffset;
+    private final int threadLastJavaPcOffset;
     private final Register thread;
+    @SuppressWarnings("unused") private final Label label;
 
-    public AArch64HotSpotCRuntimeCallEpilogueOp(int threadLastJavaSpOffset, int threadLastJavaFpOffset, Register thread) {
+    public AArch64HotSpotCRuntimeCallEpilogueOp(int threadLastJavaSpOffset, int threadLastJavaPcOffset, Register thread, Label label) {
         super(TYPE);
         this.threadLastJavaSpOffset = threadLastJavaSpOffset;
-        this.threadLastJavaFpOffset = threadLastJavaFpOffset;
+        this.threadLastJavaPcOffset = threadLastJavaPcOffset;
         this.thread = thread;
+        this.label = label;
     }
 
     @Override
     public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
         // Reset last Java frame:
         masm.str(64, zr, masm.makeAddress(thread, threadLastJavaSpOffset, 8));
-        masm.str(64, zr, masm.makeAddress(thread, threadLastJavaFpOffset, 8));
+        masm.str(64, zr, masm.makeAddress(thread, threadLastJavaPcOffset, 8));
     }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotCRuntimeCallPrologueOp.java

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,7 +26,6 @@ package org.graalvm.compiler.hotspot.aarch64;
 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG;
 import static jdk.vm.ci.aarch64.AArch64.sp;
 import static jdk.vm.ci.code.ValueUtil.asRegister;
-import static jdk.vm.ci.hotspot.aarch64.AArch64HotSpotRegisterConfig.fp;
 
 import org.graalvm.compiler.asm.Label;
 import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler;
@@ -43,16 +43,14 @@ public class AArch64HotSpotCRuntimeCallPrologueOp extends AArch64LIRInstruction
 
     private final int threadLastJavaSpOffset;
     private final int threadLastJavaPcOffset;
-    private final int threadLastJavaFpOffset;
     private final Register thread;
     @Temp({REG}) protected AllocatableValue scratch;
     private final Label label;
 
-    public AArch64HotSpotCRuntimeCallPrologueOp(int threadLastJavaSpOffset, int threadLastJavaPcOffset, int threadLastJavaFpOffset, Register thread, AllocatableValue scratch, Label label) {
+    public AArch64HotSpotCRuntimeCallPrologueOp(int threadLastJavaSpOffset, int threadLastJavaPcOffset, Register thread, AllocatableValue scratch, Label label) {
         super(TYPE);
         this.threadLastJavaSpOffset = threadLastJavaSpOffset;
         this.threadLastJavaPcOffset = threadLastJavaPcOffset;
-        this.threadLastJavaFpOffset = threadLastJavaFpOffset;
         this.thread = thread;
         this.scratch = scratch;
         this.label = label;
@@ -69,7 +67,5 @@ public class AArch64HotSpotCRuntimeCallPrologueOp extends AArch64LIRInstruction
         // Get the current PC. Use a label to patch the return address.
         masm.adr(scratchRegister, label);
         masm.str(64, scratchRegister, masm.makeAddress(thread, threadLastJavaPcOffset, 8));
-
-        masm.str(64, fp, masm.makeAddress(thread, threadLastJavaFpOffset, 8));
     }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotConstantRetrievalOp.java

@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.hotspot.aarch64;
+
+import static jdk.vm.ci.code.ValueUtil.asRegister;
+
+import java.util.ArrayList;
+import java.util.EnumSet;
+
+import jdk.vm.ci.code.CallingConvention;
+import jdk.vm.ci.meta.AllocatableValue;
+import jdk.vm.ci.meta.Constant;
+import jdk.vm.ci.meta.Value;
+
+import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler;
+import org.graalvm.compiler.core.common.spi.ForeignCallLinkage;
+import org.graalvm.compiler.lir.LIRFrameState;
+import org.graalvm.compiler.lir.LIRInstructionClass;
+import org.graalvm.compiler.lir.LIRValueUtil;
+import org.graalvm.compiler.lir.ValueProcedure;
+import org.graalvm.compiler.lir.aarch64.AArch64LIRInstruction;
+import org.graalvm.compiler.lir.asm.CompilationResultBuilder;
+
+public final class AArch64HotSpotConstantRetrievalOp extends AArch64LIRInstruction {
+    public static final LIRInstructionClass<AArch64HotSpotConstantRetrievalOp> TYPE = LIRInstructionClass.create(AArch64HotSpotConstantRetrievalOp.class);
+
+    @Def protected AllocatableValue result;
+    protected final Constant[] constants;
+    @Alive protected AllocatableValue[] constantDescriptions;
+    @Temp protected AllocatableValue[] gotSlotOffsetParameters;
+    @Temp protected AllocatableValue[] descriptionParameters;
+    @Temp protected Value[] callTemps;
+    @State protected LIRFrameState frameState;
+    private final ForeignCallLinkage callLinkage;
+    private final Object[] notes;
+
+    private class CollectTemporaries implements ValueProcedure {
+        ArrayList<Value> values = new ArrayList<>();
+
+        CollectTemporaries() {
+            forEachTemp(this);
+        }
+
+        public Value[] asArray() {
+            Value[] copy = new Value[values.size()];
+            return values.toArray(copy);
+        }
+
+        @Override
+        public Value doValue(Value value, OperandMode mode, EnumSet<OperandFlag> flags) {
+            values.add(value);
+            return value;
+        }
+    }
+
+    public AArch64HotSpotConstantRetrievalOp(Constant[] constants, AllocatableValue[] constantDescriptions, LIRFrameState frameState, ForeignCallLinkage callLinkage, Object[] notes) {
+        super(TYPE);
+        this.constantDescriptions = constantDescriptions;
+        this.constants = constants;
+        this.frameState = frameState;
+        this.notes = notes;
+        assert constants.length == notes.length;
+
+        // call arguments
+        CallingConvention callingConvention = callLinkage.getOutgoingCallingConvention();
+        this.gotSlotOffsetParameters = new AllocatableValue[constants.length];
+        int argIndex = 0;
+        for (int i = 0; i < constants.length; i++, argIndex++) {
+            this.gotSlotOffsetParameters[i] = callingConvention.getArgument(argIndex);
+        }
+        this.descriptionParameters = new AllocatableValue[constantDescriptions.length];
+        for (int i = 0; i < constantDescriptions.length; i++, argIndex++) {
+            this.descriptionParameters[i] = callingConvention.getArgument(argIndex);
+        }
+        this.result = callingConvention.getReturn();
+
+        this.callLinkage = callLinkage;
+
+        // compute registers that are killed by the stub, but are not used as other temps.
+        this.callTemps = new Value[0];
+        this.callTemps = LIRValueUtil.subtractRegisters(callLinkage.getTemporaries(), new CollectTemporaries().asArray());
+    }
+
+    @Override
+    public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+        // metadata_adr
+        for (int i = 0; i < constants.length; i++) {
+            crb.recordInlineDataInCodeWithNote(constants[i], notes[i]);
+            masm.addressOf(asRegister(gotSlotOffsetParameters[i]));
+        }
+
+        for (int i = 0; i < constantDescriptions.length; i++) {
+            masm.mov(64, asRegister(descriptionParameters[i]), asRegister(constantDescriptions[i]));
+        }
+
+        final int before = masm.position();
+        masm.bl(before);
+        final int after = masm.position();
+        crb.recordDirectCall(before, after, callLinkage, frameState);
+    }
+
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotLIRGenerator.java

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,11 +24,21 @@
 
 package org.graalvm.compiler.hotspot.aarch64;
 
+import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
+import static org.graalvm.compiler.hotspot.HotSpotBackend.INITIALIZE_KLASS_BY_SYMBOL;
+import static org.graalvm.compiler.hotspot.HotSpotBackend.RESOLVE_KLASS_BY_SYMBOL;
+import static org.graalvm.compiler.hotspot.HotSpotBackend.RESOLVE_METHOD_BY_SYMBOL_AND_LOAD_COUNTERS;
+import static org.graalvm.compiler.hotspot.HotSpotBackend.RESOLVE_STRING_BY_SYMBOL;
+import static org.graalvm.compiler.hotspot.HotSpotBackend.RESOLVE_DYNAMIC_INVOKE;
+import static org.graalvm.compiler.hotspot.meta.HotSpotConstantLoadAction.RESOLVE;
+import static org.graalvm.compiler.hotspot.meta.HotSpotConstantLoadAction.INITIALIZE;
+import static org.graalvm.compiler.hotspot.meta.HotSpotConstantLoadAction.LOAD_COUNTERS;
 import static org.graalvm.compiler.lir.LIRValueUtil.asConstant;
 import static org.graalvm.compiler.lir.LIRValueUtil.isConstantValue;
 
 import java.util.function.Function;
 
+import jdk.vm.ci.hotspot.HotSpotMetaspaceConstant;
 import org.graalvm.compiler.asm.Label;
 import org.graalvm.compiler.asm.aarch64.AArch64Address.AddressingMode;
 import org.graalvm.compiler.asm.aarch64.AArch64Assembler.ConditionFlag;
@@ -38,6 +49,7 @@ import org.graalvm.compiler.core.aarch64.AArch64LIRKindTool;
 import org.graalvm.compiler.core.common.CompressEncoding;
 import org.graalvm.compiler.core.common.LIRKind;
 import org.graalvm.compiler.core.common.calc.Condition;
+import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
 import org.graalvm.compiler.core.common.spi.ForeignCallLinkage;
 import org.graalvm.compiler.core.common.spi.LIRKindTool;
 import org.graalvm.compiler.debug.GraalError;
@@ -48,6 +60,7 @@ import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage;
 import org.graalvm.compiler.hotspot.HotSpotLIRGenerationResult;
 import org.graalvm.compiler.hotspot.HotSpotLIRGenerator;
 import org.graalvm.compiler.hotspot.HotSpotLockStack;
+import org.graalvm.compiler.hotspot.meta.HotSpotConstantLoadAction;
 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
 import org.graalvm.compiler.hotspot.stubs.Stub;
@@ -66,6 +79,9 @@ import org.graalvm.compiler.lir.aarch64.AArch64FrameMapBuilder;
 import org.graalvm.compiler.lir.aarch64.AArch64Move;
 import org.graalvm.compiler.lir.aarch64.AArch64Move.StoreOp;
 import org.graalvm.compiler.lir.aarch64.AArch64PrefetchOp;
+import org.graalvm.compiler.lir.aarch64.AArch64SaveRegistersOp;
+import org.graalvm.compiler.lir.aarch64.AArch64RestoreRegistersOp;
+
 import org.graalvm.compiler.lir.gen.LIRGenerationResult;
 
 import jdk.vm.ci.aarch64.AArch64;
@@ -84,6 +100,7 @@ import jdk.vm.ci.meta.JavaConstant;
 import jdk.vm.ci.meta.JavaKind;
 import jdk.vm.ci.meta.PlatformKind;
 import jdk.vm.ci.meta.Value;
+import org.graalvm.compiler.options.OptionValues;
 
 /**
  * LIR generator specialized for AArch64 HotSpot.
@@ -114,7 +131,7 @@ public class AArch64HotSpotLIRGenerator extends AArch64LIRGenerator implements H
         return getResult().getStub() != null;
     }
 
-    @SuppressWarnings("unused") private LIRFrameState currentRuntimeCallInfo;
+    private LIRFrameState currentRuntimeCallInfo;
 
     @Override
     protected void emitForeignCallOp(ForeignCallLinkage linkage, Value result, Value[] arguments, Value[] temps, LIRFrameState info) {
@@ -145,8 +162,45 @@ public class AArch64HotSpotLIRGenerator extends AArch64LIRGenerator implements H
         append(new AArch64CCall(nativeCallingConvention.getReturn(), ptr, argLocations));
     }
 
-    public SaveRegistersOp emitSaveAllRegisters() {
+    /**
-        throw GraalError.unimplemented();
+     * @param savedRegisters the registers saved by this operation which may be subject to pruning
+     * @param savedRegisterLocations the slots to which the registers are saved
+     * @param supportsRemove determines if registers can be pruned
+     */
+    protected AArch64SaveRegistersOp emitSaveRegisters(Register[] savedRegisters, AllocatableValue[] savedRegisterLocations, boolean supportsRemove) {
+        AArch64SaveRegistersOp save = new AArch64SaveRegistersOp(savedRegisters, savedRegisterLocations, supportsRemove);
+        append(save);
+        return save;
+    }
+
+    /**
+     * Allocate a stack slot for saving a register.
+     */
+    protected VirtualStackSlot allocateSaveRegisterLocation(Register register) {
+        PlatformKind kind = target().arch.getLargestStorableKind(register.getRegisterCategory());
+        if (kind.getVectorLength() > 1) {
+            // we don't use vector registers, so there is no need to save them
+            kind = AArch64Kind.QWORD;
+        }
+        return getResult().getFrameMapBuilder().allocateSpillSlot(LIRKind.value(kind));
+    }
+
+    /**
+     * Adds a node to the graph that saves all allocatable registers to the stack.
+     *
+     * @param supportsRemove determines if registers can be pruned
+     * @return the register save node
+     */
+    private AArch64SaveRegistersOp emitSaveAllRegisters(Register[] savedRegisters, boolean supportsRemove) {
+        AllocatableValue[] savedRegisterLocations = new AllocatableValue[savedRegisters.length];
+        for (int i = 0; i < savedRegisters.length; i++) {
+            savedRegisterLocations[i] = allocateSaveRegisterLocation(savedRegisters[i]);
+        }
+        return emitSaveRegisters(savedRegisters, savedRegisterLocations, supportsRemove);
+    }
+
+    protected void emitRestoreRegisters(AArch64SaveRegistersOp save) {
+        append(new AArch64RestoreRegistersOp(save.getSlots().clone(), save));
     }
 
     @Override
@@ -199,6 +253,7 @@ public class AArch64HotSpotLIRGenerator extends AArch64LIRGenerator implements H
     @Override
     public Value emitCompress(Value pointer, CompressEncoding encoding, boolean nonNull) {
         LIRKind inputKind = pointer.getValueKind(LIRKind.class);
+        LIRKindTool lirKindTool = getLIRKindTool();
         assert inputKind.getPlatformKind() == AArch64Kind.QWORD;
         if (inputKind.isReference(0)) {
             // oop
@@ -209,8 +264,16 @@ public class AArch64HotSpotLIRGenerator extends AArch64LIRGenerator implements H
             // metaspace pointer
             Variable result = newVariable(LIRKind.value(AArch64Kind.DWORD));
             AllocatableValue base = Value.ILLEGAL;
-            if (encoding.hasBase()) {
+            OptionValues options = getResult().getLIR().getOptions();
-                base = emitLoadConstant(LIRKind.value(AArch64Kind.QWORD), JavaConstant.forLong(encoding.getBase()));
+            if (encoding.hasBase() || GeneratePIC.getValue(options)) {
+                if (GeneratePIC.getValue(options)) {
+                    Variable baseAddress = newVariable(lirKindTool.getWordKind());
+                    AArch64HotSpotMove.BaseMove move = new AArch64HotSpotMove.BaseMove(baseAddress, config);
+                    append(move);
+                    base = baseAddress;
+                } else {
+                    base = emitLoadConstant(LIRKind.value(AArch64Kind.QWORD), JavaConstant.forLong(encoding.getBase()));
+                }
             }
             append(new AArch64HotSpotMove.CompressPointer(result, asAllocatable(pointer), base, encoding, nonNull));
             return result;
@@ -230,8 +293,16 @@ public class AArch64HotSpotLIRGenerator extends AArch64LIRGenerator implements H
             // metaspace pointer
             Variable result = newVariable(LIRKind.value(AArch64Kind.QWORD));
             AllocatableValue base = Value.ILLEGAL;
-            if (encoding.hasBase()) {
+            OptionValues options = getResult().getLIR().getOptions();
-                base = emitLoadConstant(LIRKind.value(AArch64Kind.QWORD), JavaConstant.forLong(encoding.getBase()));
+            if (encoding.hasBase() || GeneratePIC.getValue(options)) {
+                if (GeneratePIC.getValue(options)) {
+                    Variable baseAddress = newVariable(LIRKind.value(AArch64Kind.QWORD));
+                    AArch64HotSpotMove.BaseMove move = new AArch64HotSpotMove.BaseMove(baseAddress, config);
+                    append(move);
+                    base = baseAddress;
+                } else {
+                    base = emitLoadConstant(LIRKind.value(AArch64Kind.QWORD), JavaConstant.forLong(encoding.getBase()));
+                }
             }
             append(new AArch64HotSpotMove.UncompressPointer(result, asAllocatable(pointer), base, encoding, nonNull));
             return result;
@@ -270,6 +341,17 @@ public class AArch64HotSpotLIRGenerator extends AArch64LIRGenerator implements H
     @Override
     public Variable emitForeignCall(ForeignCallLinkage linkage, LIRFrameState state, Value... args) {
         HotSpotForeignCallLinkage hotspotLinkage = (HotSpotForeignCallLinkage) linkage;
+        boolean destroysRegisters = hotspotLinkage.destroysRegisters();
+
+        AArch64SaveRegistersOp save = null;
+        Stub stub = getStub();
+        if (destroysRegisters) {
+            if (stub != null && stub.preservesRegisters()) {
+                Register[] savedRegisters = getRegisterConfig().getAllocatableRegisters().toArray();
+                save = emitSaveAllRegisters(savedRegisters, true);
+            }
+        }
+
         Variable result;
         LIRFrameState debugInfo = null;
         if (hotspotLinkage.needsDebugInfo()) {
@@ -277,7 +359,7 @@ public class AArch64HotSpotLIRGenerator extends AArch64LIRGenerator implements H
             assert debugInfo != null || getStub() != null;
         }
 
-        if (linkage.destroysRegisters() || hotspotLinkage.needsJavaFrameAnchor()) {
+        if (destroysRegisters || hotspotLinkage.needsJavaFrameAnchor()) {
             HotSpotRegistersProvider registers = getProviders().getRegisters();
             Register thread = registers.getThreadRegister();
             Variable scratch = newVariable(LIRKind.value(target().arch.getWordKind()));
@@ -285,9 +367,9 @@ public class AArch64HotSpotLIRGenerator extends AArch64LIRGenerator implements H
             // We need a label for the return address.
             label = new Label();
 
-            append(new AArch64HotSpotCRuntimeCallPrologueOp(config.threadLastJavaSpOffset(), config.threadLastJavaPcOffset(), config.threadLastJavaFpOffset(), thread, scratch, label));
+            append(new AArch64HotSpotCRuntimeCallPrologueOp(config.threadLastJavaSpOffset(), config.threadLastJavaPcOffset(), thread, scratch, label));
             result = super.emitForeignCall(hotspotLinkage, debugInfo, args);
-            append(new AArch64HotSpotCRuntimeCallEpilogueOp(config.threadLastJavaSpOffset(), config.threadLastJavaFpOffset(), thread));
+            append(new AArch64HotSpotCRuntimeCallEpilogueOp(config.threadLastJavaSpOffset(), config.threadLastJavaPcOffset(), thread, label));
 
             // Clear it out so it's not being reused later.
             label = null;
@@ -295,6 +377,21 @@ public class AArch64HotSpotLIRGenerator extends AArch64LIRGenerator implements H
             result = super.emitForeignCall(hotspotLinkage, debugInfo, args);
         }
 
+        if (destroysRegisters) {
+            if (stub != null) {
+                if (stub.preservesRegisters()) {
+                    HotSpotLIRGenerationResult generationResult = getResult();
+                    LIRFrameState key = currentRuntimeCallInfo;
+                    if (key == null) {
+                        key = LIRFrameState.NO_STATE;
+                    }
+                    assert !generationResult.getCalleeSaveInfo().containsKey(key);
+                    generationResult.getCalleeSaveInfo().put(key, save);
+                    emitRestoreRegisters(save);
+                }
+            }
+        }
+
         return result;
     }
 
@@ -335,6 +432,65 @@ public class AArch64HotSpotLIRGenerator extends AArch64LIRGenerator implements H
         append(new AArch64HotSpotUnwindOp(config, exceptionParameter));
     }
 
+    @Override
+    public Value emitLoadObjectAddress(Constant constant) {
+        HotSpotObjectConstant objectConstant = (HotSpotObjectConstant) constant;
+        LIRKind kind = objectConstant.isCompressed() ? getLIRKindTool().getNarrowOopKind() : getLIRKindTool().getObjectKind();
+        Variable result = newVariable(kind);
+        append(new AArch64HotSpotLoadAddressOp(result, constant, HotSpotConstantLoadAction.RESOLVE));
+        return result;
+    }
+
+    @Override
+    public Value emitLoadMetaspaceAddress(Constant constant, HotSpotConstantLoadAction action) {
+        HotSpotMetaspaceConstant metaspaceConstant = (HotSpotMetaspaceConstant) constant;
+        LIRKind kind = metaspaceConstant.isCompressed() ? getLIRKindTool().getNarrowPointerKind() : getLIRKindTool().getWordKind();
+        Variable result = newVariable(kind);
+        append(new AArch64HotSpotLoadAddressOp(result, constant, action));
+        return result;
+    }
+
+    private Value emitConstantRetrieval(ForeignCallDescriptor foreignCall, Object[] notes, Constant[] constants, AllocatableValue[] constantDescriptions, LIRFrameState frameState) {
+        ForeignCallLinkage linkage = getForeignCalls().lookupForeignCall(foreignCall);
+        append(new AArch64HotSpotConstantRetrievalOp(constants, constantDescriptions, frameState, linkage, notes));
+        AllocatableValue result = linkage.getOutgoingCallingConvention().getReturn();
+        return emitMove(result);
+    }
+
+    private Value emitConstantRetrieval(ForeignCallDescriptor foreignCall, HotSpotConstantLoadAction action, Constant constant, AllocatableValue[] constantDescriptions, LIRFrameState frameState) {
+        Constant[] constants = new Constant[]{constant};
+        Object[] notes = new Object[]{action};
+        return emitConstantRetrieval(foreignCall, notes, constants, constantDescriptions, frameState);
+    }
+
+    @Override
+    public Value emitResolveDynamicInvoke(Constant appendix, LIRFrameState frameState) {
+        AllocatableValue[] constantDescriptions = new AllocatableValue[0];
+        return emitConstantRetrieval(RESOLVE_DYNAMIC_INVOKE, INITIALIZE, appendix, constantDescriptions, frameState);
+    }
+
+    @Override
+    public Value emitLoadConfigValue(int markId, LIRKind kind) {
+        Variable result = newVariable(kind);
+        append(new AArch64HotSpotLoadConfigValueOp(markId, result));
+        return result;
+    }
+
+    private Value emitConstantRetrieval(ForeignCallDescriptor foreignCall, HotSpotConstantLoadAction action, Constant constant, Value constantDescription, LIRFrameState frameState) {
+        AllocatableValue[] constantDescriptions = new AllocatableValue[]{asAllocatable(constantDescription)};
+        return emitConstantRetrieval(foreignCall, action, constant, constantDescriptions, frameState);
+    }
+
+    @Override
+    public Value emitObjectConstantRetrieval(Constant constant, Value constantDescription, LIRFrameState frameState) {
+        return emitConstantRetrieval(RESOLVE_STRING_BY_SYMBOL, RESOLVE, constant, constantDescription, frameState);
+    }
+
+    @Override
+    public Value emitMetaspaceConstantRetrieval(Constant constant, Value constantDescription, LIRFrameState frameState) {
+        return emitConstantRetrieval(RESOLVE_KLASS_BY_SYMBOL, RESOLVE, constant, constantDescription, frameState);
+    }
+
     @Override
     public void emitReturn(JavaKind kind, Value input) {
         AllocatableValue operand = Value.ILLEGAL;
@@ -346,6 +502,17 @@ public class AArch64HotSpotLIRGenerator extends AArch64LIRGenerator implements H
         append(new AArch64HotSpotReturnOp(operand, getStub() != null, config, thread));
     }
 
+    @Override
+    public Value emitKlassInitializationAndRetrieval(Constant constant, Value constantDescription, LIRFrameState frameState) {
+        return emitConstantRetrieval(INITIALIZE_KLASS_BY_SYMBOL, INITIALIZE, constant, constantDescription, frameState);
+    }
+
+    @Override
+    public Value emitResolveMethodAndLoadCounters(Constant method, Value klassHint, Value methodDescription, LIRFrameState frameState) {
+        AllocatableValue[] constantDescriptions = new AllocatableValue[]{asAllocatable(klassHint), asAllocatable(methodDescription)};
+        return emitConstantRetrieval(RESOLVE_METHOD_BY_SYMBOL_AND_LOAD_COUNTERS, LOAD_COUNTERS, method, constantDescriptions, frameState);
+    }
+
     /**
      * Gets the {@link Stub} this generator is generating code for or {@code null} if a stub is not
      * being generated.

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotLoadAddressOp.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.hotspot.aarch64;
+
+import static jdk.vm.ci.code.ValueUtil.asRegister;
+
+import jdk.vm.ci.aarch64.AArch64Kind;
+import jdk.vm.ci.code.Register;
+import jdk.vm.ci.meta.AllocatableValue;
+import jdk.vm.ci.meta.Constant;
+
+import org.graalvm.compiler.asm.aarch64.AArch64Address;
+import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler;
+import org.graalvm.compiler.debug.GraalError;
+import org.graalvm.compiler.lir.LIRInstructionClass;
+import org.graalvm.compiler.lir.aarch64.AArch64LIRInstruction;
+import org.graalvm.compiler.lir.asm.CompilationResultBuilder;
+
+public final class AArch64HotSpotLoadAddressOp extends AArch64LIRInstruction {
+
+    public static final LIRInstructionClass<AArch64HotSpotLoadAddressOp> TYPE = LIRInstructionClass.create(AArch64HotSpotLoadAddressOp.class);
+
+    @Def({OperandFlag.REG}) protected AllocatableValue result;
+    private final Constant constant;
+    private final Object note;
+
+    public AArch64HotSpotLoadAddressOp(AllocatableValue result, Constant constant, Object note) {
+        super(TYPE);
+        this.result = result;
+        this.constant = constant;
+        this.note = note;
+    }
+
+    @Override
+    public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+        crb.recordInlineDataInCodeWithNote(constant, note);
+        AArch64Kind kind = (AArch64Kind) result.getPlatformKind();
+        int size = 0;
+        switch (kind) {
+            case DWORD:
+                size = 32;
+                break;
+            case QWORD:
+                size = 64;
+                break;
+            default:
+                throw GraalError.shouldNotReachHere("unexpected kind: " + kind);
+        }
+        if (crb.compilationResult.isImmutablePIC()) {
+            Register dst = asRegister(result);
+            masm.addressOf(dst);
+            masm.ldr(size, dst, AArch64Address.createBaseRegisterOnlyAddress(dst));
+        } else {
+            masm.ldr(size, asRegister(result), masm.getPlaceholder(-1));
+        }
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotLoadConfigValueOp.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, Red Hat Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.hotspot.aarch64;
+
+import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
+import static jdk.vm.ci.code.ValueUtil.asRegister;
+
+import jdk.vm.ci.aarch64.AArch64Kind;
+import jdk.vm.ci.code.Register;
+import jdk.vm.ci.meta.AllocatableValue;
+
+import org.graalvm.compiler.asm.aarch64.AArch64Address;
+import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler;
+import org.graalvm.compiler.debug.GraalError;
+import org.graalvm.compiler.lir.LIRInstructionClass;
+import org.graalvm.compiler.lir.aarch64.AArch64LIRInstruction;
+import org.graalvm.compiler.lir.asm.CompilationResultBuilder;
+
+public final class AArch64HotSpotLoadConfigValueOp extends AArch64LIRInstruction {
+
+    public static final LIRInstructionClass<AArch64HotSpotLoadConfigValueOp> TYPE = LIRInstructionClass.create(AArch64HotSpotLoadConfigValueOp.class);
+
+    @Def({OperandFlag.REG}) protected AllocatableValue result;
+    private final int markId;
+
+    public AArch64HotSpotLoadConfigValueOp(int markId, AllocatableValue result) {
+        super(TYPE);
+        this.result = result;
+        this.markId = markId;
+    }
+
+    @Override
+    public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+        if (GeneratePIC.getValue(crb.getOptions())) {
+            AArch64Kind kind = (AArch64Kind) result.getPlatformKind();
+            Register reg = asRegister(result);
+            masm.adrp(reg);
+            masm.add(64, reg, reg, 1);
+            switch (kind) {
+                case BYTE:
+                    masm.ldrs(8, 32, reg, AArch64Address.createBaseRegisterOnlyAddress(reg));
+                    break;
+                case WORD:
+                    masm.ldrs(16, 32, reg, AArch64Address.createBaseRegisterOnlyAddress(reg));
+                    break;
+                case DWORD:
+                    masm.ldr(32, reg, AArch64Address.createBaseRegisterOnlyAddress(reg));
+                    break;
+                case QWORD:
+                    masm.ldr(64, reg, AArch64Address.createBaseRegisterOnlyAddress(reg));
+                    break;
+                default:
+                    throw GraalError.unimplemented();
+            }
+            masm.nop();
+        } else {
+            throw GraalError.unimplemented();
+        }
+        crb.recordMark(markId);
+    }
+
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotLoweringProvider.java

@@ -26,13 +26,13 @@ package org.graalvm.compiler.hotspot.aarch64;
 import org.graalvm.compiler.core.common.spi.ForeignCallsProvider;
 import org.graalvm.compiler.debug.DebugHandlersFactory;
 import org.graalvm.compiler.graph.Node;
-import org.graalvm.compiler.hotspot.HotSpotGraalRuntimeProvider;
 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
+import org.graalvm.compiler.hotspot.HotSpotGraalRuntimeProvider;
 import org.graalvm.compiler.hotspot.meta.DefaultHotSpotLoweringProvider;
 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
-import org.graalvm.compiler.nodes.calc.FixedBinaryNode;
 import org.graalvm.compiler.nodes.calc.FloatConvertNode;
+import org.graalvm.compiler.nodes.calc.IntegerDivRemNode;
 import org.graalvm.compiler.nodes.calc.RemNode;
 import org.graalvm.compiler.nodes.spi.LoweringTool;
 import org.graalvm.compiler.options.OptionValues;
@@ -62,8 +62,8 @@ public class AArch64HotSpotLoweringProvider extends DefaultHotSpotLoweringProvid
 
     @Override
     public void lower(Node n, LoweringTool tool) {
-        if (n instanceof FixedBinaryNode) {
+        if (n instanceof IntegerDivRemNode) {
-            integerArithmeticSnippets.lower((FixedBinaryNode) n, tool);
+            integerArithmeticSnippets.lower((IntegerDivRemNode) n, tool);
         } else if (n instanceof RemNode) {
             floatArithmeticSnippets.lower((RemNode) n, tool);
         } else if (n instanceof FloatConvertNode) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotMove.java

@@ -25,15 +25,19 @@ package org.graalvm.compiler.hotspot.aarch64;
 import static jdk.vm.ci.aarch64.AArch64.zr;
 import static jdk.vm.ci.code.ValueUtil.asRegister;
 import static jdk.vm.ci.code.ValueUtil.isRegister;
+import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.HINT;
 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.ILLEGAL;
 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG;
 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK;
 
 import org.graalvm.compiler.asm.Label;
+import org.graalvm.compiler.asm.aarch64.AArch64Address;
 import org.graalvm.compiler.asm.aarch64.AArch64Assembler;
 import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler;
 import org.graalvm.compiler.core.common.CompressEncoding;
+import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
+import org.graalvm.compiler.lir.LIRInstruction;
 import org.graalvm.compiler.lir.LIRInstructionClass;
 import org.graalvm.compiler.lir.StandardOp.LoadConstantOp;
 import org.graalvm.compiler.lir.aarch64.AArch64LIRInstruction;
@@ -80,6 +84,32 @@ public class AArch64HotSpotMove {
         }
     }
 
+    public static final class BaseMove extends AArch64LIRInstruction {
+        public static final LIRInstructionClass<BaseMove> TYPE = LIRInstructionClass.create(BaseMove.class);
+
+        @LIRInstruction.Def({REG, HINT}) protected AllocatableValue result;
+        private final GraalHotSpotVMConfig config;
+
+        public BaseMove(AllocatableValue result, GraalHotSpotVMConfig config) {
+            super(TYPE);
+            this.result = result;
+            this.config = config;
+        }
+
+        @Override
+        public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+            try (AArch64MacroAssembler.ScratchRegister sc = masm.getScratchRegister()) {
+                Register scratch = sc.getRegister();
+                masm.adrp(scratch);
+                masm.add(64, scratch, scratch, 1);
+                masm.ldr(64, asRegister(result), AArch64Address.createBaseRegisterOnlyAddress(scratch));
+                masm.nop();
+                crb.recordMark(config.MARKID_NARROW_KLASS_BASE_ADDRESS);
+            }
+        }
+
+    }
+
     /**
      * Compresses a 8-byte pointer as a 4-byte int.
      */
@@ -117,7 +147,7 @@ public class AArch64HotSpotMove {
             } else if (nonNull) {
                 masm.sub(64, resultRegister, ptr, base);
                 if (encoding.hasShift()) {
-                    masm.shl(64, resultRegister, resultRegister, encoding.getShift());
+                    masm.lshr(64, resultRegister, resultRegister, encoding.getShift());
                 }
             } else {
                 // if ptr is null it still has to be null after compression
@@ -192,11 +222,21 @@ public class AArch64HotSpotMove {
     // masm.cmov(64, result, result, ARMv8.zr, ARMv8Assembler.ConditionFlag.NE);
     // }
 
-    public static void decodeKlassPointer(AArch64MacroAssembler masm, Register result, Register ptr, Register klassBase, CompressEncoding encoding) {
+    public static void decodeKlassPointer(CompilationResultBuilder crb, AArch64MacroAssembler masm, Register result, Register ptr, CompressEncoding encoding, GraalHotSpotVMConfig config) {
-        // result = klassBase + ptr << shift
+        try (AArch64MacroAssembler.ScratchRegister sc = masm.getScratchRegister()) {
-        if (encoding.hasShift() || encoding.hasBase()) {
+            Register scratch = sc.getRegister();
-            masm.add(64, result, klassBase, ptr, AArch64Assembler.ExtendType.UXTX, encoding.getShift());
+            boolean pic = GeneratePIC.getValue(crb.getOptions());
+            if (pic || encoding.hasBase() || encoding.getShift() != 0) {
+                if (pic) {
+                    masm.addressOf(scratch);
+                    masm.ldr(64, scratch, AArch64Address.createBaseRegisterOnlyAddress(scratch));
+                    masm.add(64, result, scratch, ptr, AArch64Assembler.ExtendType.UXTX, encoding.getShift());
+                    crb.recordMark(config.MARKID_NARROW_KLASS_BASE_ADDRESS);
+                } else {
+                    masm.mov(scratch, encoding.getBase());
+                    masm.add(64, result, scratch, ptr, AArch64Assembler.ExtendType.UXTX, encoding.getShift());
+                }
+            }
         }
     }
-
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotLIRGenerator.java

@@ -236,7 +236,7 @@ public class SPARCHotSpotLIRGenerator extends SPARCLIRGenerator implements HotSp
     }
 
     @Override
-    public Variable emitLogicCompareAndSwap(Value address, Value expectedValue, Value newValue, Value trueValue, Value falseValue) {
+    public Variable emitLogicCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue, Value trueValue, Value falseValue) {
         ValueKind<?> kind = newValue.getValueKind();
         assert kind.equals(expectedValue.getValueKind());
         SPARCKind memKind = (SPARCKind) kind.getPlatformKind();
@@ -246,7 +246,7 @@ public class SPARCHotSpotLIRGenerator extends SPARCLIRGenerator implements HotSp
     }
 
     @Override
-    public Variable emitValueCompareAndSwap(Value address, Value expectedValue, Value newValue) {
+    public Variable emitValueCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue) {
         ValueKind<?> kind = newValue.getValueKind();
         assert kind.equals(expectedValue.getValueKind());
         Variable result = newVariable(newValue.getValueKind());

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CheckGraalIntrinsics.java

@@ -422,6 +422,12 @@ public class CheckGraalIntrinsics extends GraalTest {
                             "jdk/internal/util/ArraysSupport.vectorizedMismatch(Ljava/lang/Object;JLjava/lang/Object;JII)I");
         }
 
+        if (isJDK11OrHigher()) {
+            // Relevant for Java flight recorder
+            add(TO_BE_INVESTIGATED,
+                            "jdk/jfr/internal/JVM.getEventWriter()Ljava/lang/Object;");
+        }
+
         if (!getHostArchitectureName().equals("amd64")) {
             // Can we implement these on non-AMD64 platforms? C2 seems to.
             add(TO_BE_INVESTIGATED,
@@ -549,6 +555,10 @@ public class CheckGraalIntrinsics extends GraalTest {
         return GraalServices.JAVA_SPECIFICATION_VERSION >= 10;
     }
 
+    private static boolean isJDK11OrHigher() {
+        return GraalServices.JAVA_SPECIFICATION_VERSION >= 11;
+    }
+
     private static String getHostArchitectureName() {
         String arch = System.getProperty("os.arch");
         if (arch.equals("x86_64")) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CompilationWrapperTest.java

@@ -223,25 +223,19 @@ public class CompilationWrapperTest extends GraalCompilerTest {
             Assert.assertTrue(zip.toString(), zip.exists());
             Assert.assertTrue(zip + " not in " + dumpPathEntries, dumpPathEntries.contains(zip.getName()));
             try {
-                int bgv = 0;
+                int bgvOrCfgFiles = 0;
-                int cfg = 0;
                 ZipFile dd = new ZipFile(diagnosticOutputZip);
                 List<String> entries = new ArrayList<>();
                 for (Enumeration<? extends ZipEntry> e = dd.entries(); e.hasMoreElements();) {
                     ZipEntry ze = e.nextElement();
                     String name = ze.getName();
                     entries.add(name);
-                    if (name.endsWith(".bgv")) {
+                    if (name.endsWith(".bgv") || name.endsWith(".cfg")) {
-                        bgv++;
+                        bgvOrCfgFiles++;
-                    } else if (name.endsWith(".cfg")) {
-                        cfg++;
                     }
                 }
-                if (bgv == 0) {
+                if (bgvOrCfgFiles == 0) {
-                    Assert.fail(String.format("Expected at least one .bgv file in %s: %s%n%s", diagnosticOutputZip, entries, proc));
+                    Assert.fail(String.format("Expected at least one .bgv or .cfg file in %s: %s%n%s", diagnosticOutputZip, entries, proc));
-                }
-                if (cfg == 0) {
-                    Assert.fail(String.format("Expected at least one .cfg file in %s: %s%n%s", diagnosticOutputZip, entries, proc));
                 }
             } finally {
                 zip.delete();

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/aaa

@@ -1,78 +0,0 @@
-/*
- * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-package org.graalvm.compiler.hotspot.test;
-
-import org.graalvm.compiler.core.test.GraalCompilerTest;
-import org.graalvm.compiler.hotspot.meta.HotSpotClassInitializationPlugin;
-import org.graalvm.compiler.hotspot.nodes.aot.InitializeKlassNode;
-import org.graalvm.compiler.java.GraphBuilderPhase;
-import org.graalvm.compiler.nodes.StructuredGraph;
-import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions;
-import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.Plugins;
-import org.graalvm.compiler.phases.PhaseSuite;
-import org.graalvm.compiler.phases.tiers.HighTierContext;
-import org.junit.Assert;
-import org.junit.Assume;
-import org.junit.Before;
-import org.junit.Test;
-
-import jdk.vm.ci.meta.ConstantPool;
-import jdk.vm.ci.meta.ResolvedJavaMethod;
-
-public class HotSpotLazyInitializationTest extends GraalCompilerTest {
-
-    HotSpotClassInitializationPlugin classInitPlugin = new HotSpotClassInitializationPlugin();
-
-    @Override
-    protected Plugins getDefaultGraphBuilderPlugins() {
-        Plugins plugins = super.getDefaultGraphBuilderPlugins();
-        plugins.setClassInitializationPlugin(classInitPlugin);
-        return plugins;
-    }
-
-    static boolean X_initialized = false;
-
-    static class X {
-        static {
-            X_initialized = true;
-        }
-        static void foo() {}
-    }
-
-    public static void invokeStatic() {
-        X.foo();
-    }
-
-    private void test(String name) {
-        ResolvedJavaMethod method = getResolvedJavaMethod(name);
-        Assume.assumeTrue("skipping for old JVMCI", classInitPlugin.supportsLazyInitialization(method.getConstantPool()));
-        StructuredGraph graph = parseEager(method, AllowAssumptions.NO);
-        Assert.assertFalse(X_initialized);
-    }
-
-    @Test
-    public void test1() {
-        test("invokeStatic");
-    }
-
-}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/CompilerConfigurationFactory.java

@@ -101,6 +101,13 @@ public abstract class CompilerConfigurationFactory implements Comparable<Compile
         return new DefaultBackendMap(name);
     }
 
+    /**
+     * Returns a name that should uniquely identify this compiler configuration.
+     */
+    public final String getName() {
+        return name;
+    }
+
     public interface BackendMap {
         HotSpotBackendFactory getBackendFactory(Architecture arch);
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/GraalHotSpotVMConfig.java

@@ -80,6 +80,9 @@ public class GraalHotSpotVMConfig extends GraalHotSpotVMConfigBase {
     public final boolean foldStableValues = getFlag("FoldStableValues", Boolean.class);
     public final int maxVectorSize = getFlag("MaxVectorSize", Integer.class);
 
+    public final boolean verifyBeforeGC = getFlag("VerifyBeforeGC", Boolean.class);
+    public final boolean verifyAfterGC = getFlag("VerifyAfterGC", Boolean.class);
+
     public final boolean useTLAB = getFlag("UseTLAB", Boolean.class);
     public final boolean useBiasedLocking = getFlag("UseBiasedLocking", Boolean.class);
     public final boolean usePopCountInstruction = getFlag("UsePopCountInstruction", Boolean.class);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/GraalHotSpotVMConfigVersioned.java

@@ -40,12 +40,12 @@ final class GraalHotSpotVMConfigVersioned extends HotSpotVMConfigAccess {
         super(store);
     }
 
+    // JSK-8132287
+    final boolean inlineNotify = !getFlag("SyncKnobs", String.class, "").contains("InlineNotify=0");
+
     // JDK-8073583
     final boolean useCRC32CIntrinsics = getFlag("UseCRC32CIntrinsics", Boolean.class);
 
-    // JDK-8075171
-    final boolean inlineNotify = getFlag("InlineNotify", Boolean.class, true);
-
     // JDK-8046936
     final int javaThreadReservedStackActivationOffset = getFieldOffset("JavaThread::_reserved_stack_activation", Integer.class, "address");
     final int methodFlagsOffset = getFieldOffset("Method::_flags", Integer.class, "u2");

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalCompiler.java

@@ -185,7 +185,7 @@ public class HotSpotGraalCompiler implements GraalJVMCICompiler {
         result.setEntryBCI(entryBCI);
         boolean shouldDebugNonSafepoints = providers.getCodeCache().shouldDebugNonSafepoints();
         PhaseSuite<HighTierContext> graphBuilderSuite = configGraphBuilderSuite(providers.getSuites().getDefaultGraphBuilderSuite(), shouldDebugNonSafepoints, isOSR);
-        GraalCompiler.compileGraph(graph, method, providers, backend, graphBuilderSuite, optimisticOpts, profilingInfo, suites, lirSuites, result, crbf);
+        GraalCompiler.compileGraph(graph, method, providers, backend, graphBuilderSuite, optimisticOpts, profilingInfo, suites, lirSuites, result, crbf, true);
 
         if (!isOSR && useProfilingInfo) {
             ProfilingInfo profile = profilingInfo;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalRuntime.java

@@ -105,6 +105,7 @@ public final class HotSpotGraalRuntime implements HotSpotGraalRuntimeProvider {
     }
 
     private final String runtimeName;
+    private final String compilerConfigurationName;
     private final HotSpotBackend hostBackend;
     private final GlobalMetrics metricValues = new GlobalMetrics();
     private final List<SnippetCounter.Group> snippetCounterGroups;
@@ -155,6 +156,7 @@ public final class HotSpotGraalRuntime implements HotSpotGraalRuntimeProvider {
         compilationProblemsPerAction = new EnumMap<>(ExceptionAction.class);
         snippetCounterGroups = GraalOptions.SnippetCounters.getValue(options) ? new ArrayList<>() : null;
         CompilerConfiguration compilerConfiguration = compilerConfigurationFactory.createCompilerConfiguration();
+        compilerConfigurationName = compilerConfigurationFactory.getName();
 
         compiler = new HotSpotGraalCompiler(jvmciRuntime, this, options);
         management = GraalServices.loadSingle(HotSpotGraalManagementRegistration.class, false);
@@ -296,6 +298,11 @@ public final class HotSpotGraalRuntime implements HotSpotGraalRuntimeProvider {
         return backends.get(arch);
     }
 
+    @Override
+    public String getCompilerConfigurationName() {
+        return compilerConfigurationName;
+    }
+
     private long runtimeStartTime;
     private boolean shutdown;
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalRuntimeProvider.java

@@ -94,4 +94,10 @@ public interface HotSpotGraalRuntimeProvider extends GraalRuntime, RuntimeProvid
      * Gets the map used to count compilation problems at each {@link ExceptionAction} level.
      */
     Map<ExceptionAction, Integer> getCompilationProblemsPerAction();
+
+    /**
+     * Returns the unique compiler configuration name that is in use. Useful for users to find out
+     * which configuration is in use.
+     */
+    String getCompilerConfigurationName();
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -39,6 +39,7 @@ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.
 import static jdk.internal.vm.compiler.word.LocationIdentity.any;
 
 import java.lang.ref.Reference;
+import java.util.EnumMap;
 
 import org.graalvm.compiler.api.directives.GraalDirectives;
 import org.graalvm.compiler.core.common.CompressEncoding;
@@ -49,6 +50,7 @@ import org.graalvm.compiler.core.common.type.ObjectStamp;
 import org.graalvm.compiler.core.common.type.Stamp;
 import org.graalvm.compiler.core.common.type.StampFactory;
 import org.graalvm.compiler.core.common.type.StampPair;
+import org.graalvm.compiler.debug.DebugCloseable;
 import org.graalvm.compiler.debug.DebugHandlersFactory;
 import org.graalvm.compiler.debug.GraalError;
 import org.graalvm.compiler.graph.Node;
@@ -91,8 +93,8 @@ import org.graalvm.compiler.hotspot.replacements.UnsafeLoadSnippets;
 import org.graalvm.compiler.hotspot.replacements.WriteBarrierSnippets;
 import org.graalvm.compiler.hotspot.replacements.aot.ResolveConstantSnippets;
 import org.graalvm.compiler.hotspot.replacements.arraycopy.ArrayCopyNode;
-import org.graalvm.compiler.hotspot.replacements.arraycopy.ArrayCopyWithSlowPathNode;
 import org.graalvm.compiler.hotspot.replacements.arraycopy.ArrayCopySnippets;
+import org.graalvm.compiler.hotspot.replacements.arraycopy.ArrayCopyWithSlowPathNode;
 import org.graalvm.compiler.hotspot.replacements.profiling.ProfileSnippets;
 import org.graalvm.compiler.hotspot.word.KlassPointer;
 import org.graalvm.compiler.nodes.AbstractBeginNode;
@@ -118,6 +120,7 @@ import org.graalvm.compiler.nodes.calc.RemNode;
 import org.graalvm.compiler.nodes.debug.StringToBytesNode;
 import org.graalvm.compiler.nodes.debug.VerifyHeapNode;
 import org.graalvm.compiler.nodes.extended.BytecodeExceptionNode;
+import org.graalvm.compiler.nodes.extended.BytecodeExceptionNode.BytecodeExceptionKind;
 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
 import org.graalvm.compiler.nodes.extended.GetClassNode;
 import org.graalvm.compiler.nodes.extended.GuardedUnsafeLoadNode;
@@ -207,7 +210,7 @@ public class DefaultHotSpotLoweringProvider extends DefaultJavaLoweringProvider
         instanceofSnippets = new InstanceOfSnippets.Templates(options, factories, runtime, providers, target);
         newObjectSnippets = new NewObjectSnippets.Templates(options, factories, runtime, providers, target, config);
         monitorSnippets = new MonitorSnippets.Templates(options, factories, runtime, providers, target, config.useFastLocking);
-        writeBarrierSnippets = new WriteBarrierSnippets.Templates(options, factories, runtime, providers, target, config.useCompressedOops ? config.getOopEncoding() : null);
+        writeBarrierSnippets = new WriteBarrierSnippets.Templates(options, factories, runtime, providers, target, config);
         exceptionObjectSnippets = new LoadExceptionObjectSnippets.Templates(options, factories, providers, target);
         unsafeLoadSnippets = new UnsafeLoadSnippets.Templates(options, factories, providers, target);
         assertionSnippets = new AssertionSnippets.Templates(options, factories, providers, target);
@@ -223,162 +226,165 @@ public class DefaultHotSpotLoweringProvider extends DefaultJavaLoweringProvider
     }
 
     @Override
+    @SuppressWarnings("try")
     public void lower(Node n, LoweringTool tool) {
         StructuredGraph graph = (StructuredGraph) n.graph();
-        if (n instanceof Invoke) {
+        try (DebugCloseable context = n.withNodeSourcePosition()) {
-            lowerInvoke((Invoke) n, tool, graph);
+            if (n instanceof Invoke) {
-        } else if (n instanceof LoadMethodNode) {
+                lowerInvoke((Invoke) n, tool, graph);
-            lowerLoadMethodNode((LoadMethodNode) n);
+            } else if (n instanceof LoadMethodNode) {
-        } else if (n instanceof GetClassNode) {
+                lowerLoadMethodNode((LoadMethodNode) n);
-            lowerGetClassNode((GetClassNode) n, tool, graph);
+            } else if (n instanceof GetClassNode) {
-        } else if (n instanceof StoreHubNode) {
+                lowerGetClassNode((GetClassNode) n, tool, graph);
-            lowerStoreHubNode((StoreHubNode) n, graph);
+            } else if (n instanceof StoreHubNode) {
-        } else if (n instanceof OSRStartNode) {
+                lowerStoreHubNode((StoreHubNode) n, graph);
-            lowerOSRStartNode((OSRStartNode) n);
+            } else if (n instanceof OSRStartNode) {
-        } else if (n instanceof BytecodeExceptionNode) {
+                lowerOSRStartNode((OSRStartNode) n);
-            lowerBytecodeExceptionNode((BytecodeExceptionNode) n);
+            } else if (n instanceof BytecodeExceptionNode) {
-        } else if (n instanceof InstanceOfNode) {
+                lowerBytecodeExceptionNode((BytecodeExceptionNode) n);
-            InstanceOfNode instanceOfNode = (InstanceOfNode) n;
+            } else if (n instanceof InstanceOfNode) {
-            if (graph.getGuardsStage().areDeoptsFixed()) {
+                InstanceOfNode instanceOfNode = (InstanceOfNode) n;
-                instanceofSnippets.lower(instanceOfNode, tool);
+                if (graph.getGuardsStage().areDeoptsFixed()) {
-            } else {
+                    instanceofSnippets.lower(instanceOfNode, tool);
-                if (instanceOfNode.allowsNull()) {
+                } else {
-                    ValueNode object = instanceOfNode.getValue();
+                    if (instanceOfNode.allowsNull()) {
-                    LogicNode newTypeCheck = graph.addOrUniqueWithInputs(InstanceOfNode.create(instanceOfNode.type(), object, instanceOfNode.profile(), instanceOfNode.getAnchor()));
+                        ValueNode object = instanceOfNode.getValue();
-                    LogicNode newNode = LogicNode.or(graph.unique(IsNullNode.create(object)), newTypeCheck, GraalDirectives.UNLIKELY_PROBABILITY);
+                        LogicNode newTypeCheck = graph.addOrUniqueWithInputs(InstanceOfNode.create(instanceOfNode.type(), object, instanceOfNode.profile(), instanceOfNode.getAnchor()));
-                    instanceOfNode.replaceAndDelete(newNode);
+                        LogicNode newNode = LogicNode.or(graph.unique(IsNullNode.create(object)), newTypeCheck, GraalDirectives.UNLIKELY_PROBABILITY);
-                }
+                        instanceOfNode.replaceAndDelete(newNode);
-            }
+                    }
-        } else if (n instanceof InstanceOfDynamicNode) {
-            InstanceOfDynamicNode instanceOfDynamicNode = (InstanceOfDynamicNode) n;
-            if (graph.getGuardsStage().areDeoptsFixed()) {
-                instanceofSnippets.lower(instanceOfDynamicNode, tool);
-            } else {
-                ValueNode mirror = instanceOfDynamicNode.getMirrorOrHub();
-                if (mirror.stamp(NodeView.DEFAULT).getStackKind() == JavaKind.Object) {
-                    ClassGetHubNode classGetHub = graph.unique(new ClassGetHubNode(mirror));
-                    instanceOfDynamicNode.setMirror(classGetHub);
                 }
+            } else if (n instanceof InstanceOfDynamicNode) {
+                InstanceOfDynamicNode instanceOfDynamicNode = (InstanceOfDynamicNode) n;
+                if (graph.getGuardsStage().areDeoptsFixed()) {
+                    instanceofSnippets.lower(instanceOfDynamicNode, tool);
+                } else {
+                    ValueNode mirror = instanceOfDynamicNode.getMirrorOrHub();
+                    if (mirror.stamp(NodeView.DEFAULT).getStackKind() == JavaKind.Object) {
+                        ClassGetHubNode classGetHub = graph.unique(new ClassGetHubNode(mirror));
+                        instanceOfDynamicNode.setMirror(classGetHub);
+                    }
 
-                if (instanceOfDynamicNode.allowsNull()) {
+                    if (instanceOfDynamicNode.allowsNull()) {
-                    ValueNode object = instanceOfDynamicNode.getObject();
+                        ValueNode object = instanceOfDynamicNode.getObject();
-                    LogicNode newTypeCheck = graph.addOrUniqueWithInputs(
+                        LogicNode newTypeCheck = graph.addOrUniqueWithInputs(
-                                    InstanceOfDynamicNode.create(graph.getAssumptions(), tool.getConstantReflection(), instanceOfDynamicNode.getMirrorOrHub(), object, false));
+                                        InstanceOfDynamicNode.create(graph.getAssumptions(), tool.getConstantReflection(), instanceOfDynamicNode.getMirrorOrHub(), object, false));
-                    LogicNode newNode = LogicNode.or(graph.unique(IsNullNode.create(object)), newTypeCheck, GraalDirectives.UNLIKELY_PROBABILITY);
+                        LogicNode newNode = LogicNode.or(graph.unique(IsNullNode.create(object)), newTypeCheck, GraalDirectives.UNLIKELY_PROBABILITY);
-                    instanceOfDynamicNode.replaceAndDelete(newNode);
+                        instanceOfDynamicNode.replaceAndDelete(newNode);
+                    }
                 }
+            } else if (n instanceof ClassIsAssignableFromNode) {
+                if (graph.getGuardsStage().areDeoptsFixed()) {
+                    instanceofSnippets.lower((ClassIsAssignableFromNode) n, tool);
+                }
+            } else if (n instanceof NewInstanceNode) {
+                if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
+                    newObjectSnippets.lower((NewInstanceNode) n, registers, tool);
+                }
+            } else if (n instanceof DynamicNewInstanceNode) {
+                DynamicNewInstanceNode newInstanceNode = (DynamicNewInstanceNode) n;
+                if (newInstanceNode.getClassClass() == null) {
+                    JavaConstant classClassMirror = constantReflection.forObject(Class.class);
+                    ConstantNode classClass = ConstantNode.forConstant(classClassMirror, tool.getMetaAccess(), graph);
+                    newInstanceNode.setClassClass(classClass);
+                }
+                if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
+                    newObjectSnippets.lower(newInstanceNode, registers, tool);
+                }
+            } else if (n instanceof NewArrayNode) {
+                if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
+                    newObjectSnippets.lower((NewArrayNode) n, registers, tool);
+                }
+            } else if (n instanceof DynamicNewArrayNode) {
+                DynamicNewArrayNode dynamicNewArrayNode = (DynamicNewArrayNode) n;
+                if (dynamicNewArrayNode.getVoidClass() == null) {
+                    JavaConstant voidClassMirror = constantReflection.forObject(void.class);
+                    ConstantNode voidClass = ConstantNode.forConstant(voidClassMirror, tool.getMetaAccess(), graph);
+                    dynamicNewArrayNode.setVoidClass(voidClass);
+                }
+                if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
+                    newObjectSnippets.lower(dynamicNewArrayNode, registers, tool);
+                }
+            } else if (n instanceof VerifyHeapNode) {
+                if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
+                    newObjectSnippets.lower((VerifyHeapNode) n, registers, tool);
+                }
+            } else if (n instanceof RawMonitorEnterNode) {
+                if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
+                    monitorSnippets.lower((RawMonitorEnterNode) n, registers, tool);
+                }
+            } else if (n instanceof MonitorExitNode) {
+                if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
+                    monitorSnippets.lower((MonitorExitNode) n, registers, tool);
+                }
+            } else if (n instanceof ArrayCopyNode) {
+                arraycopySnippets.lower((ArrayCopyNode) n, tool);
+            } else if (n instanceof ArrayCopyWithSlowPathNode) {
+                arraycopySnippets.lower((ArrayCopyWithSlowPathNode) n, tool);
+            } else if (n instanceof G1PreWriteBarrier) {
+                writeBarrierSnippets.lower((G1PreWriteBarrier) n, registers, tool);
+            } else if (n instanceof G1PostWriteBarrier) {
+                writeBarrierSnippets.lower((G1PostWriteBarrier) n, registers, tool);
+            } else if (n instanceof G1ReferentFieldReadBarrier) {
+                writeBarrierSnippets.lower((G1ReferentFieldReadBarrier) n, registers, tool);
+            } else if (n instanceof SerialWriteBarrier) {
+                writeBarrierSnippets.lower((SerialWriteBarrier) n, tool);
+            } else if (n instanceof SerialArrayRangeWriteBarrier) {
+                writeBarrierSnippets.lower((SerialArrayRangeWriteBarrier) n, tool);
+            } else if (n instanceof G1ArrayRangePreWriteBarrier) {
+                writeBarrierSnippets.lower((G1ArrayRangePreWriteBarrier) n, registers, tool);
+            } else if (n instanceof G1ArrayRangePostWriteBarrier) {
+                writeBarrierSnippets.lower((G1ArrayRangePostWriteBarrier) n, registers, tool);
+            } else if (n instanceof NewMultiArrayNode) {
+                if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
+                    newObjectSnippets.lower((NewMultiArrayNode) n, tool);
+                }
+            } else if (n instanceof LoadExceptionObjectNode) {
+                exceptionObjectSnippets.lower((LoadExceptionObjectNode) n, registers, tool);
+            } else if (n instanceof AssertionNode) {
+                assertionSnippets.lower((AssertionNode) n, tool);
+            } else if (n instanceof StringToBytesNode) {
+                if (graph.getGuardsStage().areDeoptsFixed()) {
+                    stringToBytesSnippets.lower((StringToBytesNode) n, tool);
+                }
+            } else if (n instanceof IntegerDivRemNode) {
+                // Nothing to do for division nodes. The HotSpot signal handler catches divisions by
+                // zero and the MIN_VALUE / -1 cases.
+            } else if (n instanceof AbstractDeoptimizeNode || n instanceof UnwindNode || n instanceof RemNode || n instanceof SafepointNode) {
+                /* No lowering, we generate LIR directly for these nodes. */
+            } else if (n instanceof ClassGetHubNode) {
+                lowerClassGetHubNode((ClassGetHubNode) n, tool);
+            } else if (n instanceof HubGetClassNode) {
+                lowerHubGetClassNode((HubGetClassNode) n, tool);
+            } else if (n instanceof KlassLayoutHelperNode) {
+                lowerKlassLayoutHelperNode((KlassLayoutHelperNode) n, tool);
+            } else if (n instanceof ComputeObjectAddressNode) {
+                if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
+                    lowerComputeObjectAddressNode((ComputeObjectAddressNode) n);
+                }
+            } else if (n instanceof IdentityHashCodeNode) {
+                hashCodeSnippets.lower((IdentityHashCodeNode) n, tool);
+            } else if (n instanceof ResolveDynamicConstantNode) {
+                if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
+                    resolveConstantSnippets.lower((ResolveDynamicConstantNode) n, tool);
+                }
+            } else if (n instanceof ResolveConstantNode) {
+                if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
+                    resolveConstantSnippets.lower((ResolveConstantNode) n, tool);
+                }
+            } else if (n instanceof ResolveMethodAndLoadCountersNode) {
+                if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
+                    resolveConstantSnippets.lower((ResolveMethodAndLoadCountersNode) n, tool);
+                }
+            } else if (n instanceof InitializeKlassNode) {
+                if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
+                    resolveConstantSnippets.lower((InitializeKlassNode) n, tool);
+                }
+            } else if (n instanceof ProfileNode) {
+                profileSnippets.lower((ProfileNode) n, tool);
+            } else {
+                super.lower(n, tool);
             }
-        } else if (n instanceof ClassIsAssignableFromNode) {
-            if (graph.getGuardsStage().areDeoptsFixed()) {
-                instanceofSnippets.lower((ClassIsAssignableFromNode) n, tool);
-            }
-        } else if (n instanceof NewInstanceNode) {
-            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
-                newObjectSnippets.lower((NewInstanceNode) n, registers, tool);
-            }
-        } else if (n instanceof DynamicNewInstanceNode) {
-            DynamicNewInstanceNode newInstanceNode = (DynamicNewInstanceNode) n;
-            if (newInstanceNode.getClassClass() == null) {
-                JavaConstant classClassMirror = constantReflection.forObject(Class.class);
-                ConstantNode classClass = ConstantNode.forConstant(classClassMirror, tool.getMetaAccess(), graph);
-                newInstanceNode.setClassClass(classClass);
-            }
-            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
-                newObjectSnippets.lower(newInstanceNode, registers, tool);
-            }
-        } else if (n instanceof NewArrayNode) {
-            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
-                newObjectSnippets.lower((NewArrayNode) n, registers, tool);
-            }
-        } else if (n instanceof DynamicNewArrayNode) {
-            DynamicNewArrayNode dynamicNewArrayNode = (DynamicNewArrayNode) n;
-            if (dynamicNewArrayNode.getVoidClass() == null) {
-                JavaConstant voidClassMirror = constantReflection.forObject(void.class);
-                ConstantNode voidClass = ConstantNode.forConstant(voidClassMirror, tool.getMetaAccess(), graph);
-                dynamicNewArrayNode.setVoidClass(voidClass);
-            }
-            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
-                newObjectSnippets.lower(dynamicNewArrayNode, registers, tool);
-            }
-        } else if (n instanceof VerifyHeapNode) {
-            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
-                newObjectSnippets.lower((VerifyHeapNode) n, registers, tool);
-            }
-        } else if (n instanceof RawMonitorEnterNode) {
-            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
-                monitorSnippets.lower((RawMonitorEnterNode) n, registers, tool);
-            }
-        } else if (n instanceof MonitorExitNode) {
-            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
-                monitorSnippets.lower((MonitorExitNode) n, registers, tool);
-            }
-        } else if (n instanceof ArrayCopyNode) {
-            arraycopySnippets.lower((ArrayCopyNode) n, tool);
-        } else if (n instanceof ArrayCopyWithSlowPathNode) {
-            arraycopySnippets.lower((ArrayCopyWithSlowPathNode) n, tool);
-        } else if (n instanceof G1PreWriteBarrier) {
-            writeBarrierSnippets.lower((G1PreWriteBarrier) n, registers, tool);
-        } else if (n instanceof G1PostWriteBarrier) {
-            writeBarrierSnippets.lower((G1PostWriteBarrier) n, registers, tool);
-        } else if (n instanceof G1ReferentFieldReadBarrier) {
-            writeBarrierSnippets.lower((G1ReferentFieldReadBarrier) n, registers, tool);
-        } else if (n instanceof SerialWriteBarrier) {
-            writeBarrierSnippets.lower((SerialWriteBarrier) n, tool);
-        } else if (n instanceof SerialArrayRangeWriteBarrier) {
-            writeBarrierSnippets.lower((SerialArrayRangeWriteBarrier) n, tool);
-        } else if (n instanceof G1ArrayRangePreWriteBarrier) {
-            writeBarrierSnippets.lower((G1ArrayRangePreWriteBarrier) n, registers, tool);
-        } else if (n instanceof G1ArrayRangePostWriteBarrier) {
-            writeBarrierSnippets.lower((G1ArrayRangePostWriteBarrier) n, registers, tool);
-        } else if (n instanceof NewMultiArrayNode) {
-            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
-                newObjectSnippets.lower((NewMultiArrayNode) n, tool);
-            }
-        } else if (n instanceof LoadExceptionObjectNode) {
-            exceptionObjectSnippets.lower((LoadExceptionObjectNode) n, registers, tool);
-        } else if (n instanceof AssertionNode) {
-            assertionSnippets.lower((AssertionNode) n, tool);
-        } else if (n instanceof StringToBytesNode) {
-            if (graph.getGuardsStage().areDeoptsFixed()) {
-                stringToBytesSnippets.lower((StringToBytesNode) n, tool);
-            }
-        } else if (n instanceof IntegerDivRemNode) {
-            // Nothing to do for division nodes. The HotSpot signal handler catches divisions by
-            // zero and the MIN_VALUE / -1 cases.
-        } else if (n instanceof AbstractDeoptimizeNode || n instanceof UnwindNode || n instanceof RemNode || n instanceof SafepointNode) {
-            /* No lowering, we generate LIR directly for these nodes. */
-        } else if (n instanceof ClassGetHubNode) {
-            lowerClassGetHubNode((ClassGetHubNode) n, tool);
-        } else if (n instanceof HubGetClassNode) {
-            lowerHubGetClassNode((HubGetClassNode) n, tool);
-        } else if (n instanceof KlassLayoutHelperNode) {
-            lowerKlassLayoutHelperNode((KlassLayoutHelperNode) n, tool);
-        } else if (n instanceof ComputeObjectAddressNode) {
-            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
-                lowerComputeObjectAddressNode((ComputeObjectAddressNode) n);
-            }
-        } else if (n instanceof IdentityHashCodeNode) {
-            hashCodeSnippets.lower((IdentityHashCodeNode) n, tool);
-        } else if (n instanceof ResolveDynamicConstantNode) {
-            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
-                resolveConstantSnippets.lower((ResolveDynamicConstantNode) n, tool);
-            }
-        } else if (n instanceof ResolveConstantNode) {
-            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
-                resolveConstantSnippets.lower((ResolveConstantNode) n, tool);
-            }
-        } else if (n instanceof ResolveMethodAndLoadCountersNode) {
-            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
-                resolveConstantSnippets.lower((ResolveMethodAndLoadCountersNode) n, tool);
-            }
-        } else if (n instanceof InitializeKlassNode) {
-            if (graph.getGuardsStage().areFrameStatesAtDeopts()) {
-                resolveConstantSnippets.lower((InitializeKlassNode) n, tool);
-            }
-        } else if (n instanceof ProfileNode) {
-            profileSnippets.lower((ProfileNode) n, tool);
-        } else {
-            super.lower(n, tool);
         }
     }
 
@@ -635,59 +641,53 @@ public class DefaultHotSpotLoweringProvider extends DefaultJavaLoweringProvider
     }
 
     static final class Exceptions {
-        protected static final ArrayIndexOutOfBoundsException cachedArrayIndexOutOfBoundsException;
+        protected static final EnumMap<BytecodeExceptionKind, RuntimeException> cachedExceptions;
-        protected static final NullPointerException cachedNullPointerException;
 
         static {
-            cachedArrayIndexOutOfBoundsException = new ArrayIndexOutOfBoundsException();
+            cachedExceptions = new EnumMap<>(BytecodeExceptionKind.class);
-            cachedArrayIndexOutOfBoundsException.setStackTrace(new StackTraceElement[0]);
+            cachedExceptions.put(BytecodeExceptionKind.NULL_POINTER, clearStackTrace(new NullPointerException()));
-            cachedNullPointerException = new NullPointerException();
+            cachedExceptions.put(BytecodeExceptionKind.OUT_OF_BOUNDS, clearStackTrace(new ArrayIndexOutOfBoundsException()));
-            cachedNullPointerException.setStackTrace(new StackTraceElement[0]);
+            cachedExceptions.put(BytecodeExceptionKind.CLASS_CAST, clearStackTrace(new ClassCastException()));
+            cachedExceptions.put(BytecodeExceptionKind.ARRAY_STORE, clearStackTrace(new ArrayStoreException()));
+            cachedExceptions.put(BytecodeExceptionKind.DIVISION_BY_ZERO, clearStackTrace(new ArithmeticException()));
+        }
+
+        private static RuntimeException clearStackTrace(RuntimeException ex) {
+            ex.setStackTrace(new StackTraceElement[0]);
+            return ex;
         }
     }
 
     public static final class RuntimeCalls {
-        public static final ForeignCallDescriptor CREATE_ARRAY_STORE_EXCEPTION = new ForeignCallDescriptor("createArrayStoreException", ArrayStoreException.class, Object.class);
+        public static final EnumMap<BytecodeExceptionKind, ForeignCallDescriptor> runtimeCalls;
-        public static final ForeignCallDescriptor CREATE_CLASS_CAST_EXCEPTION = new ForeignCallDescriptor("createClassCastException", ClassCastException.class, Object.class, KlassPointer.class);
+
-        public static final ForeignCallDescriptor CREATE_NULL_POINTER_EXCEPTION = new ForeignCallDescriptor("createNullPointerException", NullPointerException.class);
+        static {
-        public static final ForeignCallDescriptor CREATE_OUT_OF_BOUNDS_EXCEPTION = new ForeignCallDescriptor("createOutOfBoundsException", ArrayIndexOutOfBoundsException.class, int.class);
+            runtimeCalls = new EnumMap<>(BytecodeExceptionKind.class);
+            runtimeCalls.put(BytecodeExceptionKind.ARRAY_STORE, new ForeignCallDescriptor("createArrayStoreException", ArrayStoreException.class, Object.class));
+            runtimeCalls.put(BytecodeExceptionKind.CLASS_CAST, new ForeignCallDescriptor("createClassCastException", ClassCastException.class, Object.class, KlassPointer.class));
+            runtimeCalls.put(BytecodeExceptionKind.NULL_POINTER, new ForeignCallDescriptor("createNullPointerException", NullPointerException.class));
+            runtimeCalls.put(BytecodeExceptionKind.OUT_OF_BOUNDS, new ForeignCallDescriptor("createOutOfBoundsException", ArrayIndexOutOfBoundsException.class, int.class, int.class));
+            runtimeCalls.put(BytecodeExceptionKind.DIVISION_BY_ZERO, new ForeignCallDescriptor("createDivisionByZeroException", ArithmeticException.class));
+        }
     }
 
-    private boolean throwCachedException(BytecodeExceptionNode node) {
+    private void throwCachedException(BytecodeExceptionNode node) {
-        Throwable exception;
+        Throwable exception = Exceptions.cachedExceptions.get(node.getExceptionKind());
-        if (node.getExceptionClass() == NullPointerException.class) {
+        assert exception != null;
-            exception = Exceptions.cachedNullPointerException;
-        } else if (node.getExceptionClass() == ArrayIndexOutOfBoundsException.class) {
-            exception = Exceptions.cachedArrayIndexOutOfBoundsException;
-        } else {
-            return false;
-        }
 
         StructuredGraph graph = node.graph();
         FloatingNode exceptionNode = ConstantNode.forConstant(constantReflection.forObject(exception), metaAccess, graph);
         graph.replaceFixedWithFloating(node, exceptionNode);
-        return true;
     }
 
     private void lowerBytecodeExceptionNode(BytecodeExceptionNode node) {
         if (OmitHotExceptionStacktrace.getValue(node.getOptions())) {
-            if (throwCachedException(node)) {
+            throwCachedException(node);
-                return;
+            return;
-            }
         }
 
-        ForeignCallDescriptor descriptor;
+        ForeignCallDescriptor descriptor = RuntimeCalls.runtimeCalls.get(node.getExceptionKind());
-        if (node.getExceptionClass() == NullPointerException.class) {
+        assert descriptor != null;
-            descriptor = RuntimeCalls.CREATE_NULL_POINTER_EXCEPTION;
-        } else if (node.getExceptionClass() == ArrayIndexOutOfBoundsException.class) {
-            descriptor = RuntimeCalls.CREATE_OUT_OF_BOUNDS_EXCEPTION;
-        } else if (node.getExceptionClass() == ArrayStoreException.class) {
-            descriptor = RuntimeCalls.CREATE_ARRAY_STORE_EXCEPTION;
-        } else if (node.getExceptionClass() == ClassCastException.class) {
-            descriptor = RuntimeCalls.CREATE_CLASS_CAST_EXCEPTION;
-        } else {
-            throw GraalError.shouldNotReachHere();
-        }
 
         StructuredGraph graph = node.graph();
         ForeignCallNode foreignCallNode = graph.add(new ForeignCallNode(foreignCalls, descriptor, node.stamp(NodeView.DEFAULT), node.getArguments()));

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java

@@ -65,10 +65,6 @@ import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Transition.
 import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Transition.STACK_INSPECTABLE_LEAF;
 import static org.graalvm.compiler.hotspot.HotSpotHostBackend.DEOPTIMIZATION_HANDLER;
 import static org.graalvm.compiler.hotspot.HotSpotHostBackend.UNCOMMON_TRAP_HANDLER;
-import static org.graalvm.compiler.hotspot.meta.DefaultHotSpotLoweringProvider.RuntimeCalls.CREATE_ARRAY_STORE_EXCEPTION;
-import static org.graalvm.compiler.hotspot.meta.DefaultHotSpotLoweringProvider.RuntimeCalls.CREATE_CLASS_CAST_EXCEPTION;
-import static org.graalvm.compiler.hotspot.meta.DefaultHotSpotLoweringProvider.RuntimeCalls.CREATE_NULL_POINTER_EXCEPTION;
-import static org.graalvm.compiler.hotspot.meta.DefaultHotSpotLoweringProvider.RuntimeCalls.CREATE_OUT_OF_BOUNDS_EXCEPTION;
 import static org.graalvm.compiler.hotspot.replacements.AssertionSnippets.ASSERTION_VM_MESSAGE_C;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.MARK_WORD_LOCATION;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_END_LOCATION;
@@ -112,6 +108,7 @@ import org.graalvm.compiler.hotspot.HotSpotGraalRuntimeProvider;
 import org.graalvm.compiler.hotspot.stubs.ArrayStoreExceptionStub;
 import org.graalvm.compiler.hotspot.stubs.ClassCastExceptionStub;
 import org.graalvm.compiler.hotspot.stubs.CreateExceptionStub;
+import org.graalvm.compiler.hotspot.stubs.DivisionByZeroExceptionStub;
 import org.graalvm.compiler.hotspot.stubs.ExceptionHandlerStub;
 import org.graalvm.compiler.hotspot.stubs.NewArrayStub;
 import org.graalvm.compiler.hotspot.stubs.NewInstanceStub;
@@ -121,6 +118,7 @@ import org.graalvm.compiler.hotspot.stubs.Stub;
 import org.graalvm.compiler.hotspot.stubs.UnwindExceptionToCallerStub;
 import org.graalvm.compiler.hotspot.stubs.VerifyOopStub;
 import org.graalvm.compiler.nodes.NamedLocationIdentity;
+import org.graalvm.compiler.nodes.extended.BytecodeExceptionNode.BytecodeExceptionKind;
 import org.graalvm.compiler.options.OptionValues;
 import org.graalvm.compiler.word.Word;
 import org.graalvm.compiler.word.WordTypes;
@@ -285,10 +283,13 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall
         link(new ExceptionHandlerStub(options, providers, foreignCalls.get(EXCEPTION_HANDLER)));
         link(new UnwindExceptionToCallerStub(options, providers, registerStubCall(UNWIND_EXCEPTION_TO_CALLER, NOT_REEXECUTABLE, SAFEPOINT, any())));
         link(new VerifyOopStub(options, providers, registerStubCall(VERIFY_OOP, REEXECUTABLE, LEAF_NOFP, NO_LOCATIONS)));
-        link(new ArrayStoreExceptionStub(options, providers, registerStubCall(CREATE_ARRAY_STORE_EXCEPTION, REEXECUTABLE, SAFEPOINT, any())));
+
-        link(new ClassCastExceptionStub(options, providers, registerStubCall(CREATE_CLASS_CAST_EXCEPTION, REEXECUTABLE, SAFEPOINT, any())));
+        EnumMap<BytecodeExceptionKind, ForeignCallDescriptor> exceptionRuntimeCalls = DefaultHotSpotLoweringProvider.RuntimeCalls.runtimeCalls;
-        link(new NullPointerExceptionStub(options, providers, registerStubCall(CREATE_NULL_POINTER_EXCEPTION, REEXECUTABLE, SAFEPOINT, any())));
+        link(new ArrayStoreExceptionStub(options, providers, registerStubCall(exceptionRuntimeCalls.get(BytecodeExceptionKind.ARRAY_STORE), REEXECUTABLE, SAFEPOINT, any())));
-        link(new OutOfBoundsExceptionStub(options, providers, registerStubCall(CREATE_OUT_OF_BOUNDS_EXCEPTION, REEXECUTABLE, SAFEPOINT, any())));
+        link(new ClassCastExceptionStub(options, providers, registerStubCall(exceptionRuntimeCalls.get(BytecodeExceptionKind.CLASS_CAST), REEXECUTABLE, SAFEPOINT, any())));
+        link(new NullPointerExceptionStub(options, providers, registerStubCall(exceptionRuntimeCalls.get(BytecodeExceptionKind.NULL_POINTER), REEXECUTABLE, SAFEPOINT, any())));
+        link(new OutOfBoundsExceptionStub(options, providers, registerStubCall(exceptionRuntimeCalls.get(BytecodeExceptionKind.OUT_OF_BOUNDS), REEXECUTABLE, SAFEPOINT, any())));
+        link(new DivisionByZeroExceptionStub(options, providers, registerStubCall(exceptionRuntimeCalls.get(BytecodeExceptionKind.DIVISION_BY_ZERO), REEXECUTABLE, SAFEPOINT, any())));
 
         linkForeignCall(options, providers, IDENTITY_HASHCODE, c.identityHashCodeAddress, PREPEND_THREAD, SAFEPOINT, NOT_REEXECUTABLE, MARK_WORD_LOCATION);
         linkForeignCall(options, providers, REGISTER_FINALIZER, c.registerFinalizerAddress, PREPEND_THREAD, SAFEPOINT, NOT_REEXECUTABLE, any());

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotNodePlugin.java

@@ -27,6 +27,7 @@ import static org.graalvm.compiler.core.common.GraalOptions.ImmutableCode;
 import org.graalvm.compiler.core.common.type.StampPair;
 import org.graalvm.compiler.nodes.ConstantNode;
 import org.graalvm.compiler.nodes.ValueNode;
+import org.graalvm.compiler.nodes.extended.GuardingNode;
 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext;
 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderTool;
 import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin;
@@ -147,16 +148,16 @@ public final class HotSpotNodePlugin implements NodePlugin, TypePlugin {
     }
 
     @Override
-    public boolean handleLoadIndexed(GraphBuilderContext b, ValueNode array, ValueNode index, JavaKind elementKind) {
+    public boolean handleLoadIndexed(GraphBuilderContext b, ValueNode array, ValueNode index, GuardingNode boundsCheck, JavaKind elementKind) {
-        if (b.parsingIntrinsic() && wordOperationPlugin.handleLoadIndexed(b, array, index, elementKind)) {
+        if (b.parsingIntrinsic() && wordOperationPlugin.handleLoadIndexed(b, array, index, boundsCheck, elementKind)) {
             return true;
         }
         return false;
     }
 
     @Override
-    public boolean handleStoreIndexed(GraphBuilderContext b, ValueNode array, ValueNode index, JavaKind elementKind, ValueNode value) {
+    public boolean handleStoreIndexed(GraphBuilderContext b, ValueNode array, ValueNode index, GuardingNode boundsCheck, GuardingNode storeCheck, JavaKind elementKind, ValueNode value) {
-        if (b.parsingIntrinsic() && wordOperationPlugin.handleStoreIndexed(b, array, index, elementKind, value)) {
+        if (b.parsingIntrinsic() && wordOperationPlugin.handleStoreIndexed(b, array, index, boundsCheck, storeCheck, elementKind, value)) {
             return true;
         }
         return false;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotWordOperationPlugin.java

@@ -45,6 +45,7 @@ import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.calc.ConditionalNode;
 import org.graalvm.compiler.nodes.calc.IsNullNode;
 import org.graalvm.compiler.nodes.calc.PointerEqualsNode;
+import org.graalvm.compiler.nodes.extended.GuardingNode;
 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext;
 import org.graalvm.compiler.nodes.java.LoadIndexedNode;
 import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType;
@@ -69,13 +70,13 @@ class HotSpotWordOperationPlugin extends WordOperationPlugin {
     }
 
     @Override
-    protected LoadIndexedNode createLoadIndexedNode(ValueNode array, ValueNode index) {
+    protected LoadIndexedNode createLoadIndexedNode(ValueNode array, ValueNode index, GuardingNode boundsCheck) {
         ResolvedJavaType arrayType = StampTool.typeOrNull(array);
         Stamp componentStamp = wordTypes.getWordStamp(arrayType.getComponentType());
         if (componentStamp instanceof MetaspacePointerStamp) {
-            return new LoadIndexedPointerNode(componentStamp, array, index);
+            return new LoadIndexedPointerNode(componentStamp, array, index, boundsCheck);
         } else {
-            return super.createLoadIndexedNode(array, index);
+            return super.createLoadIndexedNode(array, index, boundsCheck);
         }
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/LoadIndexedPointerNode.java

@@ -26,6 +26,7 @@ import org.graalvm.compiler.core.common.type.Stamp;
 import org.graalvm.compiler.graph.NodeClass;
 import org.graalvm.compiler.nodeinfo.NodeInfo;
 import org.graalvm.compiler.nodes.ValueNode;
+import org.graalvm.compiler.nodes.extended.GuardingNode;
 import org.graalvm.compiler.nodes.java.LoadIndexedNode;
 
 import jdk.vm.ci.meta.JavaKind;
@@ -35,8 +36,8 @@ public final class LoadIndexedPointerNode extends LoadIndexedNode {
 
     public static final NodeClass<LoadIndexedPointerNode> TYPE = NodeClass.create(LoadIndexedPointerNode.class);
 
-    public LoadIndexedPointerNode(Stamp stamp, ValueNode array, ValueNode index) {
+    public LoadIndexedPointerNode(Stamp stamp, ValueNode array, ValueNode index, GuardingNode boundsCheck) {
-        super(TYPE, stamp, array, index, JavaKind.Illegal);
+        super(TYPE, stamp, array, index, boundsCheck, JavaKind.Illegal);
     }
 
     @Override

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/WriteBarrierAdditionPhase.java

@@ -121,6 +121,8 @@ public class WriteBarrierAdditionPhase extends Phase {
                 boolean precise = barrierType == BarrierType.PRECISE;
                 if (config.useG1GC) {
                     if (!node.getLocationIdentity().isInit()) {
+                        // The pre barrier does nothing if the value being read is null, so it can
+                        // be explicitly skipped when this is an initializing store.
                         addG1PreWriteBarrier(node, node.getAddress(), null, true, node.getNullCheck(), graph);
                     }
                     addG1PostWriteBarrier(node, node.getAddress(), node.value(), precise, graph);
@@ -178,6 +180,8 @@ public class WriteBarrierAdditionPhase extends Phase {
     private void addArrayRangeBarriers(ArrayRangeWrite write, StructuredGraph graph) {
         if (config.useG1GC) {
             if (!write.isInitialization()) {
+                // The pre barrier does nothing if the value being read is null, so it can
+                // be explicitly skipped when this is an initializing store.
                 G1ArrayRangePreWriteBarrier g1ArrayRangePreWriteBarrier = graph.add(new G1ArrayRangePreWriteBarrier(write.getAddress(), write.getLength(), write.getElementStride()));
                 graph.addBeforeFixed(write.asNode(), g1ArrayRangePreWriteBarrier);
             }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/ObjectCloneNode.java

@@ -40,10 +40,8 @@ import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.java.LoadFieldNode;
 import org.graalvm.compiler.nodes.java.NewInstanceNode;
 import org.graalvm.compiler.nodes.java.StoreFieldNode;
-import org.graalvm.compiler.nodes.spi.ArrayLengthProvider;
 import org.graalvm.compiler.nodes.spi.LoweringTool;
 import org.graalvm.compiler.nodes.spi.Replacements;
-import org.graalvm.compiler.nodes.spi.VirtualizableAllocation;
 import org.graalvm.compiler.nodes.type.StampTool;
 import org.graalvm.compiler.replacements.nodes.BasicObjectCloneNode;
 
@@ -53,7 +51,7 @@ import jdk.vm.ci.meta.ResolvedJavaMethod;
 import jdk.vm.ci.meta.ResolvedJavaType;
 
 @NodeInfo
-public final class ObjectCloneNode extends BasicObjectCloneNode implements VirtualizableAllocation, ArrayLengthProvider {
+public final class ObjectCloneNode extends BasicObjectCloneNode {
 
     public static final NodeClass<ObjectCloneNode> TYPE = NodeClass.create(ObjectCloneNode.class);
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/WriteBarrierSnippets.java

@@ -51,6 +51,7 @@ import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
 import org.graalvm.compiler.debug.DebugHandlersFactory;
 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
+import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
 import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePostWriteBarrier;
@@ -65,6 +66,7 @@ import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier;
 import org.graalvm.compiler.hotspot.nodes.VMErrorNode;
 import org.graalvm.compiler.nodes.NamedLocationIdentity;
 import org.graalvm.compiler.nodes.NodeView;
+import org.graalvm.compiler.nodes.PiNode;
 import org.graalvm.compiler.nodes.StructuredGraph;
 import org.graalvm.compiler.nodes.ValueNode;
 import org.graalvm.compiler.nodes.extended.FixedValueAnchorNode;
@@ -79,12 +81,14 @@ import org.graalvm.compiler.nodes.spi.LoweringTool;
 import org.graalvm.compiler.nodes.type.NarrowOopStamp;
 import org.graalvm.compiler.options.OptionValues;
 import org.graalvm.compiler.replacements.Log;
+import org.graalvm.compiler.replacements.ReplacementsUtil;
 import org.graalvm.compiler.replacements.SnippetCounter;
 import org.graalvm.compiler.replacements.SnippetCounter.Group;
 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
 import org.graalvm.compiler.replacements.Snippets;
+import org.graalvm.compiler.replacements.nodes.AssertionNode;
 import org.graalvm.compiler.replacements.nodes.DirectStoreNode;
 import org.graalvm.compiler.word.Word;
 import jdk.internal.vm.compiler.word.LocationIdentity;
@@ -140,7 +144,10 @@ public class WriteBarrierSnippets implements Snippets {
     }
 
     @Snippet
-    public static void serialImpreciseWriteBarrier(Object object, @ConstantParameter Counters counters) {
+    public static void serialImpreciseWriteBarrier(Object object, @ConstantParameter boolean verifyBarrier, @ConstantParameter Counters counters) {
+        if (verifyBarrier) {
+            verifyNotArray(object);
+        }
         serialWriteBarrier(Word.objectToTrackedPointer(object), counters);
     }
 
@@ -221,8 +228,8 @@ public class WriteBarrierSnippets implements Snippets {
     }
 
     @Snippet
-    public static void g1PostWriteBarrier(Address address, Object object, Object value, @ConstantParameter boolean usePrecise, @ConstantParameter Register threadRegister,
+    public static void g1PostWriteBarrier(Address address, Object object, Object value, @ConstantParameter boolean usePrecise, @ConstantParameter boolean verifyBarrier,
-                    @ConstantParameter boolean trace, @ConstantParameter Counters counters) {
+                    @ConstantParameter Register threadRegister, @ConstantParameter boolean trace, @ConstantParameter Counters counters) {
         Word thread = registerAsWord(threadRegister);
         Object fixedValue = FixedValueAnchorNode.getObject(value);
         verifyOop(object);
@@ -232,6 +239,9 @@ public class WriteBarrierSnippets implements Snippets {
         if (usePrecise) {
             oop = Word.fromAddress(address);
         } else {
+            if (verifyBarrier) {
+                verifyNotArray(object);
+            }
             oop = Word.objectToTrackedPointer(object);
         }
         int gcCycle = 0;
@@ -298,6 +308,13 @@ public class WriteBarrierSnippets implements Snippets {
         }
     }
 
+    private static void verifyNotArray(Object object) {
+        if (object != null) {
+            // Manually build the null check and cast because we're in snippet that's lowered late.
+            AssertionNode.assertion(false, !PiNode.piCastNonNull(object, Object.class).getClass().isArray(), "imprecise card mark used with array");
+        }
+    }
+
     @Snippet
     public static void g1ArrayRangePreWriteBarrier(Address address, int length, @ConstantParameter int elementStride, @ConstantParameter Register threadRegister) {
         Word thread = registerAsWord(threadRegister);
@@ -415,11 +432,13 @@ public class WriteBarrierSnippets implements Snippets {
 
         private final CompressEncoding oopEncoding;
         private final Counters counters;
+        private final boolean verifyBarrier;
 
-        public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target,
+        public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, Group.Factory factory, HotSpotProviders providers, TargetDescription target,
-                        CompressEncoding oopEncoding) {
+                        GraalHotSpotVMConfig config) {
             super(options, factories, providers, providers.getSnippetReflection(), target);
-            this.oopEncoding = oopEncoding;
+            this.oopEncoding = config.useCompressedOops ? config.getOopEncoding() : null;
+            this.verifyBarrier = ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED || config.verifyBeforeGC || config.verifyAfterGC;
             this.counters = new Counters(factory);
         }
 
@@ -432,6 +451,7 @@ public class WriteBarrierSnippets implements Snippets {
                 args = new Arguments(serialImpreciseWriteBarrier, writeBarrier.graph().getGuardsStage(), tool.getLoweringStage());
                 OffsetAddressNode address = (OffsetAddressNode) writeBarrier.getAddress();
                 args.add("object", address.getBase());
+                args.addConst("verifyBarrier", verifyBarrier);
             }
             args.addConst("counters", counters);
             template(writeBarrier, args).instantiate(providers.getMetaAccess(), writeBarrier, DEFAULT_REPLACER, args);
@@ -519,6 +539,7 @@ public class WriteBarrierSnippets implements Snippets {
             args.add("value", value);
 
             args.addConst("usePrecise", writeBarrierPost.usePrecise());
+            args.addConst("verifyBarrier", verifyBarrier);
             args.addConst("threadRegister", registers.getThreadRegister());
             args.addConst("trace", traceBarrier(writeBarrierPost.graph()));
             args.addConst("counters", counters);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/DivisionByZeroExceptionStub.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2018, 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.hotspot.stubs;
+
+import org.graalvm.compiler.api.replacements.Snippet;
+import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
+import org.graalvm.compiler.debug.GraalError;
+import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage;
+import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
+import org.graalvm.compiler.options.OptionValues;
+import org.graalvm.compiler.replacements.nodes.CStringConstant;
+import org.graalvm.compiler.word.Word;
+
+import jdk.vm.ci.code.Register;
+
+/**
+ * Stub to allocate an {@link ArithmeticException} thrown by a bytecode for a division by zero.
+ */
+public class DivisionByZeroExceptionStub extends CreateExceptionStub {
+    public DivisionByZeroExceptionStub(OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage) {
+        super("createDivisionByZeroException", options, providers, linkage);
+    }
+
+    @Override
+    protected Object getConstantParameterValue(int index, String name) {
+        GraalError.guarantee(index == 0, "unknown parameter %s at index %d", name, index);
+        return providers.getRegisters().getThreadRegister();
+    }
+
+    @Snippet
+    private static Object createDivisionByZeroException(@ConstantParameter Register threadRegister) {
+        Word msg = CStringConstant.cstring("/ by zero");
+        return createException(threadRegister, ArithmeticException.class, msg);
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/OutOfBoundsExceptionStub.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,6 +22,9 @@
  */
 package org.graalvm.compiler.hotspot.stubs;
 
+import static org.graalvm.compiler.hotspot.stubs.StubUtil.printNumber;
+import static org.graalvm.compiler.hotspot.stubs.StubUtil.printString;
+
 import org.graalvm.compiler.api.replacements.Snippet;
 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
 import org.graalvm.compiler.debug.GraalError;
@@ -29,6 +32,7 @@ import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage;
 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
 import org.graalvm.compiler.hotspot.nodes.AllocaNode;
 import org.graalvm.compiler.options.OptionValues;
+import org.graalvm.compiler.serviceprovider.GraalServices;
 import org.graalvm.compiler.word.Word;
 
 import jdk.vm.ci.code.Register;
@@ -37,51 +41,52 @@ import jdk.vm.ci.code.Register;
  * Stub to allocate an {@link ArrayIndexOutOfBoundsException} thrown by a bytecode.
  */
 public class OutOfBoundsExceptionStub extends CreateExceptionStub {
-
     public OutOfBoundsExceptionStub(OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage) {
         super("createOutOfBoundsException", options, providers, linkage);
     }
 
+    // JDK-8201593: Print array length in ArrayIndexOutOfBoundsException.
+    private static final boolean PRINT_LENGTH_IN_EXCEPTION = GraalServices.JAVA_SPECIFICATION_VERSION >= 11;
     private static final int MAX_INT_STRING_SIZE = Integer.toString(Integer.MIN_VALUE).length();
+    private static final String STR_INDEX = "Index ";
+    private static final String STR_OUTOFBOUNDSFORLENGTH = " out of bounds for length ";
 
     @Override
     protected Object getConstantParameterValue(int index, String name) {
         switch (index) {
-            case 1:
-                return providers.getRegisters().getThreadRegister();
             case 2:
+                return providers.getRegisters().getThreadRegister();
+            case 3:
                 int wordSize = providers.getWordTypes().getWordKind().getByteCount();
-                // (MAX_INT_STRING_SIZE + 1) / wordSize, rounded up
+                int bytes;
-                return MAX_INT_STRING_SIZE / wordSize + 1;
+                if (PRINT_LENGTH_IN_EXCEPTION) {
+                    bytes = STR_INDEX.length() + STR_OUTOFBOUNDSFORLENGTH.length() + 2 * MAX_INT_STRING_SIZE;
+                } else {
+                    bytes = MAX_INT_STRING_SIZE;
+                }
+                // (required words for maximum length + nullbyte), rounded up
+                return (bytes + 1) / wordSize + 1;
+            case 4:
+                return PRINT_LENGTH_IN_EXCEPTION;
             default:
                 throw GraalError.shouldNotReachHere("unknown parameter " + name + " at index " + index);
         }
     }
 
     @Snippet
-    private static Object createOutOfBoundsException(int idx, @ConstantParameter Register threadRegister, @ConstantParameter int bufferSizeInWords) {
+    private static Object createOutOfBoundsException(int idx, int length, @ConstantParameter Register threadRegister, @ConstantParameter int bufferSizeInWords,
+                    @ConstantParameter boolean printLengthInException) {
         Word buffer = AllocaNode.alloca(bufferSizeInWords);
-
+        Word ptr;
-        long number = idx;
+        if (printLengthInException) {
-        if (number < 0) {
+            ptr = printString(buffer, STR_INDEX);
-            number = -number;
+            ptr = printNumber(ptr, idx);
+            ptr = printString(ptr, STR_OUTOFBOUNDSFORLENGTH);
+            ptr = printNumber(ptr, length);
+        } else {
+            ptr = printNumber(buffer, idx);
         }
-
-        Word ptr = buffer.add(MAX_INT_STRING_SIZE);
         ptr.writeByte(0, (byte) 0);
-        do {
+        return createException(threadRegister, ArrayIndexOutOfBoundsException.class, buffer);
-            long digit = number % 10;
-            number /= 10;
-
-            ptr = ptr.subtract(1);
-            ptr.writeByte(0, (byte) ('0' + digit));
-        } while (number > 0);
-
-        if (idx < 0) {
-            ptr = ptr.subtract(1);
-            ptr.writeByte(0, (byte) '-');
-        }
-
-        return createException(threadRegister, ArrayIndexOutOfBoundsException.class, ptr);
     }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/StubUtil.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,7 +23,7 @@
 package org.graalvm.compiler.hotspot.stubs;
 
 import static jdk.vm.ci.meta.DeoptimizationReason.RuntimeConstraint;
-import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
+import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.clearPendingException;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.getAndClearObjectResult;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHubIntrinsic;
@@ -272,4 +272,59 @@ public class StubUtil {
     static int hubOffset(@InjectedParameter GraalHotSpotVMConfig config) {
         return config.hubOffset;
     }
+
+    /**
+     * Print {@code number} as decimal string to {@code buffer}.
+     *
+     * @param buffer
+     * @param number
+     * @return A pointer pointing one byte right after the last printed digit in {@code buffer}.
+     */
+    public static Word printNumber(Word buffer, long number) {
+        long tmpNumber = number;
+        int offset;
+        if (tmpNumber <= 0) {
+            tmpNumber = -tmpNumber;
+            offset = 1;
+        } else {
+            offset = 0;
+        }
+        while (tmpNumber > 0) {
+            tmpNumber /= 10;
+            offset++;
+        }
+        tmpNumber = number < 0 ? -number : number;
+        Word ptr = buffer.add(offset);
+        do {
+            long digit = tmpNumber % 10;
+            tmpNumber /= 10;
+            ptr = ptr.subtract(1);
+            ptr.writeByte(0, (byte) ('0' + digit));
+        } while (tmpNumber > 0);
+
+        if (number < 0) {
+            ptr = ptr.subtract(1);
+            ptr.writeByte(0, (byte) '-');
+        }
+        return buffer.add(offset);
+    }
+
+    /**
+     * Copy {@code javaString} bytes to the memory location {@code ptr}.
+     *
+     * @param buffer
+     * @param javaString
+     * @return A pointer pointing one byte right after the last byte copied from {@code javaString}
+     *         to {@code ptr}
+     */
+    public static Word printString(Word buffer, String javaString) {
+        Word string = cstring(javaString);
+        int i = 0;
+        byte b;
+        while ((b = string.readByte(i)) != 0) {
+            buffer.writeByte(i, b);
+            i++;
+        }
+        return buffer.add(i);
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BciBlockMapping.java

@@ -31,6 +31,7 @@ import static org.graalvm.compiler.bytecode.Bytecodes.BALOAD;
 import static org.graalvm.compiler.bytecode.Bytecodes.BASTORE;
 import static org.graalvm.compiler.bytecode.Bytecodes.CALOAD;
 import static org.graalvm.compiler.bytecode.Bytecodes.CASTORE;
+import static org.graalvm.compiler.bytecode.Bytecodes.CHECKCAST;
 import static org.graalvm.compiler.bytecode.Bytecodes.DALOAD;
 import static org.graalvm.compiler.bytecode.Bytecodes.DASTORE;
 import static org.graalvm.compiler.bytecode.Bytecodes.DRETURN;
@@ -43,6 +44,7 @@ import static org.graalvm.compiler.bytecode.Bytecodes.GOTO;
 import static org.graalvm.compiler.bytecode.Bytecodes.GOTO_W;
 import static org.graalvm.compiler.bytecode.Bytecodes.IALOAD;
 import static org.graalvm.compiler.bytecode.Bytecodes.IASTORE;
+import static org.graalvm.compiler.bytecode.Bytecodes.IDIV;
 import static org.graalvm.compiler.bytecode.Bytecodes.IFEQ;
 import static org.graalvm.compiler.bytecode.Bytecodes.IFGE;
 import static org.graalvm.compiler.bytecode.Bytecodes.IFGT;
@@ -64,12 +66,15 @@ import static org.graalvm.compiler.bytecode.Bytecodes.INVOKEINTERFACE;
 import static org.graalvm.compiler.bytecode.Bytecodes.INVOKESPECIAL;
 import static org.graalvm.compiler.bytecode.Bytecodes.INVOKESTATIC;
 import static org.graalvm.compiler.bytecode.Bytecodes.INVOKEVIRTUAL;
+import static org.graalvm.compiler.bytecode.Bytecodes.IREM;
 import static org.graalvm.compiler.bytecode.Bytecodes.IRETURN;
 import static org.graalvm.compiler.bytecode.Bytecodes.JSR;
 import static org.graalvm.compiler.bytecode.Bytecodes.JSR_W;
 import static org.graalvm.compiler.bytecode.Bytecodes.LALOAD;
 import static org.graalvm.compiler.bytecode.Bytecodes.LASTORE;
+import static org.graalvm.compiler.bytecode.Bytecodes.LDIV;
 import static org.graalvm.compiler.bytecode.Bytecodes.LOOKUPSWITCH;
+import static org.graalvm.compiler.bytecode.Bytecodes.LREM;
 import static org.graalvm.compiler.bytecode.Bytecodes.LRETURN;
 import static org.graalvm.compiler.bytecode.Bytecodes.PUTFIELD;
 import static org.graalvm.compiler.bytecode.Bytecodes.PUTSTATIC;
@@ -650,6 +655,10 @@ public final class BciBlockMapping {
                     }
                     break;
                 }
+                case IDIV:
+                case IREM:
+                case LDIV:
+                case LREM:
                 case IASTORE:
                 case LASTORE:
                 case FASTORE:
@@ -667,6 +676,7 @@ public final class BciBlockMapping {
                 case CALOAD:
                 case SALOAD:
                 case ARRAYLENGTH:
+                case CHECKCAST:
                 case PUTSTATIC:
                 case GETSTATIC:
                 case PUTFIELD:

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BytecodeParser.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2009, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2009, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -377,7 +377,10 @@ import org.graalvm.compiler.nodes.calc.ZeroExtendNode;
 import org.graalvm.compiler.nodes.extended.AnchoringNode;
 import org.graalvm.compiler.nodes.extended.BranchProbabilityNode;
 import org.graalvm.compiler.nodes.extended.BytecodeExceptionNode;
+import org.graalvm.compiler.nodes.extended.BytecodeExceptionNode.BytecodeExceptionKind;
+import org.graalvm.compiler.nodes.extended.GuardingNode;
 import org.graalvm.compiler.nodes.extended.IntegerSwitchNode;
+import org.graalvm.compiler.nodes.extended.LoadArrayComponentHubNode;
 import org.graalvm.compiler.nodes.extended.LoadHubNode;
 import org.graalvm.compiler.nodes.extended.LoadMethodNode;
 import org.graalvm.compiler.nodes.extended.MembarNode;
@@ -398,6 +401,7 @@ import org.graalvm.compiler.nodes.graphbuilderconf.ProfilingPlugin;
 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
 import org.graalvm.compiler.nodes.java.ExceptionObjectNode;
 import org.graalvm.compiler.nodes.java.FinalFieldBarrierNode;
+import org.graalvm.compiler.nodes.java.InstanceOfDynamicNode;
 import org.graalvm.compiler.nodes.java.InstanceOfNode;
 import org.graalvm.compiler.nodes.java.LoadFieldNode;
 import org.graalvm.compiler.nodes.java.LoadIndexedNode;
@@ -423,6 +427,7 @@ import jdk.vm.ci.code.BailoutException;
 import jdk.vm.ci.code.BytecodeFrame;
 import jdk.vm.ci.code.CodeUtil;
 import jdk.vm.ci.code.site.InfopointReason;
+import jdk.vm.ci.meta.Constant;
 import jdk.vm.ci.meta.ConstantPool;
 import jdk.vm.ci.meta.ConstantReflectionProvider;
 import jdk.vm.ci.meta.DeoptimizationAction;
@@ -1133,12 +1138,12 @@ public class BytecodeParser implements GraphBuilderContext {
         finishedDispatch.setNext(target);
     }
 
-    protected ValueNode genLoadIndexed(ValueNode array, ValueNode index, JavaKind kind) {
+    protected ValueNode genLoadIndexed(ValueNode array, ValueNode index, GuardingNode boundsCheck, JavaKind kind) {
-        return LoadIndexedNode.create(graph.getAssumptions(), array, index, kind, metaAccess, constantReflection);
+        return LoadIndexedNode.create(graph.getAssumptions(), array, index, boundsCheck, kind, metaAccess, constantReflection);
     }
 
-    protected void genStoreIndexed(ValueNode array, ValueNode index, JavaKind kind, ValueNode value) {
+    protected void genStoreIndexed(ValueNode array, ValueNode index, GuardingNode boundsCheck, GuardingNode storeCheck, JavaKind kind, ValueNode value) {
-        add(new StoreIndexedNode(array, index, kind, value));
+        add(new StoreIndexedNode(array, index, boundsCheck, storeCheck, kind, value));
     }
 
     protected ValueNode genIntegerAdd(ValueNode x, ValueNode y) {
@@ -1173,12 +1178,12 @@ public class BytecodeParser implements GraphBuilderContext {
         return RemNode.create(x, y, NodeView.DEFAULT);
     }
 
-    protected ValueNode genIntegerDiv(ValueNode x, ValueNode y) {
+    protected ValueNode genIntegerDiv(ValueNode x, ValueNode y, GuardingNode zeroCheck) {
-        return SignedDivNode.create(x, y, NodeView.DEFAULT);
+        return SignedDivNode.create(x, y, zeroCheck, NodeView.DEFAULT);
     }
 
-    protected ValueNode genIntegerRem(ValueNode x, ValueNode y) {
+    protected ValueNode genIntegerRem(ValueNode x, ValueNode y, GuardingNode zeroCheck) {
-        return SignedRemNode.create(x, y, NodeView.DEFAULT);
+        return SignedRemNode.create(x, y, zeroCheck, NodeView.DEFAULT);
     }
 
     protected ValueNode genNegateOp(ValueNode x) {
@@ -1267,10 +1272,12 @@ public class BytecodeParser implements GraphBuilderContext {
     protected void genThrow() {
         genInfoPointNode(InfopointReason.BYTECODE_POSITION, null);
 
-        ValueNode exception = frameState.pop(JavaKind.Object);
+        ValueNode exception = maybeEmitExplicitNullCheck(frameState.pop(JavaKind.Object));
-        FixedGuardNode nullCheck = append(new FixedGuardNode(graph.addOrUniqueWithInputs(IsNullNode.create(exception)), NullCheckException, InvalidateReprofile, true));
+        if (!StampTool.isPointerNonNull(exception.stamp(NodeView.DEFAULT))) {
-        ValueNode nonNullException = graph.maybeAddOrUnique(PiNode.create(exception, exception.stamp(NodeView.DEFAULT).join(objectNonNull()), nullCheck));
+            FixedGuardNode nullCheck = append(new FixedGuardNode(graph.addOrUniqueWithInputs(IsNullNode.create(exception)), NullCheckException, InvalidateReprofile, true));
-        lastInstr.setNext(handleException(nonNullException, bci(), false));
+            exception = graph.maybeAddOrUnique(PiNode.create(exception, exception.stamp(NodeView.DEFAULT).join(objectNonNull()), nullCheck));
+        }
+        lastInstr.setNext(handleException(exception, bci(), false));
     }
 
     protected LogicNode createInstanceOf(TypeReference type, ValueNode object) {
@@ -1324,30 +1331,61 @@ public class BytecodeParser implements GraphBuilderContext {
         return new StateSplitProxyNode(fieldRead);
     }
 
-    protected ValueNode emitExplicitNullCheck(ValueNode receiver) {
+    protected ValueNode maybeEmitExplicitNullCheck(ValueNode receiver) {
-        if (StampTool.isPointerNonNull(receiver.stamp(NodeView.DEFAULT))) {
+        if (StampTool.isPointerNonNull(receiver.stamp(NodeView.DEFAULT)) || !needsExplicitNullCheckException(receiver)) {
             return receiver;
         }
-        BytecodeExceptionNode exception = graph.add(new BytecodeExceptionNode(metaAccess, NullPointerException.class));
+        LogicNode condition = genUnique(IsNullNode.create(receiver));
-        AbstractBeginNode falseSucc = graph.add(new BeginNode());
+        AbstractBeginNode passingSuccessor = emitBytecodeExceptionCheck(condition, false, BytecodeExceptionKind.NULL_POINTER);
-        ValueNode nonNullReceiver = graph.addOrUniqueWithInputs(PiNode.create(receiver, objectNonNull(), falseSucc));
+        return genUnique(PiNode.create(receiver, objectNonNull(), passingSuccessor));
-        append(new IfNode(graph.addOrUniqueWithInputs(IsNullNode.create(receiver)), exception, falseSucc, SLOW_PATH_PROBABILITY));
+    }
-        lastInstr = falseSucc;
+
+    protected GuardingNode maybeEmitExplicitBoundsCheck(ValueNode receiver, ValueNode index) {
+        if (!needsExplicitBoundsCheckException(receiver, index)) {
+            return null;
+        }
+        ValueNode length = append(genArrayLength(receiver));
+        LogicNode condition = genUnique(IntegerBelowNode.create(constantReflection, metaAccess, options, null, index, length, NodeView.DEFAULT));
+        return emitBytecodeExceptionCheck(condition, true, BytecodeExceptionKind.OUT_OF_BOUNDS, index, length);
+    }
+
+    protected GuardingNode maybeEmitExplicitStoreCheck(ValueNode array, JavaKind elementKind, ValueNode value) {
+        if (elementKind != JavaKind.Object || StampTool.isPointerAlwaysNull(value) || !needsExplicitStoreCheckException(array, value)) {
+            return null;
+        }
+        ValueNode arrayClass = genUnique(LoadHubNode.create(array, stampProvider, metaAccess, constantReflection));
+        ValueNode componentHub = append(LoadArrayComponentHubNode.create(arrayClass, stampProvider, metaAccess, constantReflection));
+        LogicNode condition = genUnique(InstanceOfDynamicNode.create(graph.getAssumptions(), getConstantReflection(), componentHub, value, true));
+        return emitBytecodeExceptionCheck(condition, true, BytecodeExceptionKind.ARRAY_STORE, value);
+    }
+
+    protected GuardingNode maybeEmitExplicitDivisionByZeroCheck(ValueNode y) {
+        if (!((IntegerStamp) y.stamp(NodeView.DEFAULT)).contains(0) || !needsExplicitDivisionByZeroException(y)) {
+            return null;
+        }
+        ConstantNode zero = ConstantNode.defaultForKind(y.getStackKind(), graph);
+        LogicNode condition = genUnique(IntegerEqualsNode.create(constantReflection, metaAccess, options, null, y, zero, NodeView.DEFAULT));
+        return emitBytecodeExceptionCheck(condition, false, BytecodeExceptionKind.DIVISION_BY_ZERO);
+    }
+
+    private AbstractBeginNode emitBytecodeExceptionCheck(LogicNode condition, boolean passingOnTrue, BytecodeExceptionKind exceptionKind, ValueNode... arguments) {
+        if (passingOnTrue ? condition.isTautology() : condition.isContradiction()) {
+            return null;
+        }
+
+        BytecodeExceptionNode exception = graph.add(new BytecodeExceptionNode(metaAccess, exceptionKind, arguments));
+        AbstractBeginNode passingSuccessor = graph.add(new BeginNode());
+
+        FixedNode trueSuccessor = passingOnTrue ? passingSuccessor : exception;
+        FixedNode falseSuccessor = passingOnTrue ? exception : passingSuccessor;
+        append(new IfNode(condition, trueSuccessor, falseSuccessor, SLOW_PATH_PROBABILITY));
+        lastInstr = passingSuccessor;
 
         exception.setStateAfter(createFrameState(bci(), exception));
         exception.setNext(handleException(exception, bci(), false));
         EXPLICIT_EXCEPTIONS.increment(debug);
-        return nonNullReceiver;
-    }
 
-    protected void emitExplicitBoundsCheck(ValueNode index, ValueNode length) {
+        return passingSuccessor;
-        AbstractBeginNode trueSucc = graph.add(new BeginNode());
-        BytecodeExceptionNode exception = graph.add(new BytecodeExceptionNode(metaAccess, ArrayIndexOutOfBoundsException.class, index));
-        append(new IfNode(genUnique(IntegerBelowNode.create(constantReflection, metaAccess, options, null, index, length, NodeView.DEFAULT)), trueSucc, exception, FAST_PATH_PROBABILITY));
-        lastInstr = trueSucc;
-
-        exception.setStateAfter(createFrameState(bci(), exception));
-        exception.setNext(handleException(exception, bci(), false));
     }
 
     protected ValueNode genArrayLength(ValueNode x) {
@@ -1617,7 +1655,7 @@ public class BytecodeParser implements GraphBuilderContext {
             returnType = returnType.resolve(targetMethod.getDeclaringClass());
         }
         if (invokeKind.hasReceiver()) {
-            args[0] = emitExplicitExceptions(args[0]);
+            args[0] = maybeEmitExplicitNullCheck(args[0]);
         }
 
         if (initialInvokeKind == InvokeKind.Special && !targetMethod.isConstructor()) {
@@ -1910,8 +1948,8 @@ public class BytecodeParser implements GraphBuilderContext {
 
     /**
      * Weaves a test of the receiver type to ensure the dispatch will select {@code targetMethod}
-     * and not another method that overrides it. This should only be called if there is an intrinsic
+     * and not another method that overrides it. This should only be called if there is an
-     * (i.e., an {@link InvocationPlugin}) for {@code targetMethod} and the invocation is indirect.
+     * {@link InvocationPlugin} for {@code targetMethod} and the invocation is indirect.
      *
      * The control flow woven around the intrinsic is as follows:
      *
@@ -2066,9 +2104,7 @@ public class BytecodeParser implements GraphBuilderContext {
         if (plugin != null) {
 
             if (intrinsicContext != null && intrinsicContext.isCallToOriginal(targetMethod)) {
-                // Self recursive intrinsic means the original
+                // Self recursive intrinsic means the original method should be called.
-                // method should be called.
-                assert !targetMethod.hasBytecodes() : "TODO: when does this happen?";
                 return false;
             }
 
@@ -2088,7 +2124,7 @@ public class BytecodeParser implements GraphBuilderContext {
             try (DebugCloseable context = openNodeContext(targetMethod)) {
                 if (plugin.execute(this, targetMethod, pluginReceiver, args)) {
                     afterInvocationPluginExecution(true, assertions, intrinsicGuard, invokeKind, args, targetMethod, resultType, returnType);
-                    return true;
+                    return !plugin.isDecorator();
                 } else {
                     afterInvocationPluginExecution(false, assertions, intrinsicGuard, invokeKind, args, targetMethod, resultType, returnType);
                 }
@@ -2911,75 +2947,87 @@ public class BytecodeParser implements GraphBuilderContext {
         }
     }
 
+    @SuppressWarnings("try")
     private void createUnwind() {
         assert frameState.stackSize() == 1 : frameState;
         synchronizedEpilogue(BytecodeFrame.AFTER_EXCEPTION_BCI, null, null);
-        ValueNode exception = frameState.pop(JavaKind.Object);
+        try (DebugCloseable context = openNodeContext(frameState, BytecodeFrame.UNWIND_BCI)) {
-        append(new UnwindNode(exception));
+            ValueNode exception = frameState.pop(JavaKind.Object);
+            append(new UnwindNode(exception));
+        }
     }
 
+    @SuppressWarnings("try")
     private void synchronizedEpilogue(int bci, ValueNode currentReturnValue, JavaKind currentReturnValueKind) {
-        if (method.isSynchronized()) {
+        try (DebugCloseable context = openNodeContext(frameState, bci)) {
-            if (currentReturnValue != null) {
+            if (method.isSynchronized()) {
-                frameState.push(currentReturnValueKind, currentReturnValue);
+                if (currentReturnValue != null) {
+                    frameState.push(currentReturnValueKind, currentReturnValue);
+                }
+                genMonitorExit(methodSynchronizedObject, currentReturnValue, bci);
+                assert !frameState.rethrowException();
+                finishPrepare(lastInstr, bci);
+            }
+            if (frameState.lockDepth(false) != 0) {
+                throw bailout("unbalanced monitors: too few exits exiting frame");
             }
-            genMonitorExit(methodSynchronizedObject, currentReturnValue, bci);
-            assert !frameState.rethrowException();
-            finishPrepare(lastInstr, bci);
-        }
-        if (frameState.lockDepth(false) != 0) {
-            throw bailout("unbalanced monitors: too few exits exiting frame");
         }
     }
 
+    @SuppressWarnings("try")
     private void createExceptionDispatch(ExceptionDispatchBlock block) {
-        lastInstr = finishInstruction(lastInstr, frameState);
+        try (DebugCloseable context = openNodeContext(frameState, BytecodeFrame.AFTER_EXCEPTION_BCI)) {
+            lastInstr = finishInstruction(lastInstr, frameState);
 
-        assert frameState.stackSize() == 1 : frameState;
+            assert frameState.stackSize() == 1 : frameState;
-        if (block.handler.isCatchAll()) {
+            if (block.handler.isCatchAll()) {
-            assert block.getSuccessorCount() == 1;
+                assert block.getSuccessorCount() == 1;
-            appendGoto(block.getSuccessor(0));
+                appendGoto(block.getSuccessor(0));
-            return;
+                return;
-        }
+            }
 
-        JavaType catchType = block.handler.getCatchType();
+            JavaType catchType = block.handler.getCatchType();
-        if (graphBuilderConfig.eagerResolving()) {
+            if (graphBuilderConfig.eagerResolving()) {
-            catchType = lookupType(block.handler.catchTypeCPI(), INSTANCEOF);
+                catchType = lookupType(block.handler.catchTypeCPI(), INSTANCEOF);
-        }
+            }
-        if (catchType instanceof ResolvedJavaType) {
+            if (catchType instanceof ResolvedJavaType) {
-            TypeReference checkedCatchType = TypeReference.createTrusted(graph.getAssumptions(), (ResolvedJavaType) catchType);
+                TypeReference checkedCatchType = TypeReference.createTrusted(graph.getAssumptions(), (ResolvedJavaType) catchType);
 
-            if (graphBuilderConfig.getSkippedExceptionTypes() != null) {
+                if (graphBuilderConfig.getSkippedExceptionTypes() != null) {
-                for (ResolvedJavaType skippedType : graphBuilderConfig.getSkippedExceptionTypes()) {
+                    for (ResolvedJavaType skippedType : graphBuilderConfig.getSkippedExceptionTypes()) {
-                    if (skippedType.isAssignableFrom(checkedCatchType.getType())) {
+                        if (skippedType.isAssignableFrom(checkedCatchType.getType())) {
-                        BciBlock nextBlock = block.getSuccessorCount() == 1 ? blockMap.getUnwindBlock() : block.getSuccessor(1);
+                            BciBlock nextBlock = block.getSuccessorCount() == 1 ? blockMap.getUnwindBlock() : block.getSuccessor(1);
-                        ValueNode exception = frameState.stack[0];
+                            ValueNode exception = frameState.stack[0];
-                        FixedNode trueSuccessor = graph.add(new DeoptimizeNode(InvalidateReprofile, UnreachedCode));
+                            FixedNode trueSuccessor = graph.add(new DeoptimizeNode(InvalidateReprofile, UnreachedCode));
-                        FixedNode nextDispatch = createTarget(nextBlock, frameState);
+                            FixedNode nextDispatch = createTarget(nextBlock, frameState);
-                        append(new IfNode(graph.addOrUniqueWithInputs(createInstanceOf(checkedCatchType, exception)), trueSuccessor, nextDispatch, 0));
+                            append(new IfNode(graph.addOrUniqueWithInputs(createInstanceOf(checkedCatchType, exception)), trueSuccessor, nextDispatch, 0));
-                        return;
+                            return;
+                        }
                     }
                 }
-            }
 
-            BciBlock nextBlock = block.getSuccessorCount() == 1 ? blockMap.getUnwindBlock() : block.getSuccessor(1);
+                BciBlock nextBlock = block.getSuccessorCount() == 1 ? blockMap.getUnwindBlock() : block.getSuccessor(1);
-            ValueNode exception = frameState.stack[0];
+                ValueNode exception = frameState.stack[0];
-            /* Anchor for the piNode, which must be before any LoopExit inserted by createTarget. */
+                /*
-            BeginNode piNodeAnchor = graph.add(new BeginNode());
+                 * Anchor for the piNode, which must be before any LoopExit inserted by
-            ObjectStamp checkedStamp = StampFactory.objectNonNull(checkedCatchType);
+                 * createTarget.
-            PiNode piNode = graph.addWithoutUnique(new PiNode(exception, checkedStamp));
+                 */
-            frameState.pop(JavaKind.Object);
+                BeginNode piNodeAnchor = graph.add(new BeginNode());
-            frameState.push(JavaKind.Object, piNode);
+                ObjectStamp checkedStamp = StampFactory.objectNonNull(checkedCatchType);
-            FixedNode catchSuccessor = createTarget(block.getSuccessor(0), frameState);
+                PiNode piNode = graph.addWithoutUnique(new PiNode(exception, checkedStamp));
-            frameState.pop(JavaKind.Object);
+                frameState.pop(JavaKind.Object);
-            frameState.push(JavaKind.Object, exception);
+                frameState.push(JavaKind.Object, piNode);
-            FixedNode nextDispatch = createTarget(nextBlock, frameState);
+                FixedNode catchSuccessor = createTarget(block.getSuccessor(0), frameState);
-            piNodeAnchor.setNext(catchSuccessor);
+                frameState.pop(JavaKind.Object);
-            IfNode ifNode = append(new IfNode(graph.unique(createInstanceOf(checkedCatchType, exception)), piNodeAnchor, nextDispatch, 0.5));
+                frameState.push(JavaKind.Object, exception);
-            assert ifNode.trueSuccessor() == piNodeAnchor;
+                FixedNode nextDispatch = createTarget(nextBlock, frameState);
-            piNode.setGuard(ifNode.trueSuccessor());
+                piNodeAnchor.setNext(catchSuccessor);
-        } else {
+                IfNode ifNode = append(new IfNode(graph.unique(createInstanceOf(checkedCatchType, exception)), piNodeAnchor, nextDispatch, 0.5));
-            handleUnresolvedExceptionType(catchType);
+                assert ifNode.trueSuccessor() == piNodeAnchor;
+                piNode.setGuard(ifNode.trueSuccessor());
+            } else {
+                handleUnresolvedExceptionType(catchType);
+            }
         }
     }
 
@@ -3220,22 +3268,22 @@ public class BytecodeParser implements GraphBuilderContext {
     }
 
     protected double getProfileProbability(boolean negate) {
-        double probability;
         if (profilingInfo == null) {
-            probability = 0.5;
+            return 0.5;
-        } else {
+        }
-            assert assertAtIfBytecode();
+
-            probability = profilingInfo.getBranchTakenProbability(bci());
+        assert assertAtIfBytecode();
-            if (probability < 0) {
+        double probability = profilingInfo.getBranchTakenProbability(bci());
-                assert probability == -1 : "invalid probability";
+
-                debug.log("missing probability in %s at bci %d", code, bci());
+        if (probability < 0) {
-                probability = 0.5;
+            assert probability == -1 : "invalid probability";
-            } else {
+            debug.log("missing probability in %s at bci %d", code, bci());
-                if (negate) {
+            return 0.5;
-                    // the probability coming from profile is about the original condition
+        }
-                    probability = 1 - probability;
+
-                }
+        if (negate && shouldComplementProbability()) {
-            }
+            // the probability coming from profile is about the original condition
+            probability = 1 - probability;
         }
         return probability;
     }
@@ -3277,7 +3325,10 @@ public class BytecodeParser implements GraphBuilderContext {
             BciBlock tmpBlock = trueBlock;
             trueBlock = falseBlock;
             falseBlock = tmpBlock;
-            probability = 1 - probability;
+            if (shouldComplementProbability()) {
+                // the probability coming from profile is about the original condition
+                probability = 1 - probability;
+            }
             condition = logicNegationNode.getValue();
         }
 
@@ -3288,9 +3339,13 @@ public class BytecodeParser implements GraphBuilderContext {
                 condition = genUnique(condition);
             }
 
+            NodeSourcePosition currentPosition = graph.currentNodeSourcePosition();
             if (isNeverExecutedCode(probability)) {
                 if (!graph.isOSR() || getParent() != null || graph.getEntryBCI() != trueBlock.startBci) {
-                    append(new FixedGuardNode(condition, UnreachedCode, InvalidateReprofile, true));
+                    NodeSourcePosition survivingSuccessorPosition = graph.trackNodeSourcePosition()
+                                    ? new NodeSourcePosition(currentPosition.getCaller(), currentPosition.getMethod(), falseBlock.startBci)
+                                    : null;
+                    append(new FixedGuardNode(condition, UnreachedCode, InvalidateReprofile, true, survivingSuccessorPosition));
                     if (profilingPlugin != null && profilingPlugin.shouldProfile(this, method)) {
                         profilingPlugin.profileGoto(this, method, bci(), falseBlock.startBci, stateBefore);
                     }
@@ -3299,7 +3354,10 @@ public class BytecodeParser implements GraphBuilderContext {
                 }
             } else if (isNeverExecutedCode(1 - probability)) {
                 if (!graph.isOSR() || getParent() != null || graph.getEntryBCI() != falseBlock.startBci) {
-                    append(new FixedGuardNode(condition, UnreachedCode, InvalidateReprofile, false));
+                    NodeSourcePosition survivingSuccessorPosition = graph.trackNodeSourcePosition()
+                                    ? new NodeSourcePosition(currentPosition.getCaller(), currentPosition.getMethod(), trueBlock.startBci)
+                                    : null;
+                    append(new FixedGuardNode(condition, UnreachedCode, InvalidateReprofile, false, survivingSuccessorPosition));
                     if (profilingPlugin != null && profilingPlugin.shouldProfile(this, method)) {
                         profilingPlugin.profileGoto(this, method, bci(), trueBlock.startBci, stateBefore);
                     }
@@ -3332,6 +3390,14 @@ public class BytecodeParser implements GraphBuilderContext {
         }
     }
 
+    /**
+     * Hook for subclasses to decide whether the IfNode probability should be complemented during
+     * conversion to Graal IR.
+     */
+    protected boolean shouldComplementProbability() {
+        return true;
+    }
+
     /**
      * Hook for subclasses to generate custom nodes before an IfNode.
      */
@@ -3587,29 +3653,36 @@ public class BytecodeParser implements GraphBuilderContext {
 
     private void genLoadIndexed(JavaKind kind) {
         ValueNode index = frameState.pop(JavaKind.Int);
-        ValueNode array = emitExplicitExceptions(frameState.pop(JavaKind.Object), index);
+        ValueNode array = frameState.pop(JavaKind.Object);
+
+        array = maybeEmitExplicitNullCheck(array);
+        GuardingNode boundsCheck = maybeEmitExplicitBoundsCheck(array, index);
 
         for (NodePlugin plugin : graphBuilderConfig.getPlugins().getNodePlugins()) {
-            if (plugin.handleLoadIndexed(this, array, index, kind)) {
+            if (plugin.handleLoadIndexed(this, array, index, boundsCheck, kind)) {
                 return;
             }
         }
 
-        frameState.push(kind, append(genLoadIndexed(array, index, kind)));
+        frameState.push(kind, append(genLoadIndexed(array, index, boundsCheck, kind)));
     }
 
     private void genStoreIndexed(JavaKind kind) {
         ValueNode value = frameState.pop(kind);
         ValueNode index = frameState.pop(JavaKind.Int);
-        ValueNode array = emitExplicitExceptions(frameState.pop(JavaKind.Object), index);
+        ValueNode array = frameState.pop(JavaKind.Object);
+
+        array = maybeEmitExplicitNullCheck(array);
+        GuardingNode boundsCheck = maybeEmitExplicitBoundsCheck(array, index);
+        GuardingNode storeCheck = maybeEmitExplicitStoreCheck(array, kind, value);
 
         for (NodePlugin plugin : graphBuilderConfig.getPlugins().getNodePlugins()) {
-            if (plugin.handleStoreIndexed(this, array, index, kind, value)) {
+            if (plugin.handleStoreIndexed(this, array, index, boundsCheck, storeCheck, kind, value)) {
                 return;
             }
         }
 
-        genStoreIndexed(array, index, kind, value);
+        genStoreIndexed(array, index, boundsCheck, storeCheck, kind, value);
     }
 
     private void genArithmeticOp(JavaKind kind, int opcode) {
@@ -3658,15 +3731,18 @@ public class BytecodeParser implements GraphBuilderContext {
     private void genIntegerDivOp(JavaKind kind, int opcode) {
         ValueNode y = frameState.pop(kind);
         ValueNode x = frameState.pop(kind);
+
+        GuardingNode zeroCheck = maybeEmitExplicitDivisionByZeroCheck(y);
+
         ValueNode v;
         switch (opcode) {
             case IDIV:
             case LDIV:
-                v = genIntegerDiv(x, y);
+                v = genIntegerDiv(x, y, zeroCheck);
                 break;
             case IREM:
             case LREM:
-                v = genIntegerRem(x, y);
+                v = genIntegerRem(x, y, zeroCheck);
                 break;
             default:
                 throw shouldNotReachHere();
@@ -3871,7 +3947,8 @@ public class BytecodeParser implements GraphBuilderContext {
             handleUnresolvedCheckCast(type, object);
             return;
         }
-        TypeReference checkedType = TypeReference.createTrusted(graph.getAssumptions(), (ResolvedJavaType) type);
+        ResolvedJavaType resolvedType = (ResolvedJavaType) type;
+        TypeReference checkedType = TypeReference.createTrusted(graph.getAssumptions(), resolvedType);
         JavaTypeProfile profile = getProfileForTypeCheck(checkedType);
 
         for (NodePlugin plugin : graphBuilderConfig.getPlugins().getNodePlugins()) {
@@ -3903,8 +3980,16 @@ public class BytecodeParser implements GraphBuilderContext {
             if (condition.isTautology()) {
                 castNode = object;
             } else {
-                FixedGuardNode fixedGuard = append(new FixedGuardNode(condition, DeoptimizationReason.ClassCastException, DeoptimizationAction.InvalidateReprofile, false));
+                GuardingNode guard;
-                castNode = append(PiNode.create(object, StampFactory.object(checkedType, nonNull), fixedGuard));
+                if (needsExplicitClassCastException(object)) {
+                    Constant hub = getConstantReflection().asObjectHub(resolvedType);
+                    Stamp hubStamp = getStampProvider().createHubStamp(StampFactory.object(TypeReference.createExactTrusted(resolvedType)));
+                    ConstantNode hubConstant = ConstantNode.forConstant(hubStamp, hub, getMetaAccess(), graph);
+                    guard = emitBytecodeExceptionCheck(condition, true, BytecodeExceptionKind.CLASS_CAST, object, hubConstant);
+                } else {
+                    guard = append(new FixedGuardNode(condition, DeoptimizationReason.ClassCastException, DeoptimizationAction.InvalidateReprofile, false));
+                }
+                castNode = append(PiNode.create(object, StampFactory.object(checkedType, nonNull), guard.asNode()));
             }
         }
         frameState.push(JavaKind.Object, castNode);
@@ -4124,7 +4209,7 @@ public class BytecodeParser implements GraphBuilderContext {
     }
 
     private void genGetField(JavaField field, ValueNode receiverInput) {
-        ValueNode receiver = emitExplicitExceptions(receiverInput);
+        ValueNode receiver = maybeEmitExplicitNullCheck(receiverInput);
         if (field instanceof ResolvedJavaField) {
             ResolvedJavaField resolvedField = (ResolvedJavaField) field;
             genGetField(resolvedField, receiver);
@@ -4163,29 +4248,56 @@ public class BytecodeParser implements GraphBuilderContext {
     }
 
     /**
-     * @param receiver the receiver of an object based operation
+     * Returns true if an explicit null check should be emitted for the given object.
-     * @param index the index of an array based operation that is to be tested for out of bounds.
+     *
-     *            This is null for a non-array operation.
+     * @param object The object that is accessed.
-     * @return the receiver value possibly modified to have a non-null stamp
      */
-    protected ValueNode emitExplicitExceptions(ValueNode receiver, ValueNode index) {
+    protected boolean needsExplicitNullCheckException(ValueNode object) {
-        if (needsExplicitException()) {
+        return needsExplicitException();
-            ValueNode nonNullReceiver = emitExplicitNullCheck(receiver);
-            ValueNode length = append(genArrayLength(nonNullReceiver));
-            emitExplicitBoundsCheck(index, length);
-            return nonNullReceiver;
-        }
-        return receiver;
     }
 
-    protected ValueNode emitExplicitExceptions(ValueNode receiver) {
+    /**
-        if (StampTool.isPointerNonNull(receiver) || !needsExplicitException()) {
+     * Returns true if an explicit null check should be emitted for the given object.
-            return receiver;
+     *
-        } else {
+     * @param array The array that is accessed.
-            return emitExplicitNullCheck(receiver);
+     * @param index The array index that is accessed.
-        }
+     */
+    protected boolean needsExplicitBoundsCheckException(ValueNode array, ValueNode index) {
+        return needsExplicitException();
     }
 
+    /**
+     * Returns true if an explicit check for a {@link ClassCastException} should be emitted for the
+     * given object.
+     *
+     * @param object The object that is accessed.
+     */
+    protected boolean needsExplicitClassCastException(ValueNode object) {
+        return needsExplicitException();
+    }
+
+    /**
+     * Returns true if an explicit null check should be emitted for the given object.
+     *
+     * @param array The array that is accessed.
+     * @param value The value that is stored into the array.
+     */
+    protected boolean needsExplicitStoreCheckException(ValueNode array, ValueNode value) {
+        return needsExplicitException();
+    }
+
+    /**
+     * Returns true if an explicit null check should be emitted for the given object.
+     *
+     * @param y The dividend.
+     */
+    protected boolean needsExplicitDivisionByZeroException(ValueNode y) {
+        return needsExplicitException();
+    }
+
+    /**
+     * Returns true if an explicit exception check should be emitted.
+     */
     protected boolean needsExplicitException() {
         BytecodeExceptionMode exceptionMode = graphBuilderConfig.getBytecodeExceptionMode();
         if (exceptionMode == BytecodeExceptionMode.CheckAll || StressExplicitExceptionCode.getValue(options)) {
@@ -4206,7 +4318,8 @@ public class BytecodeParser implements GraphBuilderContext {
     }
 
     private void genPutField(JavaField field, ValueNode value) {
-        ValueNode receiver = emitExplicitExceptions(frameState.pop(JavaKind.Object));
+        ValueNode receiver = maybeEmitExplicitNullCheck(frameState.pop(JavaKind.Object));
+
         if (field instanceof ResolvedJavaField) {
             ResolvedJavaField resolvedField = (ResolvedJavaField) field;
 
@@ -4703,7 +4816,7 @@ public class BytecodeParser implements GraphBuilderContext {
     }
 
     private void genArrayLength() {
-        ValueNode array = emitExplicitExceptions(frameState.pop(JavaKind.Object));
+        ValueNode array = maybeEmitExplicitNullCheck(frameState.pop(JavaKind.Object));
         frameState.push(JavaKind.Int, append(genArrayLength(array)));
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/jdk/UnsafeAllocateInstance01.java

@@ -67,6 +67,7 @@ public class UnsafeAllocateInstance01 extends JTTTest {
         runTest("testInstance");
     }
 
+    @Ignore("https://bugs.openjdk.java.net/browse/JDK-8153540")
     @Test
     public void run1() throws Throwable {
         runTest("testClassForException", UnsafeAllocateInstance01[].class);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/optimize/CE_InstanceOf.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.jtt.optimize;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.graalvm.compiler.jtt.JTTTest;
+import org.junit.Test;
+
+public class CE_InstanceOf extends JTTTest {
+
+    static class A {
+    }
+
+    static class B extends A {
+    }
+
+    static class C extends A {
+    }
+
+    public static A testRedundantCast(Object value) {
+        if (value != null && value.getClass() == A.class) {
+            return (A) value;
+        }
+        return null;
+    }
+
+    public static boolean testRedundantInstanceOf(Object value) {
+        if (value != null && value.getClass() == A.class) {
+            return (value instanceof A);
+        }
+        return false;
+    }
+
+    public static boolean testRedundantInstanceOf2(Object value) {
+        if (value.getClass() == A.class) {
+            return (value instanceof A);
+        }
+        return false;
+    }
+
+    public static boolean testNonRedundantInstanceOf(Object value) {
+        if (value instanceof A) {
+            return (value != null && value.getClass() == A.class);
+        }
+        return false;
+    }
+
+    public static Object testNonRedundantInstanceOf2(Object value) {
+        if (value != null && value.getClass() == Object[].class) {
+            return ((Object[]) value)[0];
+        }
+        return null;
+    }
+
+    private static final List<A> testArgs = Collections.unmodifiableList(Arrays.asList(new A(), new B(), null));
+
+    @Test
+    public void run0() throws Throwable {
+        for (A a : testArgs) {
+            runTest("testRedundantCast", a);
+        }
+    }
+
+    @Test
+    public void run1() throws Throwable {
+        for (A a : testArgs) {
+            runTest("testRedundantInstanceOf", a);
+        }
+    }
+
+    @Test
+    public void run2() throws Throwable {
+        for (A a : testArgs) {
+            runTest("testRedundantInstanceOf2", a);
+        }
+    }
+
+    @Test
+    public void run3() throws Throwable {
+        for (A a : testArgs) {
+            runTest("testNonRedundantInstanceOf", a);
+        }
+    }
+
+    @Test
+    public void run4() throws Throwable {
+        for (A a : testArgs) {
+            runTest("testNonRedundantInstanceOf2", a);
+        }
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/optimize/InstanceOf.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.jtt.optimize;
+
+import org.graalvm.compiler.jtt.JTTTest;
+import org.graalvm.compiler.nodes.LogicNode;
+import org.graalvm.compiler.nodes.java.InstanceOfNode;
+import org.junit.Test;
+
+/**
+ * A few tests of expected simplifications by {@link InstanceOfNode#implies(boolean, LogicNode)}.
+ */
+public class InstanceOf extends JTTTest {
+
+    static class A {
+    }
+
+    static class B extends A {
+    }
+
+    static class C extends B {
+    }
+
+    public boolean testSnippet1(Object o) {
+        if (o instanceof B) {
+            if (o instanceof A) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public boolean testSnippet2(Object o) {
+        if (o instanceof A) {
+            return true;
+        } else {
+            return o instanceof B;
+        }
+    }
+
+    @Test
+    public void test1() {
+        runTest("testSnippet1", new A());
+        runTest("testSnippet1", new B());
+        runTest("testSnippet1", new C());
+    }
+
+    @Test
+    public void test2() {
+        runTest("testSnippet2", new A());
+        runTest("testSnippet2", new B());
+        runTest("testSnippet2", new C());
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ArithmeticLIRGeneratorTool.java

@@ -37,5 +37,20 @@ public interface AArch64ArithmeticLIRGeneratorTool extends ArithmeticLIRGenerato
 
     Value emitCountTrailingZeros(Value value);
 
+    enum RoundingMode {
+        NEAREST(0),
+        DOWN(1),
+        UP(2),
+        TRUNCATE(3);
+
+        public final int encoding;
+
+        RoundingMode(int encoding) {
+            this.encoding = encoding;
+        }
+    }
+
+    Value emitRound(Value value, RoundingMode mode);
+
     void emitCompareOp(AArch64Kind cmpKind, Variable left, Value right);
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ArithmeticOp.java

@@ -74,6 +74,9 @@ public enum AArch64ArithmeticOp {
     FREM,
     FNEG,
     FABS,
+    FRINTM,
+    FRINTN,
+    FRINTP,
     SQRT;
 
     /**
@@ -133,6 +136,15 @@ public enum AArch64ArithmeticOp {
                 case FABS:
                     masm.fabs(size, dst, src);
                     break;
+                case FRINTM:
+                    masm.frintm(size, dst, src);
+                    break;
+                case FRINTN:
+                    masm.frintn(size, dst, src);
+                    break;
+                case FRINTP:
+                    masm.frintp(size, dst, src);
+                    break;
                 case SQRT:
                     masm.fsqrt(size, dst, src);
                     break;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64AtomicMove.java

@@ -26,13 +26,12 @@ import static jdk.vm.ci.code.ValueUtil.asRegister;
 
 import org.graalvm.compiler.asm.Label;
 import org.graalvm.compiler.asm.aarch64.AArch64Assembler;
+import org.graalvm.compiler.asm.aarch64.AArch64Assembler.ShiftType;
 import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler;
 import org.graalvm.compiler.lir.LIRInstructionClass;
 import org.graalvm.compiler.lir.Opcode;
 import org.graalvm.compiler.lir.asm.CompilationResultBuilder;
 
-import jdk.vm.ci.aarch64.AArch64.CPUFeature;
-import jdk.vm.ci.aarch64.AArch64.Flag;
 import jdk.vm.ci.aarch64.AArch64Kind;
 import jdk.vm.ci.code.Register;
 import jdk.vm.ci.meta.AllocatableValue;
@@ -76,15 +75,15 @@ public class AArch64AtomicMove {
             Register address = asRegister(addressValue);
             Register result = asRegister(resultValue);
             Register newVal = asRegister(newValue);
-             if (masm.supports(CPUFeature.LSE) || masm.isFlagSet(Flag.UseLSE)) {
+            if (AArch64LIRFlagsVersioned.useLSE(masm)) {
                 Register expected = asRegister(expectedValue);
                 masm.mov(size, result, expected);
-                masm.cas(size, expected, newVal, address, true /*acquire*/, true /*release*/);
+                masm.cas(size, expected, newVal, address, true /* acquire */, true /* release */);
                 AArch64Compare.gpCompare(masm, resultValue, expectedValue);
             } else {
-                // We could avoid using a scratch register here, by reusing resultValue for the stlxr
+                // We could avoid using a scratch register here, by reusing resultValue for the
-                // success flag and issue a mov resultValue, expectedValue in case of success before
+                // stlxr success flag and issue a mov resultValue, expectedValue in case of success
-                // returning.
+                // before returning.
                 Register scratch = asRegister(scratchValue);
                 Label retry = new Label();
                 Label fail = new Label();
@@ -99,4 +98,59 @@ public class AArch64AtomicMove {
             }
         }
     }
+
+    /**
+     * Load (Read) and Add instruction. Does the following atomically: <code>
+     *  ATOMIC_READ_AND_ADD(addend, result, address):
+     *    result = *address
+     *    *address = result + addend
+     *    return result
+     * </code>
+     */
+    @Opcode("ATOMIC_READ_AND_ADD")
+    public static final class AtomicReadAndAddOp extends AArch64LIRInstruction {
+        public static final LIRInstructionClass<AtomicReadAndAddOp> TYPE = LIRInstructionClass.create(AtomicReadAndAddOp.class);
+
+        private final AArch64Kind accessKind;
+
+        @Def protected AllocatableValue resultValue;
+        @Alive protected AllocatableValue addressValue;
+        @Alive protected Value deltaValue;
+        @Temp protected AllocatableValue scratchValue1;
+        @Temp protected AllocatableValue scratchValue2;
+
+        public AtomicReadAndAddOp(AArch64Kind kind, AllocatableValue result, AllocatableValue address, Value delta, AllocatableValue scratch1, AllocatableValue scratch2) {
+            super(TYPE);
+            this.accessKind = kind;
+            this.resultValue = result;
+            this.addressValue = address;
+            this.deltaValue = delta;
+            this.scratchValue1 = scratch1;
+            this.scratchValue2 = scratch2;
+        }
+
+        @Override
+        public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+            assert accessKind.isInteger();
+            final int size = accessKind.getSizeInBytes() * Byte.SIZE;
+
+            Register address = asRegister(addressValue);
+            Register delta = asRegister(deltaValue);
+            Register result = asRegister(resultValue);
+
+            if (AArch64LIRFlagsVersioned.useLSE(masm)) {
+                masm.ldadd(size, delta, result, address, true, true);
+            } else {
+                Register scratch1 = asRegister(scratchValue1);
+                Register scratch2 = asRegister(scratchValue2);
+                Label retry = new Label();
+                masm.bind(retry);
+                masm.ldaxr(size, result, address);
+                masm.add(size, scratch1, result, delta, ShiftType.LSL, 0);
+                masm.stlxr(size, scratch2, scratch1, address);
+                // if scratch2 == 0 then write successful, else retry
+                masm.cbnz(32, scratch2, retry);
+            }
+        }
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64Call.java

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,6 +23,7 @@
  */
 package org.graalvm.compiler.lir.aarch64;
 
+import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC;
 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.ILLEGAL;
 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG;
 import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK;
@@ -198,12 +200,16 @@ public class AArch64Call {
     public static void directCall(CompilationResultBuilder crb, AArch64MacroAssembler masm, InvokeTarget callTarget, Register scratch, LIRFrameState info, Label label) {
         int before = masm.position();
         if (scratch != null) {
-            /*
+            if (GeneratePIC.getValue(crb.getOptions())) {
-             * Offset might not fit into a 28-bit immediate, generate an indirect call with a 64-bit
+                masm.bl(0);
-             * immediate address which is fixed up by HotSpot.
+            } else {
-             */
+                /*
-            masm.movNativeAddress(scratch, 0L);
+                 * Offset might not fit into a 28-bit immediate, generate an indirect call with a
-            masm.blr(scratch);
+                 * 64-bit immediate address which is fixed up by HotSpot.
+                 */
+                masm.movNativeAddress(scratch, 0L);
+                masm.blr(scratch);
+            }
         } else {
             // Address is fixed up by HotSpot.
             masm.bl(0);
@@ -230,8 +236,12 @@ public class AArch64Call {
     public static void directJmp(CompilationResultBuilder crb, AArch64MacroAssembler masm, InvokeTarget callTarget) {
         try (AArch64MacroAssembler.ScratchRegister scratch = masm.getScratchRegister()) {
             int before = masm.position();
-            masm.movNativeAddress(scratch.getRegister(), 0L);
+            if (GeneratePIC.getValue(crb.getOptions())) {
-            masm.jmp(scratch.getRegister());
+                masm.jmp();
+            } else {
+                masm.movNativeAddress(scratch.getRegister(), 0L);
+                masm.jmp(scratch.getRegister());
+            }
             int after = masm.position();
             crb.recordDirectCall(before, after, callTarget, null);
             masm.ensureUniquePC();

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64LIRFlagsVersioned.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2018, 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.lir.aarch64;
+
+import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler;
+
+import jdk.vm.ci.aarch64.AArch64.CPUFeature;
+import jdk.vm.ci.aarch64.AArch64.Flag;
+
+public class AArch64LIRFlagsVersioned {
+    public static boolean useLSE(AArch64MacroAssembler masm) {
+        return masm.supports(CPUFeature.LSE) || masm.isFlagSet(Flag.UseLSE);
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64Move.java

@@ -162,7 +162,12 @@ public class AArch64Move {
         @Override
         public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
             Register dst = asRegister(result);
-            masm.loadAddress(dst, (AArch64Address) crb.recordDataReferenceInCode(data), data.getAlignment());
+            if (crb.compilationResult.isImmutablePIC()) {
+                crb.recordDataReferenceInCode(data);
+                masm.addressOf(dst);
+            } else {
+                masm.loadAddress(dst, (AArch64Address) crb.recordDataReferenceInCode(data), data.getAlignment());
+            }
         }
     }
 
@@ -192,6 +197,7 @@ public class AArch64Move {
     public static class MembarOp extends AArch64LIRInstruction {
         public static final LIRInstructionClass<MembarOp> TYPE = LIRInstructionClass.create(MembarOp.class);
 
+        // For future use.
         @SuppressWarnings("unused") private final int barriers;
 
         public MembarOp(int barriers) {
@@ -200,14 +206,15 @@ public class AArch64Move {
         }
 
         @Override
-        public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+        // The odd-looking @SuppressWarnings("all") is here because of
+        // a compiler bug which warns that crb is unused, and also
+        // warns that @SuppressWarnings("unused") is unnecessary.
+        public void emitCode(@SuppressWarnings("all") CompilationResultBuilder crb, AArch64MacroAssembler masm) {
             // As I understand it load acquire/store release have the same semantics as on IA64
             // and allow us to handle LoadStore, LoadLoad and StoreStore without an explicit
             // barrier.
             // But Graal support to figure out if a load/store is volatile is non-existant so for
-            // now
+            // now just use memory barriers everywhere.
-            // just use
-            // memory barriers everywhere.
             // if ((barrier & MemoryBarriers.STORE_LOAD) != 0) {
             masm.dmb(AArch64MacroAssembler.BarrierKind.ANY_ANY);
             // }
@@ -408,7 +415,7 @@ public class AArch64Move {
         }
     }
 
-    private static void reg2stack(CompilationResultBuilder crb, AArch64MacroAssembler masm, AllocatableValue result, AllocatableValue input) {
+    static void reg2stack(CompilationResultBuilder crb, AArch64MacroAssembler masm, AllocatableValue result, AllocatableValue input) {
         AArch64Address dest = loadStackSlotAddress(crb, masm, asStackSlot(result), Value.ILLEGAL);
         Register src = asRegister(input);
         // use the slot kind to define the operand size
@@ -421,7 +428,7 @@ public class AArch64Move {
         }
     }
 
-    private static void stack2reg(CompilationResultBuilder crb, AArch64MacroAssembler masm, AllocatableValue result, AllocatableValue input) {
+    static void stack2reg(CompilationResultBuilder crb, AArch64MacroAssembler masm, AllocatableValue result, AllocatableValue input) {
         AArch64Kind kind = (AArch64Kind) input.getPlatformKind();
         // use the slot kind to define the operand size
         final int size = kind.getSizeInBytes() * Byte.SIZE;
@@ -466,6 +473,12 @@ public class AArch64Move {
             case Float:
                 if (AArch64MacroAssembler.isFloatImmediate(input.asFloat())) {
                     masm.fmov(32, dst, input.asFloat());
+                } else if (crb.compilationResult.isImmutablePIC()) {
+                    try (ScratchRegister scr = masm.getScratchRegister()) {
+                        Register scratch = scr.getRegister();
+                        masm.mov(scratch, Float.floatToRawIntBits(input.asFloat()));
+                        masm.fmov(32, dst, scratch);
+                    }
                 } else {
                     masm.fldr(32, dst, (AArch64Address) crb.asFloatConstRef(input));
                 }
@@ -473,6 +486,12 @@ public class AArch64Move {
             case Double:
                 if (AArch64MacroAssembler.isDoubleImmediate(input.asDouble())) {
                     masm.fmov(64, dst, input.asDouble());
+                } else if (crb.compilationResult.isImmutablePIC()) {
+                    try (ScratchRegister scr = masm.getScratchRegister()) {
+                        Register scratch = scr.getRegister();
+                        masm.mov(scratch, Double.doubleToRawLongBits(input.asDouble()));
+                        masm.fmov(64, dst, scratch);
+                    }
                 } else {
                     masm.fldr(64, dst, (AArch64Address) crb.asDoubleConstRef(input));
                 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64RestoreRegistersOp.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, Red Hat Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.lir.aarch64;
+
+import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK;
+import static jdk.vm.ci.code.ValueUtil.asStackSlot;
+import static jdk.vm.ci.code.ValueUtil.isStackSlot;
+
+import java.util.Arrays;
+
+import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler;
+import org.graalvm.compiler.lir.LIRInstructionClass;
+import org.graalvm.compiler.lir.LIRValueUtil;
+import org.graalvm.compiler.lir.Opcode;
+import org.graalvm.compiler.lir.asm.CompilationResultBuilder;
+
+import jdk.vm.ci.code.Register;
+import jdk.vm.ci.code.StackSlot;
+import jdk.vm.ci.meta.AllocatableValue;
+
+/**
+ * Restores registers from stack slots.
+ */
+@Opcode("RESTORE_REGISTER")
+public class AArch64RestoreRegistersOp extends AArch64LIRInstruction {
+    public static final LIRInstructionClass<AArch64RestoreRegistersOp> TYPE = LIRInstructionClass.create(AArch64RestoreRegistersOp.class);
+
+    /**
+     * The slots from which the registers are restored.
+     */
+    @Use(STACK) protected final AllocatableValue[] slots;
+
+    /**
+     * The operation that saved the registers restored by this operation.
+     */
+    private final AArch64SaveRegistersOp save;
+
+    public AArch64RestoreRegistersOp(AllocatableValue[] values, AArch64SaveRegistersOp save) {
+        this(TYPE, values, save);
+    }
+
+    protected AArch64RestoreRegistersOp(LIRInstructionClass<? extends AArch64RestoreRegistersOp> c, AllocatableValue[] values, AArch64SaveRegistersOp save) {
+        super(c);
+        assert Arrays.asList(values).stream().allMatch(LIRValueUtil::isVirtualStackSlot);
+        this.slots = values;
+        this.save = save;
+    }
+
+    protected Register[] getSavedRegisters() {
+        return save.savedRegisters;
+    }
+
+    protected void restoreRegister(CompilationResultBuilder crb, AArch64MacroAssembler masm, Register result, StackSlot input) {
+        AArch64Move.stack2reg(crb, masm, result.asValue(), input);
+    }
+
+    @Override
+    public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+        Register[] savedRegisters = getSavedRegisters();
+        for (int i = 0; i < savedRegisters.length; i++) {
+            if (savedRegisters[i] != null) {
+                assert isStackSlot(slots[i]) : "not a StackSlot: " + slots[i];
+                restoreRegister(crb, masm, savedRegisters[i], asStackSlot(slots[i]));
+            }
+        }
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64SaveRegistersOp.java

@@ -0,0 +1,168 @@
+/*
+ * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.graalvm.compiler.lir.aarch64;
+
+import static jdk.vm.ci.code.ValueUtil.asStackSlot;
+import static jdk.vm.ci.code.ValueUtil.isStackSlot;
+import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK;
+
+import java.util.Arrays;
+
+import jdk.internal.vm.compiler.collections.EconomicSet;
+import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler;
+import org.graalvm.compiler.lir.LIRInstructionClass;
+import org.graalvm.compiler.lir.LIRValueUtil;
+import org.graalvm.compiler.lir.Opcode;
+import org.graalvm.compiler.lir.StandardOp.SaveRegistersOp;
+import org.graalvm.compiler.lir.asm.CompilationResultBuilder;
+import org.graalvm.compiler.lir.framemap.FrameMap;
+
+import jdk.vm.ci.code.Register;
+import jdk.vm.ci.code.RegisterSaveLayout;
+import jdk.vm.ci.code.StackSlot;
+import jdk.vm.ci.meta.AllocatableValue;
+
+/**
+ * Saves registers to stack slots.
+ */
+@Opcode("SAVE_REGISTER")
+public class AArch64SaveRegistersOp extends AArch64LIRInstruction implements SaveRegistersOp {
+    public static final LIRInstructionClass<AArch64SaveRegistersOp> TYPE = LIRInstructionClass.create(AArch64SaveRegistersOp.class);
+
+    /**
+     * The registers (potentially) saved by this operation.
+     */
+    protected final Register[] savedRegisters;
+
+    /**
+     * The slots to which the registers are saved.
+     */
+    @Def(STACK) protected final AllocatableValue[] slots;
+
+    /**
+     * Specifies if {@link #remove(EconomicSet)} should have an effect.
+     */
+    protected final boolean supportsRemove;
+
+    /**
+     *
+     * @param savedRegisters the registers saved by this operation which may be subject to
+     *            {@linkplain #remove(EconomicSet) pruning}
+     * @param savedRegisterLocations the slots to which the registers are saved
+     * @param supportsRemove determines if registers can be {@linkplain #remove(EconomicSet) pruned}
+     */
+    public AArch64SaveRegistersOp(Register[] savedRegisters, AllocatableValue[] savedRegisterLocations, boolean supportsRemove) {
+        this(TYPE, savedRegisters, savedRegisterLocations, supportsRemove);
+    }
+
+    public AArch64SaveRegistersOp(LIRInstructionClass<? extends AArch64SaveRegistersOp> c, Register[] savedRegisters, AllocatableValue[] savedRegisterLocations, boolean supportsRemove) {
+        super(c);
+        assert Arrays.asList(savedRegisterLocations).stream().allMatch(LIRValueUtil::isVirtualStackSlot);
+        this.savedRegisters = savedRegisters;
+        this.slots = savedRegisterLocations;
+        this.supportsRemove = supportsRemove;
+    }
+
+    protected void saveRegister(CompilationResultBuilder crb, AArch64MacroAssembler masm, StackSlot result, Register input) {
+        AArch64Move.reg2stack(crb, masm, result, input.asValue());
+    }
+
+    @Override
+    public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) {
+        for (int i = 0; i < savedRegisters.length; i++) {
+            if (savedRegisters[i] != null) {
+                assert isStackSlot(slots[i]) : "not a StackSlot: " + slots[i];
+                saveRegister(crb, masm, asStackSlot(slots[i]), savedRegisters[i]);
+            }
+        }
+    }
+
+    public AllocatableValue[] getSlots() {
+        return slots;
+    }
+
+    @Override
+    public boolean supportsRemove() {
+        return supportsRemove;
+    }
+
+    @Override
+    public int remove(EconomicSet<Register> doNotSave) {
+        if (!supportsRemove) {
+            throw new UnsupportedOperationException();
+        }
+        return prune(doNotSave, savedRegisters);
+    }
+
+    static int prune(EconomicSet<Register> toRemove, Register[] registers) {
+        int pruned = 0;
+        for (int i = 0; i < registers.length; i++) {
+            if (registers[i] != null) {
+                if (toRemove.contains(registers[i])) {
+                    registers[i] = null;
+                    pruned++;
+                }
+            }
+        }
+        return pruned;
+    }
+
+    @Override
+    public RegisterSaveLayout getMap(FrameMap frameMap) {
+        int total = 0;
+        for (int i = 0; i < savedRegisters.length; i++) {
+            if (savedRegisters[i] != null) {
+                total++;
+            }
+        }
+        Register[] keys = new Register[total];
+        int[] values = new int[total];
+        if (total != 0) {
+            int mapIndex = 0;
+            for (int i = 0; i < savedRegisters.length; i++) {
+                if (savedRegisters[i] != null) {
+                    keys[mapIndex] = savedRegisters[i];
+                    assert isStackSlot(slots[i]) : "not a StackSlot: " + slots[i];
+                    StackSlot slot = asStackSlot(slots[i]);
+                    values[mapIndex] = indexForStackSlot(frameMap, slot);
+                    mapIndex++;
+                }
+            }
+            assert mapIndex == total;
+        }
+        return new RegisterSaveLayout(keys, values);
+    }
+
+    /**
+     * Computes the index of a stack slot relative to slot 0. This is also the bit index of stack
+     * slots in the reference map.
+     *
+     * @param slot a stack slot
+     * @return the index of the stack slot
+     */
+    private static int indexForStackSlot(FrameMap frameMap, StackSlot slot) {
+        assert frameMap.offsetForStackSlot(slot) % frameMap.getTarget().wordSize == 0;
+        int value = frameMap.offsetForStackSlot(slot) / frameMap.getTarget().wordSize;
+        return value;
+    }
+}

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64Move.java

@@ -431,6 +431,12 @@ public class AMD64Move {
                 masm.lock();
             }
             switch (accessKind) {
+                case BYTE:
+                    masm.cmpxchgb(asRegister(newValue), address.toAddress());
+                    break;
+                case WORD:
+                    masm.cmpxchgw(asRegister(newValue), address.toAddress());
+                    break;
                 case DWORD:
                     masm.cmpxchgl(asRegister(newValue), address.toAddress());
                     break;
@@ -468,6 +474,12 @@ public class AMD64Move {
                 masm.lock();
             }
             switch (accessKind) {
+                case BYTE:
+                    masm.xaddb(address.toAddress(), asRegister(result));
+                    break;
+                case WORD:
+                    masm.xaddw(address.toAddress(), asRegister(result));
+                    break;
                 case DWORD:
                     masm.xaddl(address.toAddress(), asRegister(result));
                     break;
@@ -502,6 +514,12 @@ public class AMD64Move {
         public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
             move(accessKind, crb, masm, result, newValue);
             switch (accessKind) {
+                case BYTE:
+                    masm.xchgb(asRegister(result), address.toAddress());
+                    break;
+                case WORD:
+                    masm.xchgw(asRegister(result), address.toAddress());
+                    break;
                 case DWORD:
                     masm.xchgl(asRegister(result), address.toAddress());
                     break;

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/LIRGeneratorTool.java

@@ -142,17 +142,18 @@ public interface LIRGeneratorTool extends DiagnosticLIRGeneratorTool, ValueKindF
 
     void emitNullCheck(Value address, LIRFrameState state);
 
-    Variable emitLogicCompareAndSwap(Value address, Value expectedValue, Value newValue, Value trueValue, Value falseValue);
+    Variable emitLogicCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue, Value trueValue, Value falseValue);
 
-    Value emitValueCompareAndSwap(Value address, Value expectedValue, Value newValue);
+    Value emitValueCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue);
 
     /**
      * Emit an atomic read-and-add instruction.
      *
      * @param address address of the value to be read and written
+     * @param valueKind the access kind for the value to be written
      * @param delta the value to be added
      */
-    default Value emitAtomicReadAndAdd(Value address, Value delta) {
+    default Value emitAtomicReadAndAdd(Value address, ValueKind<?> valueKind, Value delta) {
         throw GraalError.unimplemented();
     }
 
@@ -160,9 +161,10 @@ public interface LIRGeneratorTool extends DiagnosticLIRGeneratorTool, ValueKindF
      * Emit an atomic read-and-write instruction.
      *
      * @param address address of the value to be read and written
+     * @param valueKind the access kind for the value to be written
      * @param newValue the new value to be written
      */
-    default Value emitAtomicReadAndWrite(Value address, Value newValue) {
+    default Value emitAtomicReadAndWrite(Value address, ValueKind<?> valueKind, Value newValue) {
         throw GraalError.unimplemented();
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop.phases/src/org/graalvm/compiler/loop/phases/LoopTransformations.java

@@ -116,12 +116,14 @@ public abstract class LoopTransformations {
         Position firstPosition = successors.next();
         AbstractBeginNode originalLoopBegin = BeginNode.begin(originalLoop.entryPoint());
         firstPosition.set(newControlSplit, originalLoopBegin);
+        originalLoopBegin.setNodeSourcePosition(firstPosition.get(firstNode).getNodeSourcePosition());
 
         while (successors.hasNext()) {
             Position position = successors.next();
             // create a new loop duplicate and connect it.
             LoopFragmentWhole duplicateLoop = originalLoop.duplicate();
             AbstractBeginNode newBegin = BeginNode.begin(duplicateLoop.entryPoint());
+            newBegin.setNodeSourcePosition(position.get(firstNode).getNodeSourcePosition());
             position.set(newControlSplit, newBegin);
 
             // For each cloned ControlSplitNode, simplify the proper path

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/CountedLoopInfo.java

@@ -118,7 +118,7 @@ public class CountedLoopInfo {
         }
         // round-away-from-zero divison: (range + stride -/+ 1) / stride
         ValueNode denominator = add(graph, range, sub(graph, absStride, one));
-        ValueNode div = unsignedDivBefore(graph, loop.entryPoint(), denominator, absStride);
+        ValueNode div = unsignedDivBefore(graph, loop.entryPoint(), denominator, absStride, null);
 
         if (assumePositive) {
             return div;
@@ -251,7 +251,7 @@ public class CountedLoopInfo {
             }
             assert graph.getGuardsStage().allowsFloatingGuards();
             overflowGuard = graph.unique(new GuardNode(cond, AbstractBeginNode.prevBegin(loop.entryPoint()), DeoptimizationReason.LoopLimitCheck, DeoptimizationAction.InvalidateRecompile, true,
-                            JavaConstant.NULL_POINTER)); // TODO gd: use speculation
+                            JavaConstant.NULL_POINTER, null)); // TODO gd: use speculation
             loop.loopBegin().setOverflowGuard(overflowGuard);
             return overflowGuard;
         }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopFragmentInside.java

@@ -543,6 +543,7 @@ public class LoopFragmentInside extends LoopFragment {
         }
     }
 
+    @SuppressWarnings("try")
     private AbstractBeginNode mergeEnds() {
         assert isDuplicate();
         List<EndNode> endsToMerge = new LinkedList<>();
@@ -562,9 +563,11 @@ public class LoopFragmentInside extends LoopFragment {
         if (endsToMerge.size() == 1) {
             AbstractEndNode end = endsToMerge.get(0);
             assert end.hasNoUsages();
-            newExit = graph.add(new BeginNode());
+            try (DebugCloseable position = end.withNodeSourcePosition()) {
-            end.replaceAtPredecessor(newExit);
+                newExit = graph.add(new BeginNode());
-            end.safeDelete();
+                end.replaceAtPredecessor(newExit);
+                end.safeDelete();
+            }
         } else {
             assert endsToMerge.size() > 1;
             AbstractMergeNode newExitMerge = graph.add(new MergeNode());

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/MathUtil.java

@@ -31,6 +31,7 @@ import org.graalvm.compiler.nodes.calc.BinaryArithmeticNode;
 import org.graalvm.compiler.nodes.calc.FixedBinaryNode;
 import org.graalvm.compiler.nodes.calc.SignedDivNode;
 import org.graalvm.compiler.nodes.calc.UnsignedDivNode;
+import org.graalvm.compiler.nodes.extended.GuardingNode;
 
 import java.util.function.BiFunction;
 
@@ -73,12 +74,12 @@ public class MathUtil {
         return BinaryArithmeticNode.sub(graph, v1, v2, NodeView.DEFAULT);
     }
 
-    public static ValueNode divBefore(StructuredGraph graph, FixedNode before, ValueNode dividend, ValueNode divisor) {
+    public static ValueNode divBefore(StructuredGraph graph, FixedNode before, ValueNode dividend, ValueNode divisor, GuardingNode zeroCheck) {
-        return fixedDivBefore(graph, before, dividend, divisor, (dend, sor) -> SignedDivNode.create(dend, sor, NodeView.DEFAULT));
+        return fixedDivBefore(graph, before, dividend, divisor, (dend, sor) -> SignedDivNode.create(dend, sor, zeroCheck, NodeView.DEFAULT));
     }
 
-    public static ValueNode unsignedDivBefore(StructuredGraph graph, FixedNode before, ValueNode dividend, ValueNode divisor) {
+    public static ValueNode unsignedDivBefore(StructuredGraph graph, FixedNode before, ValueNode dividend, ValueNode divisor, GuardingNode zeroCheck) {
-        return fixedDivBefore(graph, before, dividend, divisor, (dend, sor) -> UnsignedDivNode.create(dend, sor, NodeView.DEFAULT));
+        return fixedDivBefore(graph, before, dividend, divisor, (dend, sor) -> UnsignedDivNode.create(dend, sor, zeroCheck, NodeView.DEFAULT));
     }
 
     private static ValueNode fixedDivBefore(StructuredGraph graph, FixedNode before, ValueNode dividend, ValueNode divisor, BiFunction<ValueNode, ValueNode, ValueNode> createDiv) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.microbenchmarks/src/org/graalvm/compiler/microbenchmarks/lir/GraalCompilerState.java

@@ -319,7 +319,8 @@ public abstract class GraalCompilerState {
         assert !graph.isFrozen();
         ResolvedJavaMethod installedCodeOwner = graph.method();
         request = new Request<>(graph, installedCodeOwner, getProviders(), getBackend(), getDefaultGraphBuilderSuite(), OptimisticOptimizations.ALL,
-                        graph.getProfilingInfo(), createSuites(getOptions()), createLIRSuites(getOptions()), new CompilationResult(graph.compilationId()), CompilationResultBuilderFactory.Default);
+                        graph.getProfilingInfo(), createSuites(getOptions()), createLIRSuites(getOptions()), new CompilationResult(graph.compilationId()), CompilationResultBuilderFactory.Default,
+                        true);
     }
 
     /**

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodeinfo.processor/src/org/graalvm/compiler/nodeinfo/processor/GraphNodeProcessor.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,13 +31,12 @@ import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import javax.annotation.processing.AbstractProcessor;
 import javax.annotation.processing.FilerException;
-import javax.annotation.processing.ProcessingEnvironment;
 import javax.annotation.processing.RoundEnvironment;
 import javax.annotation.processing.SupportedAnnotationTypes;
 import javax.annotation.processing.SupportedSourceVersion;
 import javax.lang.model.SourceVersion;
+import javax.lang.model.element.AnnotationMirror;
 import javax.lang.model.element.Element;
 import javax.lang.model.element.ElementKind;
 import javax.lang.model.element.Modifier;
@@ -45,11 +44,16 @@ import javax.lang.model.element.TypeElement;
 import javax.lang.model.util.Types;
 import javax.tools.Diagnostic.Kind;
 
-import org.graalvm.compiler.nodeinfo.NodeInfo;
+import org.graalvm.compiler.processor.AbstractProcessor;
 
+/**
+ * Processor for {@value #NODE_INFO_CLASS_NAME} annotation.
+ */
 @SupportedSourceVersion(SourceVersion.RELEASE_8)
 @SupportedAnnotationTypes({"org.graalvm.compiler.nodeinfo.NodeInfo"})
 public class GraphNodeProcessor extends AbstractProcessor {
+    private static final String NODE_INFO_CLASS_NAME = "org.graalvm.compiler.nodeinfo.NodeInfo";
+
     @Override
     public SourceVersion getSupportedSourceVersion() {
         return SourceVersion.latest();
@@ -106,10 +110,6 @@ public class GraphNodeProcessor extends AbstractProcessor {
         message(kind, element, "Exception thrown during processing: %s", buf.toString());
     }
 
-    ProcessingEnvironment getProcessingEnv() {
-        return processingEnv;
-    }
-
     boolean isNodeType(Element element) {
         if (element.getKind() != ElementKind.CLASS) {
             return false;
@@ -134,17 +134,17 @@ public class GraphNodeProcessor extends AbstractProcessor {
 
         GraphNodeVerifier verifier = new GraphNodeVerifier(this);
 
-        for (Element element : roundEnv.getElementsAnnotatedWith(NodeInfo.class)) {
+        for (Element element : roundEnv.getElementsAnnotatedWith(getTypeElement(NODE_INFO_CLASS_NAME))) {
             scope = element;
             try {
                 if (!isNodeType(element)) {
-                    errorMessage(element, "%s can only be applied to Node subclasses", NodeInfo.class.getSimpleName());
+                    errorMessage(element, "%s can only be applied to Node subclasses", getSimpleName(NODE_INFO_CLASS_NAME));
                     continue;
                 }
 
-                NodeInfo nodeInfo = element.getAnnotation(NodeInfo.class);
+                AnnotationMirror nodeInfo = getAnnotation(element, getType(NODE_INFO_CLASS_NAME));
                 if (nodeInfo == null) {
-                    errorMessage(element, "Cannot get %s annotation from annotated element", NodeInfo.class.getSimpleName());
+                    errorMessage(element, "Cannot get %s annotation from annotated element", getSimpleName(NODE_INFO_CLASS_NAME));
                     continue;
                 }
 
@@ -154,7 +154,7 @@ public class GraphNodeProcessor extends AbstractProcessor {
                 if (!modifiers.contains(Modifier.FINAL) && !modifiers.contains(Modifier.ABSTRACT)) {
                     // TODO(thomaswue): Reenable this check.
                     // errorMessage(element, "%s annotated class must be either final or abstract",
-                    // NodeInfo.class.getSimpleName());
+                    // getSimpleName(NODE_INFO_CLASS_NAME));
                     // continue;
                 }
                 boolean found = false;
@@ -167,7 +167,7 @@ public class GraphNodeProcessor extends AbstractProcessor {
                     }
                 }
                 if (!found) {
-                    errorMessage(element, "%s annotated class must have a field named TYPE", NodeInfo.class.getSimpleName());
+                    errorMessage(element, "%s annotated class must have a field named TYPE", getSimpleName(NODE_INFO_CLASS_NAME));
                 }
 
                 if (!typeElement.equals(verifier.Node) && !modifiers.contains(Modifier.ABSTRACT)) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodeinfo.processor/src/org/graalvm/compiler/nodeinfo/processor/GraphNodeVerifier.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,7 +32,6 @@ import static javax.lang.model.element.Modifier.TRANSIENT;
 import java.util.List;
 import java.util.Set;
 
-import javax.annotation.processing.ProcessingEnvironment;
 import javax.lang.model.element.AnnotationMirror;
 import javax.lang.model.element.Element;
 import javax.lang.model.element.ElementKind;
@@ -42,17 +41,16 @@ import javax.lang.model.element.TypeElement;
 import javax.lang.model.element.VariableElement;
 import javax.lang.model.type.TypeMirror;
 import javax.lang.model.util.ElementFilter;
-import javax.lang.model.util.Elements;
 import javax.lang.model.util.Types;
 
+import org.graalvm.compiler.processor.AbstractProcessor;
+
 /**
  * Verifies static constraints on nodes.
  */
 public class GraphNodeVerifier {
 
-    private final GraphNodeProcessor env;
+    private final AbstractProcessor processor;
-    private final Types types;
-    private final Elements elements;
 
     // Checkstyle: stop
     private final TypeElement Input;
@@ -67,12 +65,8 @@ public class GraphNodeVerifier {
 
     // Checkstyle: resume
 
-    public GraphNodeVerifier(GraphNodeProcessor processor) {
+    public GraphNodeVerifier(AbstractProcessor processor) {
-        this.env = processor;
+        this.processor = processor;
-
-        this.types = processor.getProcessingEnv().getTypeUtils();
-        this.elements = processor.getProcessingEnv().getElementUtils();
-
         this.Input = getTypeElement("org.graalvm.compiler.graph.Node.Input");
         this.OptionalInput = getTypeElement("org.graalvm.compiler.graph.Node.OptionalInput");
         this.Successor = getTypeElement("org.graalvm.compiler.graph.Node.Successor");
@@ -88,11 +82,7 @@ public class GraphNodeVerifier {
      * @throw {@link NoClassDefFoundError} if a type element does not exist for {@code name}
      */
     public TypeElement getTypeElement(String name) {
-        TypeElement typeElement = elements.getTypeElement(name);
+        return processor.getTypeElement(name);
-        if (typeElement == null) {
-            throw new NoClassDefFoundError(name);
-        }
-        return typeElement;
     }
 
     public TypeElement getTypeElement(Class<?> cls) {
@@ -103,11 +93,8 @@ public class GraphNodeVerifier {
         return getTypeElement(name).asType();
     }
 
-    public ProcessingEnvironment getProcessingEnv() {
-        return env.getProcessingEnv();
-    }
-
     public boolean isAssignableWithErasure(Element from, Element to) {
+        Types types = processor.env().getTypeUtils();
         TypeMirror fromType = types.erasure(from.asType());
         TypeMirror toType = types.erasure(to.asType());
         return types.isAssignable(fromType, toType);
@@ -204,12 +191,12 @@ public class GraphNodeVerifier {
     }
 
     private boolean sameType(TypeMirror type1, TypeMirror type2) {
-        return env.getProcessingEnv().getTypeUtils().isSameType(type1, type2);
+        return processor.env().getTypeUtils().isSameType(type1, type2);
     }
 
     private TypeElement getSuperType(TypeElement element) {
         if (element.getSuperclass() != null) {
-            return (TypeElement) env.getProcessingEnv().getTypeUtils().asElement(element.getSuperclass());
+            return processor.asTypeElement(element.getSuperclass());
         }
         return null;
     }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes.test/src/org/graalvm/compiler/nodes/test/IntegerStampTest.java

@@ -27,7 +27,11 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 
 import java.math.BigInteger;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Stream;
 
+import org.graalvm.compiler.core.common.type.ArithmeticOpTable;
 import org.graalvm.compiler.core.common.type.ArithmeticOpTable.BinaryOp;
 import org.graalvm.compiler.core.common.type.ArithmeticOpTable.IntegerConvertOp;
 import org.graalvm.compiler.core.common.type.ArithmeticOpTable.ShiftOp;
@@ -582,4 +586,58 @@ public class IntegerStampTest extends GraphTest {
         assertEquals(longStamp.join(longEmpty), longEmpty);
         assertEquals(longStamp.meet(longEmpty), longStamp);
     }
+
+    @Test
+    public void testUnaryOpFoldEmpty() {
+        // boolean?, byte, short, int, long
+        Stream.of(1, 8, 16, 32, 64).map(bits -> StampFactory.forInteger(bits).empty()).forEach(empty -> {
+            for (ArithmeticOpTable.UnaryOp<?> op : IntegerStamp.OPS.getUnaryOps()) {
+                if (op != null) {
+                    Assert.assertTrue(op.foldStamp(empty).isEmpty());
+                }
+            }
+        });
+    }
+
+    @Test
+    public void testIntegerConvertOpWithEmpty() {
+        int[] bits = new int[]{1, 8, 16, 32, 64};
+
+        List<IntegerConvertOp<?>> extendOps = Arrays.asList(
+                        IntegerStamp.OPS.getSignExtend(),
+                        IntegerStamp.OPS.getZeroExtend());
+
+        for (int inputBits : bits) {
+            IntegerStamp emptyIn = StampFactory.forInteger(inputBits).empty();
+            for (int outputBits : bits) {
+                IntegerStamp emptyOut = StampFactory.forInteger(outputBits).empty();
+                if (inputBits <= outputBits) {
+                    for (IntegerConvertOp<?> stamp : extendOps) {
+                        IntegerStamp folded = (IntegerStamp) stamp.foldStamp(inputBits, outputBits, emptyIn);
+                        Assert.assertTrue(folded.isEmpty());
+                        Assert.assertEquals(outputBits, folded.getBits());
+
+                        // Widening is lossless, inversion is well-defined.
+                        IntegerStamp inverted = (IntegerStamp) stamp.invertStamp(inputBits, outputBits, emptyOut);
+                        Assert.assertTrue(inverted.isEmpty());
+                        Assert.assertEquals(inputBits, inverted.getBits());
+                    }
+                }
+
+                if (inputBits >= outputBits) {
+                    IntegerConvertOp<?> narrow = IntegerStamp.OPS.getNarrow();
+                    IntegerStamp folded = (IntegerStamp) narrow.foldStamp(inputBits, outputBits, emptyIn);
+                    Assert.assertTrue(folded.isEmpty());
+                    Assert.assertEquals(outputBits, folded.getBits());
+
+                    // Narrowing is lossy, inversion can potentially yield empty or unknown (null).
+                    IntegerStamp inverted = (IntegerStamp) narrow.invertStamp(inputBits, outputBits, emptyOut);
+                    Assert.assertTrue(inverted == null || inverted.isEmpty());
+                    if (inverted != null) {
+                        Assert.assertEquals(inputBits, inverted.getBits());
+                    }
+                }
+            }
+        }
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/AbstractFixedGuardNode.java

@@ -25,6 +25,7 @@ package org.graalvm.compiler.nodes;
 import org.graalvm.compiler.core.common.type.StampFactory;
 import org.graalvm.compiler.debug.DebugCloseable;
 import org.graalvm.compiler.graph.NodeClass;
+import org.graalvm.compiler.graph.NodeSourcePosition;
 import org.graalvm.compiler.graph.spi.Simplifiable;
 import org.graalvm.compiler.graph.spi.SimplifierTool;
 import org.graalvm.compiler.nodeinfo.InputType;
@@ -46,6 +47,7 @@ public abstract class AbstractFixedGuardNode extends DeoptimizingFixedWithNextNo
     protected DeoptimizationAction action;
     protected JavaConstant speculation;
     protected boolean negated;
+    protected NodeSourcePosition noDeoptSuccessorPosition;
 
     @Override
     public LogicNode getCondition() {
@@ -73,6 +75,12 @@ public abstract class AbstractFixedGuardNode extends DeoptimizingFixedWithNextNo
         this.reason = deoptReason;
     }
 
+    protected AbstractFixedGuardNode(NodeClass<? extends AbstractFixedGuardNode> c, LogicNode condition, DeoptimizationReason deoptReason, DeoptimizationAction action, JavaConstant speculation,
+                    boolean negated, NodeSourcePosition noDeoptSuccessorPosition) {
+        this(c, condition, deoptReason, action, speculation, negated);
+        this.noDeoptSuccessorPosition = noDeoptSuccessorPosition;
+    }
+
     @Override
     public DeoptimizationReason getReason() {
         return reason;
@@ -126,6 +134,7 @@ public abstract class AbstractFixedGuardNode extends DeoptimizingFixedWithNextNo
                 ifNode = graph().add(new IfNode(condition, currentNext, deopt, 1));
                 noDeoptSuccessor = ifNode.trueSuccessor();
             }
+            noDeoptSuccessor.setNodeSourcePosition(getNoDeoptSuccessorPosition());
             ((FixedWithNextNode) predecessor()).setNext(ifNode);
             this.replaceAtUsages(noDeoptSuccessor);
             GraphUtil.killWithUnusedFloatingInputs(this);
@@ -148,4 +157,14 @@ public abstract class AbstractFixedGuardNode extends DeoptimizingFixedWithNextNo
     public void setReason(DeoptimizationReason reason) {
         this.reason = reason;
     }
+
+    @Override
+    public NodeSourcePosition getNoDeoptSuccessorPosition() {
+        return noDeoptSuccessorPosition;
+    }
+
+    @Override
+    public void setNoDeoptSuccessorPosition(NodeSourcePosition noDeoptSuccessorPosition) {
+        this.noDeoptSuccessorPosition = noDeoptSuccessorPosition;
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/AbstractMergeNode.java

@@ -28,6 +28,7 @@ import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_0;
 
 import java.util.List;
 
+import org.graalvm.compiler.debug.DebugCloseable;
 import org.graalvm.compiler.graph.IterableNodeType;
 import org.graalvm.compiler.graph.Node;
 import org.graalvm.compiler.graph.NodeClass;
@@ -158,6 +159,7 @@ public abstract class AbstractMergeNode extends BeginStateSplitNode implements I
      * canonicalization.
      */
     @Override
+    @SuppressWarnings("try")
     public void simplify(SimplifierTool tool) {
         FixedNode currentNext = next();
         if (currentNext instanceof AbstractEndNode) {
@@ -190,12 +192,14 @@ public abstract class AbstractMergeNode extends BeginStateSplitNode implements I
                     tool.addToWorkList(end);
                 }
                 AbstractEndNode newEnd;
-                if (merge instanceof LoopBeginNode) {
+                try (DebugCloseable position = end.withNodeSourcePosition()) {
-                    newEnd = graph().add(new LoopEndNode((LoopBeginNode) merge));
+                    if (merge instanceof LoopBeginNode) {
-                } else {
+                        newEnd = graph().add(new LoopEndNode((LoopBeginNode) merge));
-                    EndNode tmpEnd = graph().add(new EndNode());
+                    } else {
-                    merge.addForwardEnd(tmpEnd);
+                        EndNode tmpEnd = graph().add(new EndNode());
-                    newEnd = tmpEnd;
+                        merge.addForwardEnd(tmpEnd);
+                        newEnd = tmpEnd;
+                    }
                 }
                 for (PhiNode phi : merge.phis()) {
                     ValueNode v = phi.valueAt(origLoopEnd);
@@ -233,11 +237,13 @@ public abstract class AbstractMergeNode extends BeginStateSplitNode implements I
             ValuePhiNode returnValuePhi = returnNode.result() == null || !isPhiAtMerge(returnNode.result()) ? null : (ValuePhiNode) returnNode.result();
             List<EndNode> endNodes = forwardEnds().snapshot();
             for (EndNode end : endNodes) {
-                ReturnNode newReturn = graph().add(new ReturnNode(returnValuePhi == null ? returnNode.result() : returnValuePhi.valueAt(end)));
+                try (DebugCloseable position = returnNode.withNodeSourcePosition()) {
-                if (tool != null) {
+                    ReturnNode newReturn = graph().add(new ReturnNode(returnValuePhi == null ? returnNode.result() : returnValuePhi.valueAt(end)));
-                    tool.addToWorkList(end.predecessor());
+                    if (tool != null) {
+                        tool.addToWorkList(end.predecessor());
+                    }
+                    end.replaceAtPredecessor(newReturn);
                 }
-                end.replaceAtPredecessor(newReturn);
             }
             GraphUtil.killCFG(this);
             for (EndNode end : endNodes) {

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/DeoptimizingGuard.java

@@ -22,6 +22,7 @@
  */
 package org.graalvm.compiler.nodes;
 
+import org.graalvm.compiler.graph.NodeSourcePosition;
 import org.graalvm.compiler.nodes.extended.GuardingNode;
 
 /**
@@ -35,4 +36,16 @@ public interface DeoptimizingGuard extends GuardingNode, StaticDeoptimizingNode
     void setCondition(LogicNode x, boolean negated);
 
     boolean isNegated();
+
+    NodeSourcePosition getNoDeoptSuccessorPosition();
+
+    void setNoDeoptSuccessorPosition(NodeSourcePosition noDeoptSuccessorPosition);
+
+    default void addCallerToNoDeoptSuccessorPosition(NodeSourcePosition caller) {
+        NodeSourcePosition noDeoptSuccessorPosition = getNoDeoptSuccessorPosition();
+        if (noDeoptSuccessorPosition == null) {
+            return;
+        }
+        setNoDeoptSuccessorPosition(new NodeSourcePosition(caller, noDeoptSuccessorPosition.getMethod(), noDeoptSuccessorPosition.getBCI()));
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/FixedGuardNode.java

@@ -23,12 +23,14 @@
 package org.graalvm.compiler.nodes;
 
 import org.graalvm.compiler.debug.DebugCloseable;
+
 import static org.graalvm.compiler.nodeinfo.InputType.Guard;
 import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_2;
 import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_2;
 
 import org.graalvm.compiler.graph.IterableNodeType;
 import org.graalvm.compiler.graph.NodeClass;
+import org.graalvm.compiler.graph.NodeSourcePosition;
 import org.graalvm.compiler.graph.spi.SimplifierTool;
 import org.graalvm.compiler.nodeinfo.NodeInfo;
 import org.graalvm.compiler.nodes.spi.Lowerable;
@@ -50,10 +52,18 @@ public final class FixedGuardNode extends AbstractFixedGuardNode implements Lowe
         this(condition, deoptReason, action, JavaConstant.NULL_POINTER, negated);
     }
 
+    public FixedGuardNode(LogicNode condition, DeoptimizationReason deoptReason, DeoptimizationAction action, boolean negated, NodeSourcePosition noDeoptSuccessorPosition) {
+        this(condition, deoptReason, action, JavaConstant.NULL_POINTER, negated, noDeoptSuccessorPosition);
+    }
+
     public FixedGuardNode(LogicNode condition, DeoptimizationReason deoptReason, DeoptimizationAction action, JavaConstant speculation, boolean negated) {
         super(TYPE, condition, deoptReason, action, speculation, negated);
     }
 
+    public FixedGuardNode(LogicNode condition, DeoptimizationReason deoptReason, DeoptimizationAction action, JavaConstant speculation, boolean negated, NodeSourcePosition noDeoptSuccessorPosition) {
+        super(TYPE, condition, deoptReason, action, speculation, negated, noDeoptSuccessorPosition);
+    }
+
     @Override
     public void simplify(SimplifierTool tool) {
         super.simplify(tool);
@@ -75,8 +85,10 @@ public final class FixedGuardNode extends AbstractFixedGuardNode implements Lowe
         } else if (getCondition() instanceof ShortCircuitOrNode) {
             ShortCircuitOrNode shortCircuitOr = (ShortCircuitOrNode) getCondition();
             if (isNegated() && hasNoUsages()) {
-                graph().addAfterFixed(this, graph().add(new FixedGuardNode(shortCircuitOr.getY(), getReason(), getAction(), getSpeculation(), !shortCircuitOr.isYNegated())));
+                graph().addAfterFixed(this,
-                graph().replaceFixedWithFixed(this, graph().add(new FixedGuardNode(shortCircuitOr.getX(), getReason(), getAction(), getSpeculation(), !shortCircuitOr.isXNegated())));
+                                graph().add(new FixedGuardNode(shortCircuitOr.getY(), getReason(), getAction(), getSpeculation(), !shortCircuitOr.isYNegated(), getNoDeoptSuccessorPosition())));
+                graph().replaceFixedWithFixed(this,
+                                graph().add(new FixedGuardNode(shortCircuitOr.getX(), getReason(), getAction(), getSpeculation(), !shortCircuitOr.isXNegated(), getNoDeoptSuccessorPosition())));
             }
         }
     }
@@ -87,7 +99,7 @@ public final class FixedGuardNode extends AbstractFixedGuardNode implements Lowe
         try (DebugCloseable position = this.withNodeSourcePosition()) {
             if (graph().getGuardsStage().allowsFloatingGuards()) {
                 if (getAction() != DeoptimizationAction.None) {
-                    ValueNode guard = tool.createGuard(this, getCondition(), getReason(), getAction(), getSpeculation(), isNegated()).asNode();
+                    ValueNode guard = tool.createGuard(this, getCondition(), getReason(), getAction(), getSpeculation(), isNegated(), getNoDeoptSuccessorPosition()).asNode();
                     this.replaceAtUsages(guard);
                     graph().removeFixed(this);
                 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GraphDecoder.java

@@ -1022,7 +1022,11 @@ public class GraphDecoder {
             }
         }
         if (graph.trackNodeSourcePosition() && position != null) {
-            node.setNodeSourcePosition(methodScope.getCallerBytecodePosition(position));
+            NodeSourcePosition callerBytecodePosition = methodScope.getCallerBytecodePosition(position);
+            node.setNodeSourcePosition(callerBytecodePosition);
+            if (node instanceof DeoptimizingGuard) {
+                ((DeoptimizingGuard) node).addCallerToNoDeoptSuccessorPosition(callerBytecodePosition.getCaller());
+            }
         }
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GuardNode.java

@@ -31,6 +31,7 @@ import org.graalvm.compiler.core.common.type.StampFactory;
 import org.graalvm.compiler.graph.IterableNodeType;
 import org.graalvm.compiler.graph.Node;
 import org.graalvm.compiler.graph.NodeClass;
+import org.graalvm.compiler.graph.NodeSourcePosition;
 import org.graalvm.compiler.graph.spi.Canonicalizable;
 import org.graalvm.compiler.graph.spi.CanonicalizerTool;
 import org.graalvm.compiler.nodeinfo.NodeInfo;
@@ -63,19 +64,22 @@ public class GuardNode extends FloatingAnchoredNode implements Canonicalizable,
     protected DeoptimizationAction action;
     protected JavaConstant speculation;
     protected boolean negated;
+    protected NodeSourcePosition noDeoptSuccessorPosition;
 
-    public GuardNode(LogicNode condition, AnchoringNode anchor, DeoptimizationReason reason, DeoptimizationAction action, boolean negated, JavaConstant speculation) {
+    public GuardNode(LogicNode condition, AnchoringNode anchor, DeoptimizationReason reason, DeoptimizationAction action, boolean negated, JavaConstant speculation,
-        this(TYPE, condition, anchor, reason, action, negated, speculation);
+                    NodeSourcePosition noDeoptSuccessorPosition) {
+        this(TYPE, condition, anchor, reason, action, negated, speculation, noDeoptSuccessorPosition);
     }
 
     protected GuardNode(NodeClass<? extends GuardNode> c, LogicNode condition, AnchoringNode anchor, DeoptimizationReason reason, DeoptimizationAction action, boolean negated,
-                    JavaConstant speculation) {
+                    JavaConstant speculation, NodeSourcePosition noDeoptSuccessorPosition) {
         super(c, StampFactory.forVoid(), anchor);
         this.condition = condition;
         this.reason = reason;
         this.action = action;
         this.negated = negated;
         this.speculation = speculation;
+        this.noDeoptSuccessorPosition = noDeoptSuccessorPosition;
     }
 
     /**
@@ -130,7 +134,7 @@ public class GuardNode extends FloatingAnchoredNode implements Canonicalizable,
     public Node canonical(CanonicalizerTool tool) {
         if (getCondition() instanceof LogicNegationNode) {
             LogicNegationNode negation = (LogicNegationNode) getCondition();
-            return new GuardNode(negation.getValue(), getAnchor(), reason, action, !negated, speculation);
+            return new GuardNode(negation.getValue(), getAnchor(), reason, action, !negated, speculation, noDeoptSuccessorPosition);
         }
         if (getCondition() instanceof LogicConstantNode) {
             LogicConstantNode c = (LogicConstantNode) getCondition();
@@ -158,4 +162,14 @@ public class GuardNode extends FloatingAnchoredNode implements Canonicalizable,
     public void setReason(DeoptimizationReason reason) {
         this.reason = reason;
     }
+
+    @Override
+    public NodeSourcePosition getNoDeoptSuccessorPosition() {
+        return noDeoptSuccessorPosition;
+    }
+
+    @Override
+    public void setNoDeoptSuccessorPosition(NodeSourcePosition noDeoptSuccessorPosition) {
+        this.noDeoptSuccessorPosition = noDeoptSuccessorPosition;
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/IfNode.java

@@ -29,18 +29,25 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Objects;
 
 import jdk.internal.vm.compiler.collections.EconomicMap;
 import jdk.internal.vm.compiler.collections.Equivalence;
+import org.graalvm.compiler.bytecode.BytecodeDisassembler;
+import org.graalvm.compiler.bytecode.Bytecodes;
+import org.graalvm.compiler.bytecode.Bytes;
+import org.graalvm.compiler.bytecode.ResolvedJavaMethodBytecode;
 import org.graalvm.compiler.core.common.calc.Condition;
 import org.graalvm.compiler.core.common.type.IntegerStamp;
 import org.graalvm.compiler.core.common.type.Stamp;
 import org.graalvm.compiler.core.common.type.StampFactory;
 import org.graalvm.compiler.debug.CounterKey;
+import org.graalvm.compiler.debug.DebugCloseable;
 import org.graalvm.compiler.debug.DebugContext;
 import org.graalvm.compiler.debug.GraalError;
 import org.graalvm.compiler.graph.Node;
 import org.graalvm.compiler.graph.NodeClass;
+import org.graalvm.compiler.graph.NodeSourcePosition;
 import org.graalvm.compiler.graph.iterators.NodeIterable;
 import org.graalvm.compiler.graph.spi.Canonicalizable;
 import org.graalvm.compiler.graph.spi.Simplifiable;
@@ -67,6 +74,7 @@ import jdk.vm.ci.meta.JavaConstant;
 import jdk.vm.ci.meta.JavaKind;
 import jdk.vm.ci.meta.MetaAccessProvider;
 import jdk.vm.ci.meta.PrimitiveConstant;
+import jdk.vm.ci.meta.ResolvedJavaMethod;
 import jdk.vm.ci.meta.ResolvedJavaType;
 
 /**
@@ -170,6 +178,60 @@ public final class IfNode extends ControlSplitNode implements Simplifiable, LIRL
         return super.verify();
     }
 
+    private boolean compareCallContext(NodeSourcePosition successorPosition) {
+        NodeSourcePosition position = getNodeSourcePosition();
+        NodeSourcePosition successor = successorPosition;
+        while (position != null) {
+            assertTrue(Objects.equals(position.getMethod(), successor.getMethod()), "method mismatch");
+            position = position.getCaller();
+            successor = successor.getCaller();
+        }
+        assertTrue(successor == null, "successor position has more methods");
+        return true;
+    }
+
+    @Override
+    public boolean verifySourcePosition() {
+        NodeSourcePosition sourcePosition = getNodeSourcePosition();
+        assertTrue(sourcePosition != null, "missing IfNode source position");
+
+        NodeSourcePosition trueSuccessorPosition = trueSuccessor.getNodeSourcePosition();
+        assertTrue(trueSuccessorPosition != null, "missing IfNode true successor source position");
+
+        NodeSourcePosition falseSuccessorPosition = falseSuccessor.getNodeSourcePosition();
+        assertTrue(falseSuccessorPosition != null, "missing IfNode false successor source position");
+
+        int bci = sourcePosition.getBCI();
+        ResolvedJavaMethod method = sourcePosition.getMethod();
+        int bytecode = BytecodeDisassembler.getBytecodeAt(method, bci);
+
+        if (!Bytecodes.isIfBytecode(bytecode)) {
+            return true;
+        }
+
+        byte[] code = (new ResolvedJavaMethodBytecode(method)).getCode();
+        int targetBCI = bci + Bytes.beS2(code, bci + 1);
+        int nextBCI = bci + Bytecodes.lengthOf(bytecode);
+
+        // At least one successor should have the correct BCI to indicate any possible negation that
+        // occurred after bytecode parsing
+        boolean matchingSuccessorFound = false;
+        if (trueSuccessorPosition.getBCI() == nextBCI || trueSuccessorPosition.getBCI() == targetBCI) {
+            assertTrue(compareCallContext(trueSuccessorPosition), "call context different from IfNode in trueSuccessor");
+            matchingSuccessorFound = true;
+        }
+
+        if (falseSuccessorPosition.getBCI() == nextBCI || falseSuccessorPosition.getBCI() == targetBCI) {
+            assertTrue(compareCallContext(falseSuccessorPosition), "call context different from IfNode in falseSuccessor");
+            matchingSuccessorFound = true;
+        }
+
+        assertTrue(matchingSuccessorFound, "no matching successor position found in IfNode");
+        assertTrue(trueSuccessorPosition.getBCI() != falseSuccessorPosition.getBCI(), "successor positions same in IfNode");
+
+        return true;
+    }
+
     public void eliminateNegation() {
         AbstractBeginNode oldTrueSuccessor = trueSuccessor;
         AbstractBeginNode oldFalseSuccessor = falseSuccessor;
@@ -249,6 +311,11 @@ public final class IfNode extends ControlSplitNode implements Simplifiable, LIRL
                     nextIf.setFalseSuccessor(intermediateBegin);
                     intermediateBegin.setNext(this);
                     this.setFalseSuccessor(bothFalseBegin);
+
+                    NodeSourcePosition intermediateBeginPosition = intermediateBegin.getNodeSourcePosition();
+                    intermediateBegin.setNodeSourcePosition(bothFalseBegin.getNodeSourcePosition());
+                    bothFalseBegin.setNodeSourcePosition(intermediateBeginPosition);
+
                     nextIf.setTrueSuccessorProbability(probabilityB);
                     if (probabilityB == 1.0) {
                         this.setTrueSuccessorProbability(0.0);
@@ -477,6 +544,7 @@ public final class IfNode extends ControlSplitNode implements Simplifiable, LIRL
      * @param tool
      * @return true if a replacement was done.
      */
+    @SuppressWarnings("try")
     private boolean checkForUnsignedCompare(SimplifierTool tool) {
         assert trueSuccessor().hasNoUsages() && falseSuccessor().hasNoUsages();
         if (condition() instanceof IntegerLessThanNode) {
@@ -516,18 +584,20 @@ public final class IfNode extends ControlSplitNode implements Simplifiable, LIRL
                         }
                     }
                     if (below != null) {
-                        ifNode2.setTrueSuccessor(null);
+                        try (DebugCloseable position = ifNode2.withNodeSourcePosition()) {
-                        ifNode2.setFalseSuccessor(null);
+                            ifNode2.setTrueSuccessor(null);
+                            ifNode2.setFalseSuccessor(null);
 
-                        IfNode newIfNode = graph().add(new IfNode(below, falseSucc, trueSucc, 1 - trueSuccessorProbability));
+                            IfNode newIfNode = graph().add(new IfNode(below, falseSucc, trueSucc, 1 - trueSuccessorProbability));
-                        // Remove the < 0 test.
+                            // Remove the < 0 test.
-                        tool.deleteBranch(trueSuccessor);
+                            tool.deleteBranch(trueSuccessor);
-                        graph().removeSplit(this, falseSuccessor);
+                            graph().removeSplit(this, falseSuccessor);
 
-                        // Replace the second test with the new one.
+                            // Replace the second test with the new one.
-                        ifNode2.predecessor().replaceFirstSuccessor(ifNode2, newIfNode);
+                            ifNode2.predecessor().replaceFirstSuccessor(ifNode2, newIfNode);
-                        ifNode2.safeDelete();
+                            ifNode2.safeDelete();
-                        return true;
+                            return true;
+                        }
                     }
                 }
             }
@@ -850,6 +920,7 @@ public final class IfNode extends ControlSplitNode implements Simplifiable, LIRL
      *
      * @param tool
      */
+    @SuppressWarnings("try")
     private boolean splitIfAtPhi(SimplifierTool tool) {
         if (graph().getGuardsStage().areFrameStatesAtSideEffects()) {
             // Disabled until we make sure we have no FrameState-less merges at this stage
@@ -918,12 +989,16 @@ public final class IfNode extends ControlSplitNode implements Simplifiable, LIRL
             } else if (result != condition) {
                 // Build a new IfNode using the new condition
                 BeginNode trueBegin = graph().add(new BeginNode());
+                trueBegin.setNodeSourcePosition(trueSuccessor().getNodeSourcePosition());
                 BeginNode falseBegin = graph().add(new BeginNode());
+                falseBegin.setNodeSourcePosition(falseSuccessor().getNodeSourcePosition());
 
                 if (result.graph() == null) {
                     result = graph().addOrUniqueWithInputs(result);
+                    result.setNodeSourcePosition(condition.getNodeSourcePosition());
                 }
                 IfNode newIfNode = graph().add(new IfNode(result, trueBegin, falseBegin, trueSuccessorProbability));
+                newIfNode.setNodeSourcePosition(getNodeSourcePosition());
                 merge.removeEnd(end);
                 ((FixedWithNextNode) end.predecessor()).setNext(newIfNode);
 
@@ -1053,6 +1128,7 @@ public final class IfNode extends ControlSplitNode implements Simplifiable, LIRL
         }
     }
 
+    @SuppressWarnings("try")
     private MergeNode insertMerge(AbstractBeginNode begin) {
         MergeNode merge = graph().add(new MergeNode());
         if (!begin.anchored().isEmpty()) {
@@ -1066,9 +1142,11 @@ public final class IfNode extends ControlSplitNode implements Simplifiable, LIRL
         AbstractBeginNode theBegin = begin;
         if (begin instanceof LoopExitNode) {
             // Insert an extra begin to make it easier.
-            theBegin = graph().add(new BeginNode());
+            try (DebugCloseable position = begin.withNodeSourcePosition()) {
-            begin.replaceAtPredecessor(theBegin);
+                theBegin = graph().add(new BeginNode());
-            theBegin.setNext(begin);
+                begin.replaceAtPredecessor(theBegin);
+                theBegin.setNext(begin);
+            }
         }
         FixedNode next = theBegin.next();
         next.replaceAtPredecessor(merge);

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/InliningLog.java

@@ -225,9 +225,11 @@ public class InliningLog {
         while (replacementEntries.advance()) {
             Invokable replacementInvoke = replacementEntries.getKey();
             Callsite replacementSite = replacementEntries.getValue();
-            Invokable invoke = (Invokable) replacements.get((Node) replacementInvoke);
+            if (replacementInvoke.isAlive()) {
-            Callsite site = mapping.get(replacementSite);
+                Invokable invoke = (Invokable) replacements.get((Node) replacementInvoke);
-            leaves.put(invoke, site);
+                Callsite site = mapping.get(replacementSite);
+                leaves.put(invoke, site);
+            }
         }
     }
 

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LogicNegationNode.java

@@ -31,6 +31,8 @@ import org.graalvm.compiler.graph.spi.Canonicalizable;
 import org.graalvm.compiler.graph.spi.CanonicalizerTool;
 import org.graalvm.compiler.nodeinfo.NodeInfo;
 
+import jdk.vm.ci.meta.TriState;
+
 /**
  * Logic node that negates its argument.
  */
@@ -77,4 +79,11 @@ public final class LogicNegationNode extends LogicNode implements Canonicalizabl
         return this;
     }
 
+    @Override
+    public TriState implies(boolean thisNegated, LogicNode other) {
+        if (other == getValue()) {
+            return TriState.get(thisNegated);
+        }
+        return getValue().implies(!thisNegated, other);
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LogicNode.java

@@ -25,6 +25,7 @@ package org.graalvm.compiler.nodes;
 import static org.graalvm.compiler.nodeinfo.InputType.Condition;
 import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_1;
 
+import jdk.vm.ci.meta.TriState;
 import org.graalvm.compiler.core.common.type.StampFactory;
 import org.graalvm.compiler.graph.Node.IndirectCanonicalization;
 import org.graalvm.compiler.graph.NodeClass;
@@ -75,4 +76,19 @@ public abstract class LogicNode extends FloatingNode implements IndirectCanonica
 
         return false;
     }
+
+    /**
+     * Determines what this condition implies about the other.
+     *
+     * <ul>
+     * <li>If negate(this, thisNegated) => other, returns {@link TriState#TRUE}</li>
+     * <li>If negate(this, thisNegated) => !other, returns {@link TriState#FALSE}</li>
+     * </ul>
+     *
+     * @param thisNegated whether this condition should be considered as false.
+     * @param other the other condition.
+     */
+    public TriState implies(boolean thisNegated, LogicNode other) {
+        return TriState.UNKNOWN;
+    }
 }

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LoopBeginNode.java

@@ -25,6 +25,7 @@ package org.graalvm.compiler.nodes;
 import static org.graalvm.compiler.graph.iterators.NodePredicates.isNotA;
 
 import org.graalvm.compiler.core.common.type.IntegerStamp;
+import org.graalvm.compiler.debug.DebugCloseable;
 import org.graalvm.compiler.graph.IterableNodeType;
 import org.graalvm.compiler.graph.Node;
 import org.graalvm.compiler.graph.NodeClass;
@@ -310,13 +311,16 @@ public final class LoopBeginNode extends AbstractMergeNode implements IterableNo
         return loopEnds().first();
     }
 
+    @SuppressWarnings("try")
     public void removeExits() {
         for (LoopExitNode loopexit : loopExits().snapshot()) {
-            loopexit.removeProxies();
+            try (DebugCloseable position = graph().withNodeSourcePosition(loopexit)) {
-            FrameState loopStateAfter = loopexit.stateAfter();
+                loopexit.removeProxies();
-            graph().replaceFixedWithFixed(loopexit, graph().add(new BeginNode()));
+                FrameState loopStateAfter = loopexit.stateAfter();
-            if (loopStateAfter != null) {
+                graph().replaceFixedWithFixed(loopexit, graph().add(new BeginNode()));
-                GraphUtil.tryKillUnused(loopStateAfter);
+                if (loopStateAfter != null) {
+                    GraphUtil.tryKillUnused(loopStateAfter);
+                }
             }
         }
     }