From c7219faf86821589cd9fc1d8d5cf06a7028f11d2 Mon Sep 17 00:00:00 2001 From: Jesper Wilhelmsson Date: Tue, 12 Mar 2019 19:17:42 +0100 Subject: [PATCH] 8218074: Update Graal Reviewed-by: kvn --- make/CompileJavaModules.gmk | 1 + make/test/JtregGraalUnit.gmk | 1 + .../src/jdk/tools/jaotc/Linker.java | 68 +- .../src/jdk/tools/jaotc/Main.java | 2 +- .../jdk/tools/jaotc/collect/ClassSearch.java | 5 +- .../asm/aarch64/AArch64Assembler.java | 20 +- .../asm/aarch64/AArch64MacroAssembler.java | 10 +- .../compiler/asm/amd64/AMD64Assembler.java | 109 +- .../asm/amd64/AMD64BaseAssembler.java | 28 +- .../compiler/asm/sparc/SPARCAssembler.java | 4 +- .../compiler/asm/test/AssemblerTest.java | 3 +- .../BranchTargetOutOfBoundsException.java} | 38 +- .../compiler/code/CompilationResult.java | 34 +- .../code/HexCodeFileDisassemblerProvider.java | 2 +- .../core/aarch64/test/AArch64CbzTest.java | 127 + .../aarch64/test/AArch64MembarOpTest.java | 130 + .../test/AArch64TestBitAndBranchTest.java | 190 + .../AArch64ArithmeticLIRGenerator.java | 20 - .../core/aarch64/AArch64LIRGenerator.java | 22 + .../core/aarch64/AArch64NodeMatchRules.java | 25 + .../amd64/AMD64ArithmeticLIRGenerator.java | 101 +- .../core/amd64/AMD64LIRGenerator.java | 28 +- .../compiler/core/common/GraalOptions.java | 9 +- ...peculativeExecutionAttacksMitigations.java | 4 +- .../core/common/cfg/AbstractBlockBase.java | 11 + .../compiler/core/common/cfg/CFGVerifier.java | 2 +- .../compiler/core/common/cfg/Loop.java | 102 +- ...itigateSpeculativeExecutionAttacksHelp.txt | 9 + .../core/common/type/AbstractObjectStamp.java | 7 + .../common/type/AbstractPointerStamp.java | 6 + .../core/common/type/IllegalStamp.java | 4 + .../core/common/type/ObjectStamp.java | 45 + .../core/common/type/PrimitiveStamp.java | 5 + .../compiler/core/common/type/Stamp.java | 14 +- .../compiler/core/common/type/StampPair.java | 20 + .../common/type/SymbolicJVMCIReference.java | 36 + .../compiler/core/common/type/VoidStamp.java | 4 + .../core/common/util/UnsignedLong.java | 4 + .../core/test/CheckGraalInvariants.java | 4 + .../test/ConditionalEliminationTestBase.java | 2 +- .../compiler/core/test/CountedLoopTest2.java | 130 + .../compiler/core/test/DumpPathTest.java | 3 +- .../compiler/core/test/GraalCompilerTest.java | 14 +- .../compiler/core/test/GraphEncoderTest.java | 9 +- .../core/test/GuardPrioritiesTest.java | 2 +- .../core/test/IntegerDivRemConstantTest.java | 293 ++ .../core/test/MemoryScheduleTest.java | 2 +- .../compiler/core/test/NestedLoopTest.java | 6 +- .../core/test/SwitchCanonicalizerTest.java | 126 + .../test/VerifyCallerSensitiveMethods.java | 2 +- .../compiler/core/test/VerifyDebugUsage.java | 6 +- .../core/test/backend/BackendTest.java | 3 +- .../deopt/RethrowDeoptMaterializeTest.java | 2 +- .../core/test/ea/EscapeAnalysisTest.java | 2 +- .../graalvm/compiler/core/GraalCompiler.java | 183 +- .../compiler/core/gen/LIRCompilerBackend.java | 245 ++ .../core/gen/LIRGenerationProvider.java | 65 + .../compiler/core/gen/NodeLIRBuilder.java | 9 + .../compiler/core/phases/HighTier.java | 2 +- .../graalvm/compiler/core/phases/LowTier.java | 3 + .../graalvm/compiler/core/target/Backend.java | 112 +- .../graalvm/compiler/debug/DebugContext.java | 7 +- .../graalvm/compiler/debug/DebugOptions.java | 60 +- .../compiler/debug/IgvDumpChannel.java | 58 +- .../graalvm/compiler/debug/PathUtilities.java | 4 +- .../debug/doc-files/PrintGraphHelp.txt | 6 + .../graph/test/graphio/GraphOutputTest.java | 284 ++ .../org/graalvm/compiler/graph/NodeClass.java | 2 - .../aarch64/AArch64HotSpotBackend.java | 45 +- ...tSpotJumpToExceptionHandlerInCallerOp.java | 4 +- .../amd64/test/BinaryMathStubTest.java | 78 + .../hotspot/amd64/test/UnaryMathStubTest.java | 101 + .../amd64/AMD64HotSpotAddressLowering.java | 6 +- .../hotspot/amd64/AMD64HotSpotBackend.java | 33 +- .../amd64/AMD64HotSpotBackendFactory.java | 7 +- .../AMD64HotSpotForeignCallsProvider.java | 40 +- ...tSpotJumpToExceptionHandlerInCallerOp.java | 6 +- .../amd64/AMD64HotSpotLIRGenerator.java | 2 +- .../amd64/AMD64HotSpotLoweringProvider.java | 81 +- .../hotspot/amd64/AMD64HotSpotMaths.java | 88 - .../compiler/hotspot/amd64/AMD64MathStub.java | 59 +- .../amd64/AMD64X87MathIntrinsicNode.java | 111 + .../hotspot/amd64/AMD64X87MathSnippets.java | 117 + .../test/StringUTF16ToBytesGetCharsTest.java | 142 + .../test/BenchmarkCounterOverflowTest.java | 33 +- .../hotspot/sparc/SPARCHotSpotBackend.java | 26 +- .../hotspot/sparc/SPARCHotSpotMove.java | 42 +- .../hotspot/test/CheckGraalIntrinsics.java | 81 +- .../hotspot/test/CompileTheWorld.java | 6 +- ...inateRedundantInitializationPhaseTest.java | 224 + .../compiler/hotspot/test/GraalOSRTest.java | 15 + .../hotspot/test/JVMCIInfopointErrorTest.java | 14 +- .../hotspot/test/OptionsInFileTest.java | 7 +- .../hotspot/test/ReservedStackAccessTest.java | 8 +- .../test/WriteBarrierAdditionTest.java | 8 +- .../test/WriteBarrierVerificationTest.java | 12 +- .../compiler/hotspot/CompilationTask.java | 10 +- .../hotspot/GraalHotSpotVMConfig.java | 50 +- .../compiler/hotspot/HotSpotBackend.java | 74 +- .../hotspot/HotSpotCompiledCodeBuilder.java | 10 +- .../compiler/hotspot/HotSpotDataBuilder.java | 8 +- .../hotspot/HotSpotGraalCompilerFactory.java | 7 +- .../HotSpotGraalJVMCIServiceLocator.java | 3 +- .../hotspot/HotSpotGraalOptionValues.java | 19 +- .../compiler/hotspot/HotSpotGraalRuntime.java | 14 +- .../hotspot/HotSpotReplacementsImpl.java | 149 +- .../hotspot/HotSpotTTYStreamProvider.java | 4 +- .../compiler/hotspot/JVMCIVersionCheck.java | 3 +- .../hotspot/SymbolicSnippetEncoder.java | 1034 +++++ .../g1}/G1ArrayRangePostWriteBarrier.java | 6 +- .../g1}/G1ArrayRangePreWriteBarrier.java | 6 +- .../compiler/hotspot/gc/g1/G1BarrierSet.java | 135 + .../{nodes => gc/g1}/G1PostWriteBarrier.java | 7 +- .../{nodes => gc/g1}/G1PreWriteBarrier.java | 12 +- .../g1}/G1ReferentFieldReadBarrier.java | 7 +- .../shared}/ArrayRangeWriteBarrier.java | 5 +- .../hotspot/gc/shared/BarrierSet.java | 45 + .../gc/shared/CardTableBarrierSet.java | 113 + .../shared}/ObjectWriteBarrier.java | 3 +- .../shared}/SerialArrayRangeWriteBarrier.java | 2 +- .../shared}/SerialWriteBarrier.java | 2 +- .../meta/DefaultHotSpotLoweringProvider.java | 23 +- .../HotSpotClassInitializationPlugin.java | 11 +- .../HotSpotGraalConstantFieldProvider.java | 4 +- .../meta/HotSpotGraphBuilderPlugins.java | 84 +- .../meta/HotSpotHostForeignCallsProvider.java | 90 +- .../hotspot/meta/HotSpotNodePlugin.java | 9 +- .../meta/HotSpotUnsafeSubstitutions.java | 2 +- .../meta/HotSpotWordOperationPlugin.java | 2 +- .../compiler/hotspot/nodes/AllocaNode.java | 2 +- .../hotspot/nodes/DimensionsNode.java | 2 +- .../hotspot/nodes/type/KlassPointerStamp.java | 7 + .../phases/OnStackReplacementPhase.java | 40 +- .../phases/WriteBarrierAdditionPhase.java | 161 +- .../phases/WriteBarrierVerificationPhase.java | 8 +- .../ArraysSupportSubstitutions.java | 47 + .../CounterModeSubstitutions.java | 93 + .../replacements/DigestBaseSubstitutions.java | 93 + .../HotSpotClassSubstitutions.java | 13 +- .../replacements/HotSpotReplacementsUtil.java | 17 +- .../replacements/NewObjectSnippets.java | 39 +- .../replacements/SHA2Substitutions.java | 2 +- .../replacements/SHA5Substitutions.java | 12 +- .../replacements/SHASubstitutions.java | 12 +- .../StringUTF16Substitutions.java | 78 + .../replacements/WriteBarrierSnippets.java | 14 +- .../stubs/OutOfBoundsExceptionStub.java | 4 +- .../graalvm/compiler/hotspot/stubs/Stub.java | 3 +- .../graalvm/compiler/java/BytecodeParser.java | 116 +- .../optimize/IfNodeCanonicalizationsTest.java | 366 ++ .../jtt/optimize/SwitchHashTableTest.java | 635 +++ .../compiler/lir/aarch64/AArch64Call.java | 8 +- .../lir/aarch64/AArch64ControlFlow.java | 137 +- .../compiler/lir/aarch64/AArch64Move.java | 37 +- .../lir/amd64/AMD64ArrayEqualsOp.java | 712 +-- .../compiler/lir/amd64/AMD64ControlFlow.java | 101 +- .../lir/amd64/AMD64HotSpotHelper.java | 65 + .../compiler/lir/amd64/AMD64MathCosOp.java | 869 ++++ .../compiler/lir/amd64/AMD64MathExpOp.java | 459 ++ .../lir/amd64/AMD64MathIntrinsicBinaryOp.java | 2063 +-------- .../lir/amd64/AMD64MathIntrinsicUnaryOp.java | 3826 +---------------- .../compiler/lir/amd64/AMD64MathLog10Op.java | 424 ++ .../compiler/lir/amd64/AMD64MathLogOp.java | 402 ++ .../compiler/lir/amd64/AMD64MathPowOp.java | 1979 +++++++++ .../compiler/lir/amd64/AMD64MathSinOp.java | 902 ++++ .../compiler/lir/amd64/AMD64MathTanOp.java | 1119 +++++ .../graalvm/compiler/lir/amd64/AMD64Move.java | 2 +- .../lir/jtt/SPARCBranchBailoutTest.java | 16 +- .../src/org/graalvm/compiler/lir/LIR.java | 9 +- .../graalvm/compiler/lir/LIRValueUtil.java | 5 + .../org/graalvm/compiler/lir/StandardOp.java | 7 +- .../org/graalvm/compiler/lir/Variable.java | 2 +- .../lsra/LinearScanLifetimeAnalysisPhase.java | 14 +- .../SSALinearScanLifetimeAnalysisPhase.java | 41 +- .../lir/asm/CompilationResultBuilder.java | 85 +- .../graalvm/compiler/lir/asm/DataBuilder.java | 10 +- .../lir/gen/ArithmeticLIRGenerator.java | 7 +- .../compiler/lir/gen/LIRGenerator.java | 60 +- .../compiler/lir/gen/LIRGeneratorTool.java | 27 + .../compiler/lir/hashing/HashFunction.java | 156 + .../graalvm/compiler/lir/hashing/Hasher.java | 161 + .../org/graalvm/compiler/lir/ssa/SSAUtil.java | 51 +- .../ConvertDeoptimizeToGuardPhase.java | 118 +- .../loop/phases/LoopTransformations.java | 27 +- .../compiler/loop/BasicInductionVariable.java | 4 +- .../compiler/loop/CountedLoopInfo.java | 80 +- .../compiler/loop/DefaultLoopPolicies.java | 36 +- .../DerivedConvertedInductionVariable.java | 4 +- .../loop/DerivedOffsetInductionVariable.java | 4 +- .../loop/DerivedScaledInductionVariable.java | 4 +- .../compiler/loop/InductionVariable.java | 2 +- .../src/org/graalvm/compiler/loop/LoopEx.java | 24 +- .../graalvm/compiler/loop/LoopFragment.java | 180 +- .../compiler/loop/LoopFragmentInside.java | 5 - .../compiler/loop/LoopFragmentWhole.java | 46 +- .../org/graalvm/compiler/loop/MathUtil.java | 4 +- .../lir/GraalCompilerState.java | 18 +- .../test/IfNodeCanonicalizationTest.java | 2 +- .../nodes/AbstractFixedGuardNode.java | 8 + .../nodes/ComputeObjectAddressNode.java | 3 +- .../graalvm/compiler/nodes/ConstantNode.java | 8 + .../graalvm/compiler/nodes/EncodedGraph.java | 2 +- .../graalvm/compiler/nodes/FrameState.java | 4 + .../graalvm/compiler/nodes/GraphDecoder.java | 3 +- .../graalvm/compiler/nodes/GraphEncoder.java | 31 +- .../org/graalvm/compiler/nodes/IfNode.java | 69 +- .../org/graalvm/compiler/nodes/LogicNode.java | 12 + .../graalvm/compiler/nodes/LoopBeginNode.java | 21 +- .../graalvm/compiler/nodes/LoopExitNode.java | 10 + .../org/graalvm/compiler/nodes/PiNode.java | 23 +- .../nodes/calc/BinaryArithmeticNode.java | 32 + .../nodes/calc/IntegerDivRemNode.java | 2 +- .../nodes/calc/IntegerEqualsNode.java | 77 + .../nodes/calc/IntegerLessThanNode.java | 27 +- .../nodes/calc}/IntegerMulHighNode.java | 5 +- .../compiler/nodes/calc/IsNullNode.java | 7 - .../compiler/nodes/calc/RightShiftNode.java | 19 + .../compiler/nodes/calc/SignedDivNode.java | 6 +- .../compiler/nodes/calc/SignedRemNode.java | 58 +- .../nodes/calc/UnsignedRightShiftNode.java | 16 + .../org/graalvm/compiler/nodes/cfg/Block.java | 17 - .../compiler/nodes/cfg/ControlFlowGraph.java | 74 +- .../compiler/nodes/extended/JavaReadNode.java | 7 +- .../GeneratedInvocationPlugin.java | 8 + .../GraphBuilderConfiguration.java | 133 +- .../graphbuilderconf/GraphBuilderContext.java | 31 +- .../graphbuilderconf/InvocationPlugins.java | 79 +- .../MethodSubstitutionPlugin.java | 5 + .../nodes/graphbuilderconf/NodePlugin.java | 3 + .../compiler/nodes/java/NewArrayNode.java | 2 +- .../memory/address/IndexAddressNode.java | 82 + .../nodes/spi/NodeLIRBuilderTool.java | 2 + .../compiler/nodes/spi/Replacements.java | 4 +- .../compiler/nodes/type/NarrowOopStamp.java | 7 + .../compiler/nodes/util/GraphUtil.java | 2 +- .../options/processor/OptionProcessor.java | 2 +- .../compiler/options/OptionDescriptor.java | 16 +- .../graalvm/compiler/options/OptionKey.java | 12 + .../compiler/options/OptionValues.java | 2 +- .../compiler/options/OptionsParser.java | 23 +- .../compiler/phases/common/FixReadsPhase.java | 104 +- .../phases/common/OptimizeDivPhase.java | 208 + .../phases/common/RemoveValueProxyPhase.java | 16 +- .../phases/graph/ReentrantBlockIterator.java | 8 +- .../compiler/phases/util/GraphOrder.java | 4 +- .../printer/GraphPrinterDumpHandler.java | 3 +- .../aarch64/AArch64GraphBuilderPlugins.java | 6 +- .../amd64/AMD64GraphBuilderPlugins.java | 59 +- .../amd64/AMD64MathSubstitutions.java | 85 - .../amd64/AMD64StringLatin1Substitutions.java | 66 +- .../amd64/AMD64StringSubstitutions.java | 1 - .../amd64/AMD64StringUTF16Substitutions.java | 122 +- .../processor/GeneratedPlugin.java | 6 +- .../test/DeoptimizeOnIntegerExactTest.java | 3 +- .../test/IntegerExactFoldTest.java | 19 +- .../test/ReplacementsParseTest.java | 38 +- .../test/StringCompareToTest.java | 16 +- .../test/StringIndexOfTestBase.java | 24 + .../ClassfileBytecodeProviderTest.java | 14 +- .../replacements/CachingPEGraphDecoder.java | 2 +- .../DefaultJavaLoweringProvider.java | 66 +- .../compiler/replacements/GraphKit.java | 20 +- .../replacements/IntrinsicGraphBuilder.java | 3 +- .../compiler/replacements/PEGraphDecoder.java | 26 +- .../replacements/ReplacementsImpl.java | 17 +- .../compiler/replacements/SnippetCounter.java | 7 +- .../replacements/SnippetIntegerHistogram.java | 2 + .../replacements/SnippetTemplate.java | 6 +- .../StandardGraphBuilderPlugins.java | 138 +- .../StringUTF16Substitutions.java | 62 + .../arraycopy/ArrayCopyCallNode.java | 9 +- .../replacements/arraycopy/ArrayCopyNode.java | 21 +- .../arraycopy/ArrayCopySnippets.java | 27 +- .../nodes/ArrayCompareToNode.java | 2 +- .../nodes/ArrayRegionEqualsNode.java | 23 +- .../nodes/BasicArrayCopyNode.java | 7 +- .../nodes/BasicObjectCloneNode.java | 2 +- .../nodes/BinaryMathIntrinsicNode.java | 6 +- .../replacements/nodes/MacroNode.java | 17 +- .../nodes/UnaryMathIntrinsicNode.java | 82 +- .../nodes/arithmetic/IntegerAddExactNode.java | 75 +- .../IntegerAddExactOverflowNode.java | 101 + .../IntegerExactArithmeticNode.java | 19 +- .../IntegerExactArithmeticSplitNode.java | 27 - .../arithmetic/IntegerExactOverflowNode.java | 124 + .../nodes/arithmetic/IntegerMulExactNode.java | 50 +- .../IntegerMulExactOverflowNode.java | 100 + .../nodes/arithmetic/IntegerSubExactNode.java | 42 +- .../IntegerSubExactOverflowNode.java | 102 + .../serviceprovider/GraalServices.java | 64 +- .../serviceprovider/JavaVersionUtil.java | 58 + .../SpeculationReasonGroup.java | 169 + .../org/graalvm/compiler/test/GraalTest.java | 5 +- .../graalvm/compiler/test/SubprocessUtil.java | 4 +- .../virtual/phases/ea/EffectsClosure.java | 8 +- .../src/org/graalvm/graphio/GraphOutput.java | 47 +- .../org/graalvm/graphio/GraphProtocol.java | 107 +- .../src/org/graalvm/graphio/ProtocolImpl.java | 4 +- .../graalvm/util/DirectAnnotationAccess.java | 47 + .../graalvm/util/GuardedAnnotationAccess.java | 108 + test/hotspot/jtreg/ProblemList-graal.txt | 1 - .../jtreg/compiler/graalunit/EA9Test.java | 38 + .../compiler/graalunit/HotspotJdk9Test.java | 38 + .../jtreg/compiler/graalunit/TestPackages.txt | 1 + 304 files changed, 17324 insertions(+), 8639 deletions(-) rename src/jdk.internal.vm.compiler/share/classes/{org.graalvm.compiler.java/src/org/graalvm/compiler/java/IntegerExactOpSpeculation.java => org.graalvm.compiler.asm/src/org/graalvm/compiler/asm/BranchTargetOutOfBoundsException.java} (50%) create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64CbzTest.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64MembarOpTest.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64TestBitAndBranchTest.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/doc-files/MitigateSpeculativeExecutionAttacksHelp.txt create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/SymbolicJVMCIReference.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/CountedLoopTest2.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/IntegerDivRemConstantTest.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/SwitchCanonicalizerTest.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/LIRCompilerBackend.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/LIRGenerationProvider.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/doc-files/PrintGraphHelp.txt create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph.test/src/org/graalvm/compiler/graph/test/graphio/GraphOutputTest.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/BinaryMathStubTest.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/UnaryMathStubTest.java delete mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotMaths.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64X87MathIntrinsicNode.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64X87MathSnippets.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.jdk9.test/src/org/graalvm/compiler/hotspot/jdk9/test/StringUTF16ToBytesGetCharsTest.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/EliminateRedundantInitializationPhaseTest.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/SymbolicSnippetEncoder.java rename src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/{nodes => gc/g1}/G1ArrayRangePostWriteBarrier.java (90%) rename src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/{nodes => gc/g1}/G1ArrayRangePreWriteBarrier.java (90%) create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1BarrierSet.java rename src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/{nodes => gc/g1}/G1PostWriteBarrier.java (85%) rename src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/{nodes => gc/g1}/G1PreWriteBarrier.java (88%) rename src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/{nodes => gc/g1}/G1ReferentFieldReadBarrier.java (91%) rename src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/{nodes => gc/shared}/ArrayRangeWriteBarrier.java (92%) create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/BarrierSet.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/CardTableBarrierSet.java rename src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/{nodes => gc/shared}/ObjectWriteBarrier.java (95%) rename src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/{nodes => gc/shared}/SerialArrayRangeWriteBarrier.java (97%) rename src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/{nodes => gc/shared}/SerialWriteBarrier.java (97%) create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/ArraysSupportSubstitutions.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/CounterModeSubstitutions.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/DigestBaseSubstitutions.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/StringUTF16Substitutions.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/optimize/IfNodeCanonicalizationsTest.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/optimize/SwitchHashTableTest.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64HotSpotHelper.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathCosOp.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathExpOp.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathLog10Op.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathLogOp.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathPowOp.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathSinOp.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathTanOp.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/hashing/HashFunction.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/hashing/Hasher.java rename src/jdk.internal.vm.compiler/share/classes/{org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common => org.graalvm.compiler.loop.phases/src/org/graalvm/compiler/loop/phases}/ConvertDeoptimizeToGuardPhase.java (64%) rename src/jdk.internal.vm.compiler/share/classes/{org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic => org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc}/IntegerMulHighNode.java (95%) create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/memory/address/IndexAddressNode.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/OptimizeDivPhase.java delete mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64MathSubstitutions.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StringUTF16Substitutions.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerAddExactOverflowNode.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerExactOverflowNode.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerMulExactOverflowNode.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerSubExactOverflowNode.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/JavaVersionUtil.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/SpeculationReasonGroup.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.util/src/org/graalvm/util/DirectAnnotationAccess.java create mode 100644 src/jdk.internal.vm.compiler/share/classes/org.graalvm.util/src/org/graalvm/util/GuardedAnnotationAccess.java create mode 100644 test/hotspot/jtreg/compiler/graalunit/EA9Test.java create mode 100644 test/hotspot/jtreg/compiler/graalunit/HotspotJdk9Test.java diff --git a/make/CompileJavaModules.gmk b/make/CompileJavaModules.gmk index 0db3f3cd42f..7290e086387 100644 --- a/make/CompileJavaModules.gmk +++ b/make/CompileJavaModules.gmk @@ -461,6 +461,7 @@ jdk.internal.vm.compiler_EXCLUDES += \ org.graalvm.compiler.debug.test \ org.graalvm.compiler.graph.test \ org.graalvm.compiler.hotspot.amd64.test \ + org.graalvm.compiler.hotspot.jdk9.test \ org.graalvm.compiler.hotspot.lir.test \ org.graalvm.compiler.hotspot.sparc.test \ org.graalvm.compiler.hotspot.test \ diff --git a/make/test/JtregGraalUnit.gmk b/make/test/JtregGraalUnit.gmk index 0c0eaccb518..96939566ab9 100644 --- a/make/test/JtregGraalUnit.gmk +++ b/make/test/JtregGraalUnit.gmk @@ -92,6 +92,7 @@ ifeq ($(INCLUDE_GRAAL), true) $(SRC_DIR)/org.graalvm.compiler.debug.test/src \ $(SRC_DIR)/org.graalvm.compiler.graph.test/src \ $(SRC_DIR)/org.graalvm.compiler.hotspot.amd64.test/src \ + $(SRC_DIR)/org.graalvm.compiler.hotspot.jdk9.test/src \ $(SRC_DIR)/org.graalvm.compiler.hotspot.lir.test/src \ $(SRC_DIR)/org.graalvm.compiler.hotspot.sparc.test/src \ $(SRC_DIR)/org.graalvm.compiler.hotspot.test/src \ diff --git a/src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/Linker.java b/src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/Linker.java index 969f03b1136..9e2d1f83425 100644 --- a/src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/Linker.java +++ b/src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/Linker.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,6 +29,10 @@ import java.io.BufferedReader; import java.io.File; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; import java.util.stream.Stream; final class Linker { @@ -46,9 +50,12 @@ final class Linker { return libraryFileName; } + private static Stream getLines(InputStream stream) { + return new BufferedReader(new InputStreamReader(stream)).lines(); + } + private static String getString(InputStream stream) { - BufferedReader br = new BufferedReader(new InputStreamReader(stream)); - Stream lines = br.lines(); + Stream lines = getLines(stream); StringBuilder sb = new StringBuilder(); lines.iterator().forEachRemaining(e -> sb.append(e)); return sb.toString(); @@ -150,9 +157,18 @@ final class Linker { } /** - * Search for Visual Studio link.exe Search Order is: VS2013, VS2015, VS2012. + * Search for Visual Studio link.exe Search Order is: VS2017+, VS2013, VS2015, VS2012. */ - private static String getWindowsLinkPath() { + private static String getWindowsLinkPath() throws Exception { + try { + Path vc141NewerLinker = getVC141AndNewerLinker(); + if (vc141NewerLinker != null) { + return vc141NewerLinker.toString(); + } + } catch (Exception e) { + e.printStackTrace(); + } + String link = "\\VC\\bin\\amd64\\link.exe"; /** @@ -183,10 +199,46 @@ final class Linker { return null; } + private static Path getVC141AndNewerLinker() throws Exception { + String programFilesX86 = System.getenv("ProgramFiles(x86)"); + if (programFilesX86 == null) { + throw new InternalError("Could not read the ProgramFiles(x86) environment variable"); + } + Path vswhere = Paths.get(programFilesX86 + "\\Microsoft Visual Studio\\Installer\\vswhere.exe"); + if (!Files.exists(vswhere)) { + return null; + } + + ProcessBuilder processBuilder = new ProcessBuilder(vswhere.toString(), "-requires", + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", "-property", "installationPath", "-latest"); + processBuilder.redirectOutput(ProcessBuilder.Redirect.PIPE); + processBuilder.redirectError(ProcessBuilder.Redirect.PIPE); + Process process = processBuilder.start(); + final int exitCode = process.waitFor(); + if (exitCode != 0) { + String errorMessage = getString(process.getErrorStream()); + if (errorMessage.isEmpty()) { + errorMessage = getString(process.getInputStream()); + } + throw new InternalError(errorMessage); + } + + String installationPath = getLines(process.getInputStream()).findFirst().orElseThrow(() -> new InternalError("Unexpected empty output from vswhere")); + Path vcToolsVersionFilePath = Paths.get(installationPath, "VC\\Auxiliary\\Build\\Microsoft.VCToolsVersion.default.txt"); + List vcToolsVersionFileLines = Files.readAllLines(vcToolsVersionFilePath); + if (vcToolsVersionFileLines.isEmpty()) { + throw new InternalError(vcToolsVersionFilePath.toString() + " is empty"); + } + String vcToolsVersion = vcToolsVersionFileLines.get(0); + Path linkPath = Paths.get(installationPath, "VC\\Tools\\MSVC", vcToolsVersion, "bin\\Hostx64\\x64\\link.exe"); + if (!Files.exists(linkPath)) { + throw new InternalError("Linker at path " + linkPath.toString() + " does not exist"); + } + + return linkPath; + } + // @formatter:off (workaround for Eclipse formatting bug) - /** - * Visual Studio supported versions Search Order is: VS2013, VS2015, VS2012. - */ enum VSVERSIONS { VS2013("VS120COMNTOOLS", "C:\\Program Files (x86)\\Microsoft Visual Studio 12.0\\VC\\bin\\amd64\\link.exe"), VS2015("VS140COMNTOOLS", "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\amd64\\link.exe"), diff --git a/src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/Main.java b/src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/Main.java index 31dc6bf441a..472f9c742e4 100644 --- a/src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/Main.java +++ b/src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/Main.java @@ -166,7 +166,7 @@ public final class Main { printer.printInfo(classesToCompile.size() + " classes found"); } - OptionValues graalOptions = HotSpotGraalOptionValues.HOTSPOT_OPTIONS; + OptionValues graalOptions = HotSpotGraalOptionValues.defaultOptions(); // Setting -Dgraal.TieredAOT overrides --compile-for-tiered if (!TieredAOT.hasBeenSet(graalOptions)) { graalOptions = new OptionValues(graalOptions, TieredAOT, options.tiered); diff --git a/src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/collect/ClassSearch.java b/src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/collect/ClassSearch.java index 51e45afc38f..2f00f38aff8 100644 --- a/src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/collect/ClassSearch.java +++ b/src/jdk.aot/share/classes/jdk.tools.jaotc/src/jdk/tools/jaotc/collect/ClassSearch.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -30,6 +30,7 @@ import jdk.tools.jaotc.LoadedClass; import java.util.ArrayList; import java.util.List; import java.util.function.BiConsumer; +import java.io.File; public final class ClassSearch { private final List providers = new ArrayList<>(); @@ -106,7 +107,7 @@ public final class ClassSearch { public static List makeList(String type, String argument) { List list = new ArrayList<>(); - String[] elements = argument.split(":"); + String[] elements = argument.split(File.pathSeparator); for (String element : elements) { list.add(new SearchFor(element, type)); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64Assembler.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64Assembler.java index 79739b54800..f43cfa3d315 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64Assembler.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64Assembler.java @@ -880,15 +880,15 @@ public abstract class AArch64Assembler extends Assembler { protected void tbnz(Register reg, int uimm6, int imm16, int pos) { assert reg.getRegisterCategory().equals(CPU); assert NumUtil.isUnsignedNbit(6, uimm6); - assert NumUtil.isSignedNbit(18, imm16); - assert (imm16 & 3) == 0; + assert NumUtil.isSignedNbit(16, imm16) : String.format("Offset value must fit in 16 bits signed: 0x%x", imm16); + assert (imm16 & 3) == 0 : String.format("Lower two bits must be zero: 0x%x", imm16 & 3); // size bit is overloaded as top bit of uimm6 bit index int size = (((uimm6 >> 5) & 1) == 0 ? 32 : 64); // remaining 5 bits are encoded lower down - int uimm5 = uimm6 >> 1; - int offset = (imm16 & NumUtil.getNbitNumberInt(16)) >> 2; + int uimm5 = uimm6 & 0x1F; + int imm14 = (imm16 & NumUtil.getNbitNumberInt(16)) >> 2; InstructionType type = generalFromSize(size); - int encoding = type.encoding | TBNZ.encoding | (uimm5 << 19) | (offset << 5) | rd(reg); + int encoding = type.encoding | TBNZ.encoding | (uimm5 << 19) | (imm14 << 5) | rd(reg); if (pos == -1) { emitInt(encoding); } else { @@ -907,15 +907,15 @@ public abstract class AArch64Assembler extends Assembler { protected void tbz(Register reg, int uimm6, int imm16, int pos) { assert reg.getRegisterCategory().equals(CPU); assert NumUtil.isUnsignedNbit(6, uimm6); - assert NumUtil.isSignedNbit(18, imm16); - assert (imm16 & 3) == 0; + assert NumUtil.isSignedNbit(16, imm16) : String.format("Offset value must fit in 16 bits signed: 0x%x", imm16); + assert (imm16 & 3) == 0 : String.format("Lower two bits must be zero: 0x%x", imm16 & 3); // size bit is overloaded as top bit of uimm6 bit index int size = (((uimm6 >> 5) & 1) == 0 ? 32 : 64); // remaining 5 bits are encoded lower down - int uimm5 = uimm6 >> 1; - int offset = (imm16 & NumUtil.getNbitNumberInt(16)) >> 2; + int uimm5 = uimm6 & 0x1F; + int imm14 = (imm16 & NumUtil.getNbitNumberInt(16)) >> 2; InstructionType type = generalFromSize(size); - int encoding = type.encoding | TBZ.encoding | (uimm5 << 19) | (offset << 5) | rd(reg); + int encoding = type.encoding | TBZ.encoding | (uimm5 << 19) | (imm14 << 5) | rd(reg); if (pos == -1) { emitInt(encoding); } else { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64MacroAssembler.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64MacroAssembler.java index 017a9d6e1c3..af3f5494266 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64MacroAssembler.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.aarch64/src/org/graalvm/compiler/asm/aarch64/AArch64MacroAssembler.java @@ -33,6 +33,9 @@ import static org.graalvm.compiler.asm.aarch64.AArch64Address.AddressingMode.REG import static org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler.AddressGenerationPlan.WorkPlan.ADD_TO_BASE; import static org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler.AddressGenerationPlan.WorkPlan.ADD_TO_INDEX; import static org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler.AddressGenerationPlan.WorkPlan.NO_WORK; + +import org.graalvm.compiler.asm.BranchTargetOutOfBoundsException; + import static jdk.vm.ci.aarch64.AArch64.CPU; import static jdk.vm.ci.aarch64.AArch64.r8; import static jdk.vm.ci.aarch64.AArch64.r9; @@ -1452,7 +1455,7 @@ public class AArch64MacroAssembler extends AArch64Assembler { * * @param cmp general purpose register. May not be null, zero-register or stackpointer. * @param uimm6 Unsigned 6-bit bit index. - * @param label Can only handle 21-bit word-aligned offsets for now. May be unbound. Non null. + * @param label Can only handle 16-bit word-aligned offsets for now. May be unbound. Non null. */ public void tbnz(Register cmp, int uimm6, Label label) { assert NumUtil.isUnsignedNbit(6, uimm6); @@ -1472,7 +1475,7 @@ public class AArch64MacroAssembler extends AArch64Assembler { * * @param cmp general purpose register. May not be null, zero-register or stackpointer. * @param uimm6 Unsigned 6-bit bit index. - * @param label Can only handle 21-bit word-aligned offsets for now. May be unbound. Non null. + * @param label Can only handle 16-bit word-aligned offsets for now. May be unbound. Non null. */ public void tbz(Register cmp, int uimm6, Label label) { assert NumUtil.isUnsignedNbit(6, uimm6); @@ -1681,6 +1684,9 @@ public class AArch64MacroAssembler extends AArch64Assembler { int sizeEncoding = information & NumUtil.getNbitNumberInt(6); int regEncoding = information >>> 6; Register reg = AArch64.cpuRegisters.get(regEncoding); + if (!NumUtil.isSignedNbit(16, branchOffset)) { + throw new BranchTargetOutOfBoundsException(true, "Branch target %d out of bounds", branchOffset); + } switch (type) { case BRANCH_BIT_NONZERO: super.tbnz(reg, sizeEncoding, branchOffset, branch); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Assembler.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Assembler.java index 912822e9d17..2236ebbac46 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Assembler.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64Assembler.java @@ -683,7 +683,7 @@ public class AMD64Assembler extends AMD64BaseAssembler { emitImmediate(asm, size, imm); int nextInsnPos = asm.position(); if (annotateImm && asm.codePatchingAnnotationConsumer != null) { - asm.codePatchingAnnotationConsumer.accept(new ImmediateOperandAnnotation(insnPos, immPos, nextInsnPos - immPos, nextInsnPos)); + asm.codePatchingAnnotationConsumer.accept(new OperandDataAnnotation(insnPos, immPos, nextInsnPos - immPos, nextInsnPos)); } } @@ -700,7 +700,7 @@ public class AMD64Assembler extends AMD64BaseAssembler { emitImmediate(asm, size, imm); int nextInsnPos = asm.position(); if (annotateImm && asm.codePatchingAnnotationConsumer != null) { - asm.codePatchingAnnotationConsumer.accept(new ImmediateOperandAnnotation(insnPos, immPos, nextInsnPos - immPos, nextInsnPos)); + asm.codePatchingAnnotationConsumer.accept(new OperandDataAnnotation(insnPos, immPos, nextInsnPos - immPos, nextInsnPos)); } } } @@ -2023,7 +2023,7 @@ public class AMD64Assembler extends AMD64BaseAssembler { emitInt(imm32); int nextInsnPos = position(); if (annotateImm && codePatchingAnnotationConsumer != null) { - codePatchingAnnotationConsumer.accept(new ImmediateOperandAnnotation(insnPos, immPos, nextInsnPos - immPos, nextInsnPos)); + codePatchingAnnotationConsumer.accept(new OperandDataAnnotation(insnPos, immPos, nextInsnPos - immPos, nextInsnPos)); } } @@ -2201,10 +2201,11 @@ public class AMD64Assembler extends AMD64BaseAssembler { } public final void movswl(Register dst, AMD64Address src) { - prefix(src, dst); - emitByte(0x0F); - emitByte(0xBF); - emitOperandHelper(dst, src, 0); + AMD64RMOp.MOVSX.emit(this, DWORD, dst, src); + } + + public final void movswq(Register dst, AMD64Address src) { + AMD64RMOp.MOVSX.emit(this, QWORD, dst, src); } public final void movw(AMD64Address dst, int imm16) { @@ -2222,6 +2223,13 @@ public class AMD64Assembler extends AMD64BaseAssembler { emitOperandHelper(src, dst, 0); } + public final void movw(Register dst, AMD64Address src) { + emitByte(0x66); + prefix(src, dst); + emitByte(0x8B); + emitOperandHelper(dst, src, 0); + } + public final void movzbl(Register dst, AMD64Address src) { prefix(src, dst); emitByte(0x0F); @@ -2237,11 +2245,16 @@ public class AMD64Assembler extends AMD64BaseAssembler { AMD64RMOp.MOVZXB.emit(this, QWORD, dst, src); } + public final void movzbq(Register dst, AMD64Address src) { + AMD64RMOp.MOVZXB.emit(this, QWORD, dst, src); + } + public final void movzwl(Register dst, AMD64Address src) { - prefix(src, dst); - emitByte(0x0F); - emitByte(0xB7); - emitOperandHelper(dst, src, 0); + AMD64RMOp.MOVZX.emit(this, DWORD, dst, src); + } + + public final void movzwq(Register dst, AMD64Address src) { + AMD64RMOp.MOVZX.emit(this, QWORD, dst, src); } public final void negl(Register dst) { @@ -2557,16 +2570,63 @@ public class AMD64Assembler extends AMD64BaseAssembler { emitModRM(dst, src); } - // Insn: VPMOVZXBW xmm1, xmm2/m64 - - public final void pmovzxbw(Register dst, AMD64Address src) { + private void pmovSZx(Register dst, AMD64Address src, int op) { assert supports(CPUFeature.SSE4_1); assert inRC(XMM, dst); simdPrefix(dst, Register.None, src, PD, P_0F38, false); - emitByte(0x30); + emitByte(op); emitOperandHelper(dst, src, 0); } + public final void pmovsxbw(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x20); + } + + public final void pmovsxbd(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x21); + } + + public final void pmovsxbq(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x22); + } + + public final void pmovsxwd(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x23); + } + + public final void pmovsxwq(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x24); + } + + public final void pmovsxdq(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x25); + } + + // Insn: VPMOVZXBW xmm1, xmm2/m64 + public final void pmovzxbw(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x30); + } + + public final void pmovzxbd(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x31); + } + + public final void pmovzxbq(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x32); + } + + public final void pmovzxwd(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x33); + } + + public final void pmovzxwq(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x34); + } + + public final void pmovzxdq(Register dst, AMD64Address src) { + pmovSZx(dst, src, 0x35); + } + public final void pmovzxbw(Register dst, Register src) { assert supports(CPUFeature.SSE4_1); assert inRC(XMM, dst) && inRC(XMM, src); @@ -2881,6 +2941,10 @@ public class AMD64Assembler extends AMD64BaseAssembler { XOR.rmOp.emit(this, DWORD, dst, src); } + public final void xorq(Register dst, Register src) { + XOR.rmOp.emit(this, QWORD, dst, src); + } + public final void xorpd(Register dst, Register src) { SSEOp.XOR.emit(this, PD, dst, src); } @@ -3045,7 +3109,7 @@ public class AMD64Assembler extends AMD64BaseAssembler { emitLong(imm64); int nextInsnPos = position(); if (annotateImm && codePatchingAnnotationConsumer != null) { - codePatchingAnnotationConsumer.accept(new ImmediateOperandAnnotation(insnPos, immPos, nextInsnPos - immPos, nextInsnPos)); + codePatchingAnnotationConsumer.accept(new OperandDataAnnotation(insnPos, immPos, nextInsnPos - immPos, nextInsnPos)); } } @@ -3189,6 +3253,19 @@ public class AMD64Assembler extends AMD64BaseAssembler { emitModRM(5, dst); } + public final void sarq(Register dst, int imm8) { + assert isShiftCount(imm8 >> 1) : "illegal shift count"; + prefixq(dst); + if (imm8 == 1) { + emitByte(0xD1); + emitModRM(7, dst); + } else { + emitByte(0xC1); + emitModRM(7, dst); + emitByte(imm8); + } + } + public final void sbbq(Register dst, Register src) { SBB.rmOp.emit(this, QWORD, dst, src); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64BaseAssembler.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64BaseAssembler.java index d550c5327d0..8d070618f66 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64BaseAssembler.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.amd64/src/org/graalvm/compiler/asm/amd64/AMD64BaseAssembler.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -210,7 +210,7 @@ public abstract class AMD64BaseAssembler extends Assembler { } } - public abstract static class OperandDataAnnotation extends CodeAnnotation { + public static class OperandDataAnnotation extends CodeAnnotation { /** * The position (bytes from the beginning of the method) of the operand. */ @@ -239,30 +239,10 @@ public abstract class AMD64BaseAssembler extends Assembler { } } - /** - * Annotation that stores additional information about the displacement of a - * {@link Assembler#getPlaceholder placeholder address} that needs patching. - */ - protected static class AddressDisplacementAnnotation extends OperandDataAnnotation { - AddressDisplacementAnnotation(int instructionPosition, int operandPosition, int operandSize, int nextInstructionPosition) { - super(instructionPosition, operandPosition, operandSize, nextInstructionPosition); - } - } - - /** - * Annotation that stores additional information about the immediate operand, e.g., of a call - * instruction, that needs patching. - */ - protected static class ImmediateOperandAnnotation extends OperandDataAnnotation { - ImmediateOperandAnnotation(int instructionPosition, int operandPosition, int operandSize, int nextInstructionPosition) { - super(instructionPosition, operandPosition, operandSize, nextInstructionPosition); - } - } - protected void annotatePatchingImmediate(int operandOffset, int operandSize) { if (codePatchingAnnotationConsumer != null) { int pos = position(); - codePatchingAnnotationConsumer.accept(new ImmediateOperandAnnotation(pos, pos + operandOffset, operandSize, pos + operandOffset + operandSize)); + codePatchingAnnotationConsumer.accept(new OperandDataAnnotation(pos, pos + operandOffset, operandSize, pos + operandOffset + operandSize)); } } @@ -581,7 +561,7 @@ public abstract class AMD64BaseAssembler extends Assembler { assert index.equals(Register.None) : "cannot use RIP relative addressing with index register"; emitByte(0x05 | regenc); if (codePatchingAnnotationConsumer != null && addr.instructionStartPosition >= 0) { - codePatchingAnnotationConsumer.accept(new AddressDisplacementAnnotation(addr.instructionStartPosition, position(), 4, position() + 4 + additionalInstructionSize)); + codePatchingAnnotationConsumer.accept(new OperandDataAnnotation(addr.instructionStartPosition, position(), 4, position() + 4 + additionalInstructionSize)); } emitInt(disp); } else if (base.isValid()) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.sparc/src/org/graalvm/compiler/asm/sparc/SPARCAssembler.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.sparc/src/org/graalvm/compiler/asm/sparc/SPARCAssembler.java index 393910c6ad1..867e6d4812c 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.sparc/src/org/graalvm/compiler/asm/sparc/SPARCAssembler.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.sparc/src/org/graalvm/compiler/asm/sparc/SPARCAssembler.java @@ -136,9 +136,9 @@ import java.util.List; import java.util.Map; import org.graalvm.compiler.asm.Assembler; +import org.graalvm.compiler.asm.BranchTargetOutOfBoundsException; import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.core.common.NumUtil; -import org.graalvm.compiler.core.common.PermanentBailoutException; import org.graalvm.compiler.debug.GraalError; import jdk.vm.ci.code.Register; @@ -1281,7 +1281,7 @@ public abstract class SPARCAssembler extends Assembler { public int setDisp(int inst, int d) { assert this.match(inst); if (!isValidDisp(d)) { - throw new PermanentBailoutException("Too large displacement 0x%x in field %s in instruction %s", d, this.disp, this); + throw new BranchTargetOutOfBoundsException(true, "Too large displacement 0x%x in field %s in instruction %s", d, this.disp, this); } return this.disp.setBits(inst, d); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.test/src/org/graalvm/compiler/asm/test/AssemblerTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.test/src/org/graalvm/compiler/asm/test/AssemblerTest.java index c1c39701a08..8efca05d69e 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.test/src/org/graalvm/compiler/asm/test/AssemblerTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm.test/src/org/graalvm/compiler/asm/test/AssemblerTest.java @@ -32,6 +32,7 @@ import org.graalvm.compiler.api.test.Graal; import org.graalvm.compiler.code.CompilationResult; import org.graalvm.compiler.code.DisassemblerProvider; import org.graalvm.compiler.core.common.CompilationIdentifier; +import org.graalvm.compiler.core.gen.LIRGenerationProvider; import org.graalvm.compiler.core.target.Backend; import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.nodes.StructuredGraph; @@ -90,7 +91,7 @@ public abstract class AssemblerTest extends GraalTest { RegisterConfig registerConfig = codeCache.getRegisterConfig(); CompilationIdentifier compilationId = backend.getCompilationIdentifier(method); StructuredGraph graph = new StructuredGraph.Builder(options, debug).method(method).compilationId(compilationId).build(); - CallingConvention cc = backend.newLIRGenerationResult(compilationId, null, null, graph, null).getCallingConvention(); + CallingConvention cc = ((LIRGenerationProvider) backend).newLIRGenerationResult(compilationId, null, null, graph, null).getCallingConvention(); CompilationResult compResult = new CompilationResult(graph.compilationId()); byte[] targetCode = test.generateCode(compResult, codeCache.getTarget(), registerConfig, cc); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/IntegerExactOpSpeculation.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm/src/org/graalvm/compiler/asm/BranchTargetOutOfBoundsException.java similarity index 50% rename from src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/IntegerExactOpSpeculation.java rename to src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm/src/org/graalvm/compiler/asm/BranchTargetOutOfBoundsException.java index 4701cb1d2da..1b8b6e93813 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/IntegerExactOpSpeculation.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.asm/src/org/graalvm/compiler/asm/BranchTargetOutOfBoundsException.java @@ -22,41 +22,23 @@ */ -package org.graalvm.compiler.java; +package org.graalvm.compiler.asm; -import jdk.vm.ci.meta.ResolvedJavaMethod; -import jdk.vm.ci.meta.SpeculationLog.SpeculationReason; +import org.graalvm.compiler.core.common.GraalBailoutException; -public final class IntegerExactOpSpeculation implements SpeculationReason { +@SuppressWarnings("serial") +public class BranchTargetOutOfBoundsException extends GraalBailoutException { - public enum IntegerExactOp { - INTEGER_ADD_EXACT, - INTEGER_INCREMENT_EXACT, - INTEGER_SUBTRACT_EXACT, - INTEGER_DECREMENT_EXACT, - INTEGER_MULTIPLY_EXACT + public BranchTargetOutOfBoundsException(boolean permanent, String format, Object... args) { + super(permanent, format, args); } - protected final String methodDescriptor; - protected final IntegerExactOp op; - - public IntegerExactOpSpeculation(ResolvedJavaMethod method, IntegerExactOp op) { - this.methodDescriptor = method.format("%H.%n(%p)%R"); - this.op = op; + public BranchTargetOutOfBoundsException(String format, Object... args) { + super(format, args); } - @Override - public int hashCode() { - return methodDescriptor.hashCode() * 31 + op.ordinal(); - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof IntegerExactOpSpeculation) { - IntegerExactOpSpeculation other = (IntegerExactOpSpeculation) obj; - return op.equals(other.op) && methodDescriptor.equals(other.methodDescriptor); - } - return false; + public BranchTargetOutOfBoundsException(Throwable cause, String format, Object... args) { + super(cause, format, args); } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.code/src/org/graalvm/compiler/code/CompilationResult.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.code/src/org/graalvm/compiler/code/CompilationResult.java index cdbd465c254..6a2f43bbc3a 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.code/src/org/graalvm/compiler/code/CompilationResult.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.code/src/org/graalvm/compiler/code/CompilationResult.java @@ -55,6 +55,7 @@ import jdk.vm.ci.meta.Assumptions.Assumption; import jdk.vm.ci.meta.InvokeTarget; import jdk.vm.ci.meta.ResolvedJavaField; import jdk.vm.ci.meta.ResolvedJavaMethod; +import jdk.vm.ci.meta.SpeculationLog; /** * Represents the output from compiling a method, including the compiled machine code, associated @@ -220,6 +221,11 @@ public class CompilationResult { */ private ResolvedJavaMethod[] methods; + /** + * The {@link SpeculationLog} log used during compilation. + */ + private SpeculationLog speculationLog; + /** * The list of fields that were accessed from the bytecodes. */ @@ -372,6 +378,21 @@ public class CompilationResult { return methods; } + /** + * Sets the {@link SpeculationLog} log used during compilation. + */ + public void setSpeculationLog(SpeculationLog speculationLog) { + checkOpen(); + this.speculationLog = speculationLog; + } + + /** + * Gets the {@link SpeculationLog} log, if any, used during compilation. + */ + public SpeculationLog getSpeculationLog() { + return speculationLog; + } + /** * Sets the fields that were referenced from the bytecodes that were used as input to the * compilation. @@ -615,7 +636,7 @@ public class CompilationResult { /** * @return the code annotations or {@code null} if there are none */ - public List getAnnotations() { + public List getCodeAnnotations() { if (annotations == null) { return Collections.emptyList(); } @@ -706,7 +727,8 @@ public class CompilationResult { * Clears the information in this object pertaining to generating code. That is, the * {@linkplain #getMarks() marks}, {@linkplain #getInfopoints() infopoints}, * {@linkplain #getExceptionHandlers() exception handlers}, {@linkplain #getDataPatches() data - * patches} and {@linkplain #getAnnotations() annotations} recorded in this object are cleared. + * patches} and {@linkplain #getCodeAnnotations() annotations} recorded in this object are + * cleared. */ public void resetForEmittingCode() { checkOpen(); @@ -721,6 +743,14 @@ public class CompilationResult { } } + public void clearInfopoints() { + infopoints.clear(); + } + + public void clearExceptionHandlers() { + exceptionHandlers.clear(); + } + private void checkOpen() { if (closed) { throw new IllegalStateException(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.code/src/org/graalvm/compiler/code/HexCodeFileDisassemblerProvider.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.code/src/org/graalvm/compiler/code/HexCodeFileDisassemblerProvider.java index 190bd6e9621..65306bc551d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.code/src/org/graalvm/compiler/code/HexCodeFileDisassemblerProvider.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.code/src/org/graalvm/compiler/code/HexCodeFileDisassemblerProvider.java @@ -78,7 +78,7 @@ public class HexCodeFileDisassemblerProvider implements DisassemblerProvider { long start = installedCode == null ? 0L : installedCode.getStart(); HexCodeFile hcf = new HexCodeFile(code, start, target.arch.getName(), target.wordSize * 8); if (compResult != null) { - HexCodeFile.addAnnotations(hcf, compResult.getAnnotations()); + HexCodeFile.addAnnotations(hcf, compResult.getCodeAnnotations()); addExceptionHandlersComment(compResult, hcf); Register fp = regConfig.getFrameRegister(); RefMapFormatter slotFormatter = new DefaultRefMapFormatter(target.wordSize, fp, 0); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64CbzTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64CbzTest.java new file mode 100644 index 00000000000..c9c54da019c --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64CbzTest.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, Arm Limited. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + + +package org.graalvm.compiler.core.aarch64.test; + +import org.graalvm.compiler.lir.LIRInstruction; +import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow; +import org.junit.Test; + +import java.util.function.Predicate; + +public class AArch64CbzTest extends AArch64MatchRuleTest { + private static final Predicate predicate = op -> (op instanceof AArch64ControlFlow.CompareBranchZeroOp); + + public static int equalsTo(int x) { + if (x == 0) { + return 1; + } else { + return x - 1; + } + } + + public static int notEqualsTo(int x) { + if (x != 0) { + return x + 2; + } else { + return 3; + } + } + + public static String isNull(String s) { + if (s == null) { + return "abc"; + } else { + return s + "abc"; + } + } + + public static String isNotNull(String s) { + if (s != null) { + return s + "abc"; + } else { + return "abc"; + } + } + + public static String objectEqualsNull(String s1, String s2) { + if (s1.equals(null)) { + return s1 + "abc"; + } else { + return s2 + "abd"; + } + } + + public static String objectEquals(String s1, String s2) { + if (s1.equals(s2)) { + return s1 + "abc"; + } else { + return s2 + "abd"; + } + } + + @Test + public void testEqualsTo() { + test("equalsTo", 0); + test("equalsTo", 1); + checkLIR("equalsTo", predicate, 1); + } + + @Test + public void testNotEqualsTo() { + test("notEqualsTo", 0); + test("notEqualsTo", 1); + checkLIR("notEqualsTo", predicate, 1); + } + + @Test + public void testIsNull() { + test("isNull", new Object[]{null}); + test("isNull", "abc"); + checkLIR("isNull", predicate, 1); + } + + @Test + public void testIsNotNull() { + test("isNotNull", new Object[]{null}); + test("isNotNull", "abc"); + checkLIR("isNotNull", predicate, 1); + } + + @Test + public void testObjectEqualsNull() { + test("objectEqualsNull", "ab", "ac"); + test("objectEqualsNull", "abc", "abc"); + checkLIR("objectEqualsNull", predicate, 1); + } + + @Test + public void testObjectEquals() { + test("objectEquals", "ab", "ac"); + test("objectEquals", "abc", "abc"); + checkLIR("objectEquals", predicate, 0); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64MembarOpTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64MembarOpTest.java new file mode 100644 index 00000000000..8f16c4a6bc1 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64MembarOpTest.java @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, Arm Limited. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.core.aarch64.test; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assume.assumeTrue; + +import java.util.ArrayList; +import java.util.List; + +import jdk.internal.vm.compiler.collections.Pair; +import org.graalvm.compiler.asm.aarch64.AArch64Assembler.BarrierKind; +import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler; +import org.graalvm.compiler.code.CompilationResult; +import org.graalvm.compiler.core.gen.LIRGenerationProvider; +import org.graalvm.compiler.core.test.backend.BackendTest; +import org.graalvm.compiler.lir.aarch64.AArch64Move.MembarOp; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; +import org.graalvm.compiler.lir.asm.CompilationResultBuilderFactory; +import org.graalvm.compiler.lir.gen.LIRGenerationResult; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.junit.Before; +import org.junit.Test; + +import jdk.vm.ci.aarch64.AArch64; +import jdk.vm.ci.code.MemoryBarriers; +import jdk.vm.ci.runtime.JVMCI; +import jdk.vm.ci.runtime.JVMCIBackend; + +public class AArch64MembarOpTest extends BackendTest { + + private final JVMCIBackend providers; + private final CompilationResultBuilder crb; + + public AArch64MembarOpTest() { + this.providers = JVMCI.getRuntime().getHostJVMCIBackend(); + + final StructuredGraph graph = parseEager("stub", StructuredGraph.AllowAssumptions.YES); + LIRGenerationResult lirGenRes = getLIRGenerationResult(graph); + CompilationResult compResult = new CompilationResult(graph.compilationId()); + this.crb = ((LIRGenerationProvider) getBackend()).newCompilationResultBuilder(lirGenRes, lirGenRes.getFrameMap(), compResult, CompilationResultBuilderFactory.Default); + } + + public void stub() { + } + + @Before + public void checkAArch64() { + assumeTrue("skipping AArch64 specific test", JVMCI.getRuntime().getHostJVMCIBackend().getTarget().arch instanceof AArch64); + } + + @Test + public void runNormalMembarTests() { + List> cases = new ArrayList<>(); + cases.add(Pair.create(MemoryBarriers.LOAD_LOAD, BarrierKind.LOAD_LOAD)); + cases.add(Pair.create(MemoryBarriers.LOAD_STORE, BarrierKind.LOAD_LOAD)); + cases.add(Pair.create(MemoryBarriers.LOAD_LOAD | MemoryBarriers.LOAD_STORE, BarrierKind.LOAD_LOAD)); + cases.add(Pair.create(MemoryBarriers.STORE_LOAD, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_LOAD | MemoryBarriers.LOAD_LOAD, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_LOAD | MemoryBarriers.LOAD_STORE, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_LOAD | MemoryBarriers.LOAD_LOAD | MemoryBarriers.LOAD_STORE, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_STORE, BarrierKind.STORE_STORE)); + cases.add(Pair.create(MemoryBarriers.STORE_STORE | MemoryBarriers.LOAD_LOAD, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_STORE | MemoryBarriers.LOAD_STORE, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_STORE | MemoryBarriers.LOAD_LOAD | MemoryBarriers.LOAD_STORE, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_STORE | MemoryBarriers.STORE_LOAD, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_STORE | MemoryBarriers.STORE_LOAD | MemoryBarriers.LOAD_LOAD, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_STORE | MemoryBarriers.STORE_LOAD | MemoryBarriers.LOAD_STORE, BarrierKind.ANY_ANY)); + cases.add(Pair.create(MemoryBarriers.STORE_STORE | MemoryBarriers.STORE_LOAD | MemoryBarriers.LOAD_STORE | MemoryBarriers.LOAD_LOAD, BarrierKind.ANY_ANY)); + + for (Pair c : cases) { + assertArrayEquals(new MembarOpActual(c.getLeft()).emit(new AArch64MacroAssembler(providers.getTarget())), + new MembarOpExpected(c.getRight()).emit(new AArch64MacroAssembler(providers.getTarget()))); + } + } + + @Test(expected = AssertionError.class) + public void runExceptionalTests() { + new MembarOpActual(16).emit(new AArch64MacroAssembler(providers.getTarget())); + } + + private class MembarOpActual { + private MembarOp op; + + MembarOpActual(int barriers) { + op = new MembarOp(barriers); + } + + byte[] emit(AArch64MacroAssembler masm) { + op.emitCode(crb, masm); + return masm.close(false); + } + } + + private class MembarOpExpected { + private BarrierKind barrierKind; + + MembarOpExpected(BarrierKind barrierKind) { + this.barrierKind = barrierKind; + } + + byte[] emit(AArch64MacroAssembler masm) { + masm.dmb(barrierKind); + return masm.close(false); + } + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64TestBitAndBranchTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64TestBitAndBranchTest.java new file mode 100644 index 00000000000..da34e7ce9f5 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64.test/src/org/graalvm/compiler/core/aarch64/test/AArch64TestBitAndBranchTest.java @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.core.aarch64.test; + +import static org.junit.Assume.assumeTrue; + +import java.util.function.Predicate; + +import org.graalvm.compiler.api.directives.GraalDirectives; +import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler; +import org.graalvm.compiler.lir.LIR; +import org.graalvm.compiler.lir.LIRInstruction; +import org.graalvm.compiler.lir.LIRInstructionClass; +import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow; +import org.graalvm.compiler.lir.aarch64.AArch64LIRInstruction; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; +import org.graalvm.compiler.lir.gen.LIRGenerationResult; +import org.graalvm.compiler.lir.gen.LIRGeneratorTool; +import org.graalvm.compiler.lir.jtt.LIRTest; +import org.graalvm.compiler.lir.jtt.LIRTestSpecification; +import org.graalvm.compiler.lir.phases.LIRPhase; +import org.graalvm.compiler.lir.phases.LIRSuites; +import org.graalvm.compiler.lir.phases.PreAllocationOptimizationPhase.PreAllocationOptimizationContext; +import org.graalvm.compiler.options.OptionValues; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import jdk.vm.ci.aarch64.AArch64; +import jdk.vm.ci.code.TargetDescription; +import jdk.vm.ci.meta.Value; + +public class AArch64TestBitAndBranchTest extends LIRTest { + private static final Predicate checkForBitTestAndBranchOp = op -> (op instanceof AArch64ControlFlow.BitTestAndBranchOp); + private LIR lir; + + @Before + public void checkAArch64() { + assumeTrue("skipping AArch64 specific test", getTarget().arch instanceof AArch64); + } + + public static long testBit42Snippet(long a, long b, long c) { + if ((a & (1 << 42)) == 0) { + return b; + } else { + return c; + } + } + + @Test + public void testBit42() { + test("testBit42Snippet", 1L << 42L, Long.MAX_VALUE, Long.MIN_VALUE); + test("testBit42Snippet", ~(1L << 42L), Long.MAX_VALUE, Long.MIN_VALUE); + checkLIR("testBit42Snippet", checkForBitTestAndBranchOp, 1); + } + + private static final LargeOpSpec largeOpSingleNop = new LargeOpSpec((1 << 14 - 2), 2); + + /** + * Tests the graceful case, where the estimation for + * {@link CompilationResultBuilder#labelWithinRange(LIRInstruction, org.graalvm.compiler.asm.Label, int)} + * holds. + */ + public static int testBitTestAndBranchSingleSnippet(int a) { + int res; + if (a % 2 == 0) { + res = fillOps(largeOpSingleNop, 1); + } else { + res = fillOps(largeOpSingleNop, 2); + } + return GraalDirectives.opaque(res); + } + + @Test + public void testBitTestAndBranchSingle() { + runTest("testBitTestAndBranchSingleSnippet", 1); + checkLIR("testBitTestAndBranchSingleSnippet", checkForBitTestAndBranchOp, 1); + } + + private static final LargeOpSpec largeOpFourNop = new LargeOpSpec((1 << 14 - 2), 8); + + /** + * Tests the case, where the estimation for + * {@link CompilationResultBuilder#labelWithinRange(LIRInstruction, org.graalvm.compiler.asm.Label, int)} + * does not hold and the code generation must be redone with large branches. + */ + public static int testBitTestAndBranchFourSnippet(int a) { + int res; + if (a % 2 == 0) { + res = fillOps(largeOpFourNop, 1); + } else { + res = fillOps(largeOpFourNop, 2); + } + return GraalDirectives.opaque(res); + } + + @Test + public void testBitTestAndBranchFour() { + runTest("testBitTestAndBranchFourSnippet", 1); + checkLIR("testBitTestAndBranchFourSnippet", checkForBitTestAndBranchOp, 1); + } + + private static class LargeOpSpec extends LIRTestSpecification { + private final int n; + private final int nopCount; + + LargeOpSpec(int n, int nopCount) { + super(); + this.n = n; + this.nopCount = nopCount; + } + + @Override + public void generate(LIRGeneratorTool gen, Value a) { + for (int i = 0; i < n; i++) { + gen.append(new NoOp(nopCount)); + } + setResult(a); + } + } + + public static class NoOp extends AArch64LIRInstruction { + private static final LIRInstructionClass TYPE = LIRInstructionClass.create(NoOp.class); + private final int nopCount; + + public NoOp(int nopCount) { + super(TYPE); + this.nopCount = nopCount; + } + + @Override + protected void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { + for (int i = 0; i < nopCount; i++) { + masm.nop(); + } + } + } + + @LIRIntrinsic + public static int fillOps(@SuppressWarnings("unused") LargeOpSpec s, int a) { + return a; + } + + @Override + protected LIRSuites createLIRSuites(OptionValues options) { + LIRSuites suites = super.createLIRSuites(options); + suites.getPreAllocationOptimizationStage().appendPhase(new CheckPhase()); + return suites; + } + + public class CheckPhase extends LIRPhase { + @Override + protected void run(TargetDescription target, LIRGenerationResult lirGenRes, PreAllocationOptimizationContext context) { + lir = lirGenRes.getLIR(); + } + } + + protected void checkLIR(String methodName, Predicate predicate, int expected) { + compile(getResolvedJavaMethod(methodName), null); + int actualOpNum = 0; + for (LIRInstruction ins : lir.getLIRforBlock(lir.codeEmittingOrder()[0])) { + if (predicate.test(ins)) { + actualOpNum++; + } + } + Assert.assertEquals(expected, actualOpNum); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64ArithmeticLIRGenerator.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64ArithmeticLIRGenerator.java index c3fe49f7460..fe487d56ab2 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64ArithmeticLIRGenerator.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64ArithmeticLIRGenerator.java @@ -484,26 +484,6 @@ public class AArch64ArithmeticLIRGenerator extends ArithmeticLIRGenerator implem getLIRGen().append(new StoreOp(kind, storeAddress, input, state)); } - @Override - public Value emitMathLog(Value input, boolean base10) { - throw GraalError.unimplemented(); - } - - @Override - public Value emitMathCos(Value input) { - throw GraalError.unimplemented(); - } - - @Override - public Value emitMathSin(Value input) { - throw GraalError.unimplemented(); - } - - @Override - public Value emitMathTan(Value input) { - throw GraalError.unimplemented(); - } - @Override public void emitCompareOp(AArch64Kind cmpKind, Variable left, Value right) { throw GraalError.unimplemented(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64LIRGenerator.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64LIRGenerator.java index 42f409c4fd1..5d1eee000eb 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64LIRGenerator.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64LIRGenerator.java @@ -52,6 +52,7 @@ import org.graalvm.compiler.lir.aarch64.AArch64ByteSwapOp; import org.graalvm.compiler.lir.aarch64.AArch64Compare; import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow; import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.BranchOp; +import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.CompareBranchZeroOp; import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.CondMoveOp; import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.CondSetOp; import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow.StrategySwitchOp; @@ -257,6 +258,27 @@ public abstract class AArch64LIRGenerator extends LIRGenerator { @Override public void emitCompareBranch(PlatformKind cmpKind, Value left, Value right, Condition cond, boolean unorderedIsTrue, LabelRef trueDestination, LabelRef falseDestination, double trueDestinationProbability) { + if (cond == Condition.EQ) { + // emit cbz instruction for IsNullNode. + assert !LIRValueUtil.isNullConstant(left) : "emitNullCheckBranch()'s null input should be in right."; + if (LIRValueUtil.isNullConstant(right)) { + append(new CompareBranchZeroOp(asAllocatable(left), trueDestination, falseDestination, trueDestinationProbability)); + return; + } + + // emit cbz instruction for IntegerEquals when any of the inputs is zero. + AArch64Kind kind = (AArch64Kind) cmpKind; + if (kind.isInteger()) { + if (isIntConstant(left, 0)) { + append(new CompareBranchZeroOp(asAllocatable(right), trueDestination, falseDestination, trueDestinationProbability)); + return; + } else if (isIntConstant(right, 0)) { + append(new CompareBranchZeroOp(asAllocatable(left), trueDestination, falseDestination, trueDestinationProbability)); + return; + } + } + } + boolean mirrored = emitCompare(cmpKind, left, right, cond, unorderedIsTrue); Condition finalCondition = mirrored ? cond.mirror() : cond; boolean finalUnorderedIsTrue = mirrored ? !unorderedIsTrue : unorderedIsTrue; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64NodeMatchRules.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64NodeMatchRules.java index 29e9061f720..1c57eaea609 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64NodeMatchRules.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.aarch64/src/org/graalvm/compiler/core/aarch64/AArch64NodeMatchRules.java @@ -37,11 +37,14 @@ import org.graalvm.compiler.core.match.ComplexMatchResult; import org.graalvm.compiler.core.match.MatchRule; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.lir.LIRFrameState; +import org.graalvm.compiler.lir.LabelRef; import org.graalvm.compiler.lir.Variable; import org.graalvm.compiler.lir.aarch64.AArch64ArithmeticOp; +import org.graalvm.compiler.lir.aarch64.AArch64ControlFlow; import org.graalvm.compiler.lir.gen.LIRGeneratorTool; import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.DeoptimizingNode; +import org.graalvm.compiler.nodes.IfNode; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.calc.AddNode; @@ -178,6 +181,28 @@ public class AArch64NodeMatchRules extends NodeMatchRules { return builder -> getArithmeticLIRGenerator().emitMSub(operand(a), operand(b), operand(c)); } + /** + * ((x & (1 << n)) == 0) -> tbz/tbnz n label. + */ + @MatchRule("(If (IntegerTest value Constant=a))") + public ComplexMatchResult testBitAndBranch(IfNode root, ValueNode value, ConstantNode a) { + if (value.getStackKind().isNumericInteger()) { + long constant = a.asJavaConstant().asLong(); + if (Long.bitCount(constant) == 1) { + int bitToTest = Long.numberOfTrailingZeros(constant); + return builder -> { + LabelRef trueDestination = getLIRBlock(root.trueSuccessor()); + LabelRef falseDestination = getLIRBlock(root.falseSuccessor()); + AllocatableValue src = moveSp(gen.asAllocatable(operand(value))); + double trueDestinationProbability = root.getTrueSuccessorProbability(); + gen.append(new AArch64ControlFlow.BitTestAndBranchOp(trueDestination, falseDestination, src, trueDestinationProbability, bitToTest)); + return null; + }; + } + } + return null; + } + @Override public AArch64LIRGenerator getLIRGeneratorTool() { return (AArch64LIRGenerator) gen; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64ArithmeticLIRGenerator.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64ArithmeticLIRGenerator.java index 57877831dbc..eb927296142 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64ArithmeticLIRGenerator.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64ArithmeticLIRGenerator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2009, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -80,13 +80,6 @@ import static org.graalvm.compiler.lir.LIRValueUtil.isConstantValue; import static org.graalvm.compiler.lir.LIRValueUtil.isJavaConstant; import static org.graalvm.compiler.lir.amd64.AMD64Arithmetic.DREM; import static org.graalvm.compiler.lir.amd64.AMD64Arithmetic.FREM; -import static org.graalvm.compiler.lir.amd64.AMD64MathIntrinsicBinaryOp.BinaryIntrinsicOpcode.POW; -import static org.graalvm.compiler.lir.amd64.AMD64MathIntrinsicUnaryOp.UnaryIntrinsicOpcode.COS; -import static org.graalvm.compiler.lir.amd64.AMD64MathIntrinsicUnaryOp.UnaryIntrinsicOpcode.EXP; -import static org.graalvm.compiler.lir.amd64.AMD64MathIntrinsicUnaryOp.UnaryIntrinsicOpcode.LOG; -import static org.graalvm.compiler.lir.amd64.AMD64MathIntrinsicUnaryOp.UnaryIntrinsicOpcode.LOG10; -import static org.graalvm.compiler.lir.amd64.AMD64MathIntrinsicUnaryOp.UnaryIntrinsicOpcode.SIN; -import static org.graalvm.compiler.lir.amd64.AMD64MathIntrinsicUnaryOp.UnaryIntrinsicOpcode.TAN; import org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64BinaryArithmetic; import org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64MIOp; @@ -116,8 +109,13 @@ import org.graalvm.compiler.lir.amd64.AMD64ArithmeticLIRGeneratorTool; import org.graalvm.compiler.lir.amd64.AMD64Binary; import org.graalvm.compiler.lir.amd64.AMD64BinaryConsumer; import org.graalvm.compiler.lir.amd64.AMD64ClearRegisterOp; -import org.graalvm.compiler.lir.amd64.AMD64MathIntrinsicBinaryOp; -import org.graalvm.compiler.lir.amd64.AMD64MathIntrinsicUnaryOp; +import org.graalvm.compiler.lir.amd64.AMD64MathCosOp; +import org.graalvm.compiler.lir.amd64.AMD64MathExpOp; +import org.graalvm.compiler.lir.amd64.AMD64MathLog10Op; +import org.graalvm.compiler.lir.amd64.AMD64MathLogOp; +import org.graalvm.compiler.lir.amd64.AMD64MathPowOp; +import org.graalvm.compiler.lir.amd64.AMD64MathSinOp; +import org.graalvm.compiler.lir.amd64.AMD64MathTanOp; import org.graalvm.compiler.lir.amd64.AMD64Move; import org.graalvm.compiler.lir.amd64.AMD64MulDivOp; import org.graalvm.compiler.lir.amd64.AMD64ShiftOp; @@ -127,7 +125,6 @@ import org.graalvm.compiler.lir.amd64.vector.AMD64VectorBinary; import org.graalvm.compiler.lir.amd64.vector.AMD64VectorBinary.AVXBinaryOp; import org.graalvm.compiler.lir.amd64.vector.AMD64VectorUnary; import org.graalvm.compiler.lir.gen.ArithmeticLIRGenerator; -import org.graalvm.compiler.lir.gen.LIRGenerator; import jdk.vm.ci.amd64.AMD64; import jdk.vm.ci.amd64.AMD64.CPUFeature; @@ -152,41 +149,11 @@ public class AMD64ArithmeticLIRGenerator extends ArithmeticLIRGenerator implemen private static final RegisterValue RCX_I = AMD64.rcx.asValue(LIRKind.value(AMD64Kind.DWORD)); - public AMD64ArithmeticLIRGenerator(AllocatableValue nullRegisterValue, Maths maths) { + public AMD64ArithmeticLIRGenerator(AllocatableValue nullRegisterValue) { this.nullRegisterValue = nullRegisterValue; - this.maths = maths == null ? new Maths() { - } : maths; } private final AllocatableValue nullRegisterValue; - private final Maths maths; - - /** - * Interface for emitting LIR for selected {@link Math} routines. A {@code null} return value - * for any method in this interface means the caller must emit the LIR itself. - */ - public interface Maths { - - @SuppressWarnings("unused") - default Variable emitLog(LIRGenerator gen, Value input, boolean base10) { - return null; - } - - @SuppressWarnings("unused") - default Variable emitCos(LIRGenerator gen, Value input) { - return null; - } - - @SuppressWarnings("unused") - default Variable emitSin(LIRGenerator gen, Value input) { - return null; - } - - @SuppressWarnings("unused") - default Variable emitTan(LIRGenerator gen, Value input) { - return null; - } - } @Override public Variable emitNegate(Value inputVal) { @@ -781,6 +748,7 @@ public class AMD64ArithmeticLIRGenerator extends ArithmeticLIRGenerator implemen } } + @Override public Variable emitRor(Value a, Value b) { switch ((AMD64Kind) a.getPlatformKind()) { case DWORD: @@ -1103,65 +1071,36 @@ public class AMD64ArithmeticLIRGenerator extends ArithmeticLIRGenerator implemen @Override public Value emitMathLog(Value input, boolean base10) { - LIRGenerator gen = getLIRGen(); - Variable result = maths.emitLog(gen, input, base10); - if (result == null) { - result = gen.newVariable(LIRKind.combine(input)); - AllocatableValue stackSlot = gen.getResult().getFrameMapBuilder().allocateSpillSlot(LIRKind.value(AMD64Kind.QWORD)); - gen.append(new AMD64MathIntrinsicUnaryOp(getAMD64LIRGen(), base10 ? LOG10 : LOG, result, asAllocatable(input), stackSlot)); + if (base10) { + return new AMD64MathLog10Op().emitLIRWrapper(getLIRGen(), input); + } else { + return new AMD64MathLogOp().emitLIRWrapper(getLIRGen(), input); } - return result; } @Override public Value emitMathCos(Value input) { - LIRGenerator gen = getLIRGen(); - Variable result = maths.emitCos(gen, input); - if (result == null) { - result = gen.newVariable(LIRKind.combine(input)); - AllocatableValue stackSlot = gen.getResult().getFrameMapBuilder().allocateSpillSlot(LIRKind.value(AMD64Kind.QWORD)); - gen.append(new AMD64MathIntrinsicUnaryOp(getAMD64LIRGen(), COS, result, asAllocatable(input), stackSlot)); - } - return result; + return new AMD64MathCosOp().emitLIRWrapper(getLIRGen(), input); } @Override public Value emitMathSin(Value input) { - LIRGenerator gen = getLIRGen(); - Variable result = maths.emitSin(gen, input); - if (result == null) { - result = gen.newVariable(LIRKind.combine(input)); - AllocatableValue stackSlot = gen.getResult().getFrameMapBuilder().allocateSpillSlot(LIRKind.value(AMD64Kind.QWORD)); - gen.append(new AMD64MathIntrinsicUnaryOp(getAMD64LIRGen(), SIN, result, asAllocatable(input), stackSlot)); - } - return result; + return new AMD64MathSinOp().emitLIRWrapper(getLIRGen(), input); } @Override public Value emitMathTan(Value input) { - LIRGenerator gen = getLIRGen(); - Variable result = maths.emitTan(gen, input); - if (result == null) { - result = gen.newVariable(LIRKind.combine(input)); - AllocatableValue stackSlot = gen.getResult().getFrameMapBuilder().allocateSpillSlot(LIRKind.value(AMD64Kind.QWORD)); - gen.append(new AMD64MathIntrinsicUnaryOp(getAMD64LIRGen(), TAN, result, asAllocatable(input), stackSlot)); - } - return result; + return new AMD64MathTanOp().emitLIRWrapper(getLIRGen(), input); } @Override public Value emitMathExp(Value input) { - Variable result = getLIRGen().newVariable(LIRKind.combine(input)); - AllocatableValue stackSlot = getLIRGen().getResult().getFrameMapBuilder().allocateSpillSlot(LIRKind.value(AMD64Kind.QWORD)); - getLIRGen().append(new AMD64MathIntrinsicUnaryOp(getAMD64LIRGen(), EXP, result, asAllocatable(input), stackSlot)); - return result; + return new AMD64MathExpOp().emitLIRWrapper(getLIRGen(), input); } @Override - public Value emitMathPow(Value input1, Value input2) { - Variable result = getLIRGen().newVariable(LIRKind.combine(input1)); - getLIRGen().append(new AMD64MathIntrinsicBinaryOp(getAMD64LIRGen(), POW, result, asAllocatable(input1), asAllocatable(input2))); - return result; + public Value emitMathPow(Value x, Value y) { + return new AMD64MathPowOp().emitLIRWrapper(getLIRGen(), x, y); } protected AMD64LIRGenerator getAMD64LIRGen() { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64LIRGenerator.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64LIRGenerator.java index 66d2a64b10a..30ebd3f5d7b 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64LIRGenerator.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.amd64/src/org/graalvm/compiler/core/amd64/AMD64LIRGenerator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2009, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -41,6 +41,8 @@ import static org.graalvm.compiler.lir.LIRValueUtil.isConstantValue; import static org.graalvm.compiler.lir.LIRValueUtil.isIntConstant; import static org.graalvm.compiler.lir.LIRValueUtil.isJavaConstant; +import java.util.Optional; + import org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64BinaryArithmetic; import org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64MIOp; import org.graalvm.compiler.asm.amd64.AMD64Assembler.AMD64RMOp; @@ -81,6 +83,7 @@ import org.graalvm.compiler.lir.amd64.AMD64ControlFlow.FloatCondSetOp; import org.graalvm.compiler.lir.amd64.AMD64ControlFlow.ReturnOp; import org.graalvm.compiler.lir.amd64.AMD64ControlFlow.StrategySwitchOp; import org.graalvm.compiler.lir.amd64.AMD64ControlFlow.TableSwitchOp; +import org.graalvm.compiler.lir.amd64.AMD64ControlFlow.HashTableSwitchOp; import org.graalvm.compiler.lir.amd64.AMD64LFenceOp; import org.graalvm.compiler.lir.amd64.AMD64Move; import org.graalvm.compiler.lir.amd64.AMD64Move.CompareAndSwapOp; @@ -93,6 +96,7 @@ import org.graalvm.compiler.lir.amd64.AMD64ZapRegistersOp; import org.graalvm.compiler.lir.amd64.AMD64ZapStackOp; import org.graalvm.compiler.lir.gen.LIRGenerationResult; import org.graalvm.compiler.lir.gen.LIRGenerator; +import org.graalvm.compiler.lir.hashing.Hasher; import org.graalvm.compiler.phases.util.Providers; import jdk.vm.ci.amd64.AMD64; @@ -545,7 +549,14 @@ public abstract class AMD64LIRGenerator extends LIRGenerator { @Override public Variable emitArrayEquals(JavaKind kind, Value array1, Value array2, Value length, int constantLength, boolean directPointers) { Variable result = newVariable(LIRKind.value(AMD64Kind.DWORD)); - append(new AMD64ArrayEqualsOp(this, kind, result, array1, array2, asAllocatable(length), constantLength, directPointers, getMaxVectorSize())); + append(new AMD64ArrayEqualsOp(this, kind, kind, result, array1, array2, asAllocatable(length), constantLength, directPointers, getMaxVectorSize())); + return result; + } + + @Override + public Variable emitArrayEquals(JavaKind kind1, JavaKind kind2, Value array1, Value array2, Value length, int constantLength, boolean directPointers) { + Variable result = newVariable(LIRKind.value(AMD64Kind.DWORD)); + append(new AMD64ArrayEqualsOp(this, kind1, kind2, result, array1, array2, asAllocatable(length), constantLength, directPointers, getMaxVectorSize())); return result; } @@ -634,6 +645,19 @@ public abstract class AMD64LIRGenerator extends LIRGenerator { append(new TableSwitchOp(lowKey, defaultTarget, targets, key, newVariable(LIRKind.value(target().arch.getWordKind())), newVariable(key.getValueKind()))); } + @Override + protected Optional hasherFor(JavaConstant[] keyConstants, double minDensity) { + return Hasher.forKeys(keyConstants, minDensity); + } + + @Override + protected void emitHashTableSwitch(Hasher hasher, JavaConstant[] keys, LabelRef defaultTarget, LabelRef[] targets, Value value) { + Value index = hasher.hash(value, arithmeticLIRGen); + Variable scratch = newVariable(LIRKind.value(target().arch.getWordKind())); + Variable entryScratch = newVariable(LIRKind.value(target().arch.getWordKind())); + append(new HashTableSwitchOp(keys, defaultTarget, targets, value, index, scratch, entryScratch)); + } + @Override public void emitPause() { append(new AMD64PauseOp()); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/GraalOptions.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/GraalOptions.java index c16869e7ddf..869c7edaf96 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/GraalOptions.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/GraalOptions.java @@ -83,8 +83,8 @@ public final class GraalOptions { @Option(help = "", type = OptionType.Debug) public static final OptionKey EscapeAnalyzeOnly = new OptionKey<>(null); - @Option(help = "", type = OptionType.Expert) - public static final OptionKey MaximumEscapeAnalysisArrayLength = new OptionKey<>(32); + @Option(help = "The maximum length of an array that will be escape analyzed.", type = OptionType.Expert) + public static final OptionKey MaximumEscapeAnalysisArrayLength = new OptionKey<>(128); @Option(help = "", type = OptionType.Debug) public static final OptionKey PEAInliningHints = new OptionKey<>(false); @@ -142,7 +142,7 @@ public final class GraalOptions { public static final OptionKey VerifyPhases = new OptionKey<>(false); // Debug settings: - @Option(help = "", type = OptionType.Debug) + @Option(help = "Start tracing compiled GC barriers after N garbage collections (disabled if N <= 0).", type = OptionType.Debug) public static final OptionKey GCDebugStartCycle = new OptionKey<>(-1); @Option(help = "Perform platform dependent validation of the Java heap at returns", type = OptionType.Debug) @@ -206,9 +206,6 @@ public final class GraalOptions { @Option(help = "Generate position independent code", type = OptionType.Expert) public static final OptionKey GeneratePIC = new OptionKey<>(false); - @Option(help = "", type = OptionType.Expert) - public static final OptionKey CallArrayCopy = new OptionKey<>(true); - // Runtime settings @Option(help = "", type = OptionType.Expert) public static final OptionKey SupportJsrBytecodes = new OptionKey<>(true); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/SpeculativeExecutionAttacksMitigations.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/SpeculativeExecutionAttacksMitigations.java index d37c3dd0f67..7273bfeb8ff 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/SpeculativeExecutionAttacksMitigations.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/SpeculativeExecutionAttacksMitigations.java @@ -37,9 +37,9 @@ public enum SpeculativeExecutionAttacksMitigations { public static class Options { // @formatter:off - @Option(help = "Select a strategy to mitigate speculative execution attacks (e.g., SPECTRE)", type = OptionType.User) + @Option(help = "file:doc-files/MitigateSpeculativeExecutionAttacksHelp.txt") public static final EnumOptionKey MitigateSpeculativeExecutionAttacks = new EnumOptionKey<>(None); - @Option(help = "Use index masking after bounds check to mitigate speculative execution attacks", type = OptionType.User) + @Option(help = "Use index masking after bounds check to mitigate speculative execution attacks.", type = OptionType.User) public static final OptionKey UseIndexMasking = new OptionKey<>(false); // @formatter:on } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/cfg/AbstractBlockBase.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/cfg/AbstractBlockBase.java index 871390e99b0..20eca67c260 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/cfg/AbstractBlockBase.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/cfg/AbstractBlockBase.java @@ -24,6 +24,8 @@ package org.graalvm.compiler.core.common.cfg; +import java.util.Comparator; + public abstract class AbstractBlockBase> { protected int id; @@ -171,4 +173,13 @@ public abstract class AbstractBlockBase> { public int hashCode() { return id; } + + public static class BlockIdComparator implements Comparator> { + @Override + public int compare(AbstractBlockBase o1, AbstractBlockBase o2) { + return Integer.compare(o1.getId(), o2.getId()); + } + } + + public static final BlockIdComparator BLOCK_ID_COMPARATOR = new BlockIdComparator(); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/cfg/CFGVerifier.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/cfg/CFGVerifier.java index b251d372ac2..15c047d26aa 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/cfg/CFGVerifier.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/cfg/CFGVerifier.java @@ -121,7 +121,7 @@ public class CFGVerifier { } } - for (T block : loop.getExits()) { + for (T block : loop.getLoopExits()) { assert block.getId() >= loop.getHeader().getId(); Loop blockLoop = block.getLoop(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/cfg/Loop.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/cfg/Loop.java index 98705d8cc09..572105f4010 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/cfg/Loop.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/cfg/Loop.java @@ -25,19 +25,28 @@ package org.graalvm.compiler.core.common.cfg; +import static org.graalvm.compiler.core.common.cfg.AbstractBlockBase.BLOCK_ID_COMPARATOR; + import java.util.ArrayList; +import java.util.Collections; import java.util.List; public abstract class Loop> { private final Loop parent; - private final List> children; + private final ArrayList> children; private final int depth; private final int index; private final T header; - private final List blocks; - private final List exits; + private final ArrayList blocks; + private final ArrayList exits; + /** + * Natural exits, ignoring LoopExitNodes. + * + * @see #getNaturalExits() + */ + private final ArrayList naturalExits; protected Loop(Loop parent, int index, T header) { this.parent = parent; @@ -51,6 +60,7 @@ public abstract class Loop> { this.blocks = new ArrayList<>(); this.children = new ArrayList<>(); this.exits = new ArrayList<>(); + this.naturalExits = new ArrayList<>(); } public abstract long numBackedges(); @@ -84,12 +94,87 @@ public abstract class Loop> { return blocks; } - public List getExits() { + /** + * Returns the loop exits. + * + * This might be a conservative set: before framestate assignment it matches the LoopExitNodes + * even if earlier blocks could be considered as exits. After framestate assignments, this is + * the same as {@link #getNaturalExits()}. + * + *

+ * LoopExitNodes are inserted in the control-flow during parsing and are natural exits: they are + * the earliest block at which we are guaranteed to have exited the loop. However, after some + * transformations of the graph, the natural exit might go up but the LoopExitNodes are not + * updated. + *

+ * + *

+ * For example in: + * + * 

+     * for (int i = 0; i < N; i++) {
+     *     if (c) {
+     *         // Block 1
+     *         if (dummy) {
+     *             // ...
+     *         } else {
+     *             // ...
+     *         }
+     *         if (b) {
+     *             continue;
+     *         } else {
+     *             // Block 2
+     *             // LoopExitNode
+     *             break;
+     *         }
+     *     }
+     * }
+     *
+ * + * After parsing, the natural exits match the LoopExitNode: Block 2 is a natural exit and has a + * LoopExitNode. If the {@code b} condition gets canonicalized to {@code false}, the natural + * exit moves to Block 1 while the LoopExitNode remains in Block 2. In such a scenario, + * {@code getLoopExits()} will contain block 2 while {@link #getNaturalExits()} will contain + * block 1. + * + * + * @see #getNaturalExits() + */ + public List getLoopExits() { return exits; } - public void addExit(T t) { - exits.add(t); + public boolean isLoopExit(T block) { + assert isSorted(exits); + return Collections.binarySearch(exits, block, BLOCK_ID_COMPARATOR) >= 0; + } + + /** + * Returns the natural exit points: these are the earliest block that are guaranteed to never + * reach a back-edge. + * + * This can not be used in the context of preserving or using loop-closed form. + * + * @see #getLoopExits() + */ + public ArrayList getNaturalExits() { + return naturalExits; + } + + public boolean isNaturalExit(T block) { + assert isSorted(naturalExits); + return Collections.binarySearch(naturalExits, block, BLOCK_ID_COMPARATOR) >= 0; + } + + private static > boolean isSorted(List list) { + int lastId = Integer.MIN_VALUE; + for (AbstractBlockBase block : list) { + if (block.getId() < lastId) { + return false; + } + lastId = block.getId(); + } + return true; } /** @@ -115,4 +200,9 @@ public abstract class Loop> { public int hashCode() { return index + depth * 31; } + + @Override + public boolean equals(Object obj) { + return super.equals(obj); + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/doc-files/MitigateSpeculativeExecutionAttacksHelp.txt b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/doc-files/MitigateSpeculativeExecutionAttacksHelp.txt new file mode 100644 index 00000000000..a291061c01c --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/doc-files/MitigateSpeculativeExecutionAttacksHelp.txt @@ -0,0 +1,9 @@ +Select a strategy to mitigate speculative execution attacks (e.g., SPECTRE). +The accepted values are: + None - No mitigations are used in JIT compiled code. + AllTargets - All branches are protected against speculative attacks. + This has a significant performance impact. + GuardTargets - Only branches that preserve Java memory safety are protected. + This has less performance impact than AllTargets. + NonDeoptGuardTargets - GuardTargets except that branches which deoptimize are not + protected since they can not be executed repeatedly. diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/AbstractObjectStamp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/AbstractObjectStamp.java index 82ae67aa663..671df48f18a 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/AbstractObjectStamp.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/AbstractObjectStamp.java @@ -48,6 +48,13 @@ public abstract class AbstractObjectStamp extends AbstractPointerStamp { this.exactType = exactType; } + @Override + public void accept(Visitor v) { + super.accept(v); + v.visitObject(type); + v.visitBoolean(exactType); + } + protected abstract AbstractObjectStamp copyWith(ResolvedJavaType newType, boolean newExactType, boolean newNonNull, boolean newAlwaysNull); @Override diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/AbstractPointerStamp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/AbstractPointerStamp.java index 737bc096fa4..20216652222 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/AbstractPointerStamp.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/AbstractPointerStamp.java @@ -36,6 +36,12 @@ public abstract class AbstractPointerStamp extends Stamp { private final boolean nonNull; private final boolean alwaysNull; + @Override + public void accept(Visitor v) { + v.visitBoolean(nonNull); + v.visitBoolean(alwaysNull); + } + protected AbstractPointerStamp(boolean nonNull, boolean alwaysNull) { this.nonNull = nonNull; this.alwaysNull = alwaysNull; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/IllegalStamp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/IllegalStamp.java index b13c5e831bf..f2228b922bd 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/IllegalStamp.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/IllegalStamp.java @@ -45,6 +45,10 @@ public final class IllegalStamp extends Stamp { private IllegalStamp() { } + @Override + public void accept(Visitor v) { + } + @Override public JavaKind getStackKind() { return JavaKind.Illegal; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/ObjectStamp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/ObjectStamp.java index fe0a831c10a..8db8896c9b2 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/ObjectStamp.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/ObjectStamp.java @@ -31,6 +31,7 @@ import jdk.vm.ci.meta.Constant; import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.MemoryAccessProvider; import jdk.vm.ci.meta.ResolvedJavaType; +import jdk.vm.ci.meta.UnresolvedJavaType; public class ObjectStamp extends AbstractObjectStamp { @@ -92,4 +93,48 @@ public class ObjectStamp extends AbstractObjectStamp { return null; } } + + /** + * Convert an ObjectStamp into a representation that can be resolved symbolically into the + * original stamp. + */ + @Override + public SymbolicJVMCIReference makeSymbolic() { + if (type() == null) { + return null; + } + return new SymbolicObjectStamp(this); + } + + static class SymbolicObjectStamp implements SymbolicJVMCIReference { + UnresolvedJavaType type; + private boolean exactType; + private boolean nonNull; + private boolean alwaysNull; + + SymbolicObjectStamp(ObjectStamp stamp) { + if (stamp.type() != null) { + type = UnresolvedJavaType.create(stamp.type().getName()); + } + exactType = stamp.isExactType(); + nonNull = stamp.nonNull(); + alwaysNull = stamp.alwaysNull(); + } + + @Override + public ObjectStamp resolve(ResolvedJavaType accessingClass) { + return new ObjectStamp(type != null ? type.resolve(accessingClass) : null, exactType, nonNull, alwaysNull); + } + + @Override + public String toString() { + return "SymbolicObjectStamp{" + + "declaringType=" + type + + ", exactType=" + exactType + + ", nonNull=" + nonNull + + ", alwaysNull=" + alwaysNull + + '}'; + } + } + } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/PrimitiveStamp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/PrimitiveStamp.java index 41775617dbb..c7070aea7ad 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/PrimitiveStamp.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/PrimitiveStamp.java @@ -39,6 +39,11 @@ public abstract class PrimitiveStamp extends ArithmeticStamp { this.bits = bits; } + @Override + public void accept(Visitor v) { + v.visitInt(bits); + } + /** * The width in bits of the value described by this stamp. */ diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/Stamp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/Stamp.java index dc163fe00a4..1fe5f813ce4 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/Stamp.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/Stamp.java @@ -26,6 +26,7 @@ package org.graalvm.compiler.core.common.type; import org.graalvm.compiler.core.common.LIRKind; import org.graalvm.compiler.core.common.spi.LIRKindTool; +import org.graalvm.compiler.serviceprovider.SpeculationReasonGroup.SpeculationContextObject; import jdk.vm.ci.meta.Constant; import jdk.vm.ci.meta.JavaKind; @@ -36,7 +37,7 @@ import jdk.vm.ci.meta.ResolvedJavaType; /** * A stamp is the basis for a type system. */ -public abstract class Stamp { +public abstract class Stamp implements SpeculationContextObject { protected Stamp() { } @@ -185,4 +186,15 @@ public abstract class Stamp { } return false; } + + /** + * Convert a Stamp into a representation that can be resolved symbolically into the original + * stamp. If this stamp contains no references to JVMCI types then simply return null. + */ + public SymbolicJVMCIReference makeSymbolic() { + return null; + } + + @Override + public abstract String toString(); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/StampPair.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/StampPair.java index 7d702bf9e59..be901c7d1ac 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/StampPair.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/StampPair.java @@ -24,6 +24,8 @@ package org.graalvm.compiler.core.common.type; +import java.util.Objects; + /** * A pair of stamp with one being the stamp that can be trusted and the other one being a guess that * needs a dynamic check to be used. @@ -63,4 +65,22 @@ public final class StampPair { return trustedStamp + " (unchecked=" + uncheckedStamp + ")"; } } + + @Override + public int hashCode() { + return trustedStamp.hashCode() + 11 + (uncheckedStamp != null ? uncheckedStamp.hashCode() : 0); + + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (obj instanceof StampPair) { + StampPair other = (StampPair) obj; + return trustedStamp.equals(other.trustedStamp) && Objects.equals(uncheckedStamp, other.uncheckedStamp); + } + return false; + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/SymbolicJVMCIReference.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/SymbolicJVMCIReference.java new file mode 100644 index 00000000000..0ccf49cbfd5 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/SymbolicJVMCIReference.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.core.common.type; + +import jdk.vm.ci.meta.ResolvedJavaType; + +/** + * This interface represents an object which contains a symbolic reference to a JVMCI type or method + * that can be converted back into the original object by looking up types relative to an + * {@code accessingClass}. + */ +public interface SymbolicJVMCIReference { + T resolve(ResolvedJavaType accessingClass); +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/VoidStamp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/VoidStamp.java index e7ba3ee6d16..7a953949e22 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/VoidStamp.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/type/VoidStamp.java @@ -42,6 +42,10 @@ public final class VoidStamp extends Stamp { private VoidStamp() { } + @Override + public void accept(Visitor v) { + } + @Override public Stamp unrestricted() { return this; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/UnsignedLong.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/UnsignedLong.java index 92a5378bc45..df0a7cf432b 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/UnsignedLong.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.common/src/org/graalvm/compiler/core/common/util/UnsignedLong.java @@ -43,6 +43,10 @@ public final class UnsignedLong { return Long.compareUnsigned(value, unsignedValue) < 0; } + public boolean isGreaterThan(long unsignedValue) { + return Long.compareUnsigned(value, unsignedValue) > 0; + } + public boolean isLessOrEqualTo(long unsignedValue) { return Long.compareUnsigned(value, unsignedValue) <= 0; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/CheckGraalInvariants.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/CheckGraalInvariants.java index 30e69e4b2df..19b4481e040 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/CheckGraalInvariants.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/CheckGraalInvariants.java @@ -157,6 +157,10 @@ public class CheckGraalInvariants extends GraalCompilerTest { if (className.equals("org.graalvm.compiler.serviceprovider.GraalServices$Lazy")) { return false; } + } else { + if (className.equals("jdk.vm.ci.services.JVMCIClassLoaderFactory")) { + return false; + } } return true; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ConditionalEliminationTestBase.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ConditionalEliminationTestBase.java index 5e53054c703..79a96b909d2 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ConditionalEliminationTestBase.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ConditionalEliminationTestBase.java @@ -33,7 +33,7 @@ import org.graalvm.compiler.phases.OptimisticOptimizations; import org.graalvm.compiler.phases.OptimisticOptimizations.Optimization; import org.graalvm.compiler.phases.common.CanonicalizerPhase; import org.graalvm.compiler.phases.common.ConditionalEliminationPhase; -import org.graalvm.compiler.phases.common.ConvertDeoptimizeToGuardPhase; +import org.graalvm.compiler.loop.phases.ConvertDeoptimizeToGuardPhase; import org.graalvm.compiler.phases.common.IterativeConditionalEliminationPhase; import org.graalvm.compiler.phases.common.LoweringPhase; import org.graalvm.compiler.phases.schedule.SchedulePhase; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/CountedLoopTest2.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/CountedLoopTest2.java new file mode 100644 index 00000000000..e5e9ed4895a --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/CountedLoopTest2.java @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.core.test; + +import org.graalvm.compiler.api.directives.GraalDirectives; +import org.graalvm.compiler.graph.Node; +import org.graalvm.compiler.graph.iterators.FilteredNodeIterable; +import org.graalvm.compiler.loop.LoopEx; +import org.graalvm.compiler.loop.LoopsData; +import org.graalvm.compiler.nodes.DeoptimizingNode; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions; +import org.junit.Assert; +import org.junit.Test; + +public class CountedLoopTest2 extends GraalCompilerTest { + public static float countedDeoptLoop0(int n) { + float v = 0; + for (int i = 0; i < n; i++) { + v += 2.1f * i; + GraalDirectives.controlFlowAnchor(); + } + GraalDirectives.deoptimizeAndInvalidate(); + return v; + } + + @Test + public void test0() { + test("countedDeoptLoop0"); + } + + public static float countedDeoptLoop1(int n) { + float v = 0; + for (int i = 0; i < n; i++) { + v += 2.1f * i; + GraalDirectives.controlFlowAnchor(); + } + if (v > 0) { + if (v / 55 < 3) { + v -= 2; + GraalDirectives.controlFlowAnchor(); + } else { + v += 6; + GraalDirectives.controlFlowAnchor(); + } + } else { + v += 1; + GraalDirectives.controlFlowAnchor(); + } + GraalDirectives.deoptimizeAndInvalidate(); + return v; + } + + @Test + public void test1() { + test("countedDeoptLoop1"); + } + + public static float countedDeoptLoop2(int n, float init) { + float v = init; + if (v > 0) { + if (v / 55 < 3) { + for (int i = 0; i < n; i++) { + v += 2.1f * i; + GraalDirectives.controlFlowAnchor(); + } + } else { + for (int i = 0; i < n; i++) { + v += 1.1f * i; + GraalDirectives.controlFlowAnchor(); + } + } + } else { + for (int i = 0; i < n; i++) { + v += -0.1f * i; + GraalDirectives.controlFlowAnchor(); + } + } + GraalDirectives.deoptimizeAndInvalidate(); + return v; + } + + @Test + public void test2() { + test("countedDeoptLoop2", 3); + } + + private void test(String methodName) { + test(methodName, 1); + } + + private void test(String methodName, int nLoops) { + StructuredGraph graph = parseEager(methodName, AllowAssumptions.YES); + LoopsData loops = new LoopsData(graph); + Assert.assertEquals(nLoops, loops.loops().size()); + for (LoopEx loop : loops.loops()) { + Assert.assertTrue(loop.detectCounted()); + } + + StructuredGraph finalGraph = getFinalGraph(methodName); + loops = new LoopsData(finalGraph); + Assert.assertEquals(nLoops, loops.loops().size()); + FilteredNodeIterable nonStartDeopts = finalGraph.getNodes().filter(n -> { + return n instanceof DeoptimizingNode.DeoptBefore && ((DeoptimizingNode.DeoptBefore) n).stateBefore().bci > 0; + }); + Assert.assertTrue(nonStartDeopts.isNotEmpty()); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/DumpPathTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/DumpPathTest.java index b0027a365c1..10abf6d580e 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/DumpPathTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/DumpPathTest.java @@ -31,6 +31,7 @@ import java.nio.file.Path; import jdk.internal.vm.compiler.collections.EconomicMap; import org.graalvm.compiler.debug.DebugOptions; +import org.graalvm.compiler.debug.DebugOptions.PrintGraphTarget; import org.graalvm.compiler.options.OptionKey; import org.graalvm.compiler.options.OptionValues; import org.junit.Test; @@ -52,7 +53,7 @@ public class DumpPathTest extends GraalCompilerTest { String[] extensions = new String[]{".cfg", ".bgv", ".graph-strings"}; EconomicMap, Object> overrides = OptionValues.newOptionMap(); overrides.put(DebugOptions.DumpPath, dumpDirectoryPath.toString()); - overrides.put(DebugOptions.PrintGraphFile, true); + overrides.put(DebugOptions.PrintGraph, PrintGraphTarget.File); overrides.put(DebugOptions.PrintCanonicalGraphStrings, true); overrides.put(DebugOptions.Dump, "*"); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GraalCompilerTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GraalCompilerTest.java index ac3aba0ce5c..7955472cf60 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GraalCompilerTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GraalCompilerTest.java @@ -111,7 +111,7 @@ import org.graalvm.compiler.phases.OptimisticOptimizations; import org.graalvm.compiler.phases.Phase; import org.graalvm.compiler.phases.PhaseSuite; import org.graalvm.compiler.phases.common.CanonicalizerPhase; -import org.graalvm.compiler.phases.common.ConvertDeoptimizeToGuardPhase; +import org.graalvm.compiler.loop.phases.ConvertDeoptimizeToGuardPhase; import org.graalvm.compiler.phases.common.inlining.InliningPhase; import org.graalvm.compiler.phases.common.inlining.info.InlineInfo; import org.graalvm.compiler.phases.common.inlining.policy.GreedyInliningPolicy; @@ -899,11 +899,21 @@ public abstract class GraalCompilerTest extends GraalTest { return actual; } + private static final List> C2_OMIT_STACK_TRACE_IN_FAST_THROW_EXCEPTIONS = Arrays.asList( + ArithmeticException.class, + ArrayIndexOutOfBoundsException.class, + ArrayStoreException.class, + ClassCastException.class, + NullPointerException.class); + protected void assertEquals(Result expect, Result actual) { if (expect.exception != null) { Assert.assertTrue("expected " + expect.exception, actual.exception != null); Assert.assertEquals("Exception class", expect.exception.getClass(), actual.exception.getClass()); - Assert.assertEquals("Exception message", expect.exception.getMessage(), actual.exception.getMessage()); + // C2 can optimize out the stack trace and message in some cases + if (expect.exception.getMessage() != null || !C2_OMIT_STACK_TRACE_IN_FAST_THROW_EXCEPTIONS.contains(expect.exception.getClass())) { + Assert.assertEquals("Exception message", expect.exception.getMessage(), actual.exception.getMessage()); + } } else { if (actual.exception != null) { throw new AssertionError("expected " + expect.returnValue + " but got an exception", actual.exception); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GraphEncoderTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GraphEncoderTest.java index a5626670630..374b013f4d1 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GraphEncoderTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GraphEncoderTest.java @@ -30,16 +30,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import jdk.vm.ci.meta.ResolvedJavaMethod; - -import org.junit.Test; - import org.graalvm.compiler.nodes.EncodedGraph; import org.graalvm.compiler.nodes.GraphEncoder; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions; import org.graalvm.compiler.phases.common.CanonicalizerPhase; import org.graalvm.compiler.phases.tiers.PhaseContext; +import org.junit.Test; + +import jdk.vm.ci.meta.ResolvedJavaMethod; public class GraphEncoderTest extends GraalCompilerTest { @@ -80,7 +79,7 @@ public class GraphEncoderTest extends GraalCompilerTest { for (StructuredGraph originalGraph : originalGraphs) { EncodedGraph encodedGraph = new EncodedGraph(encoder.getEncoding(), startOffsets.get(originalGraph), encoder.getObjects(), encoder.getNodeClasses(), originalGraph); - GraphEncoder.verifyEncoding(originalGraph, encodedGraph, getTarget().arch); + encoder.verifyEncoding(originalGraph, encodedGraph); } } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GuardPrioritiesTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GuardPrioritiesTest.java index f304bb5ec29..61bca4184e1 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GuardPrioritiesTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/GuardPrioritiesTest.java @@ -35,6 +35,7 @@ import java.util.Iterator; import org.graalvm.compiler.api.directives.GraalDirectives; import org.graalvm.compiler.core.common.GraalOptions; import org.graalvm.compiler.graph.iterators.NodeIterable; +import org.graalvm.compiler.loop.phases.ConvertDeoptimizeToGuardPhase; import org.graalvm.compiler.nodes.GuardNode; import org.graalvm.compiler.nodes.ParameterNode; import org.graalvm.compiler.nodes.StructuredGraph; @@ -42,7 +43,6 @@ import org.graalvm.compiler.nodes.calc.IntegerLowerThanNode; import org.graalvm.compiler.nodes.calc.IsNullNode; import org.graalvm.compiler.nodes.spi.LoweringTool; import org.graalvm.compiler.phases.common.CanonicalizerPhase; -import org.graalvm.compiler.phases.common.ConvertDeoptimizeToGuardPhase; import org.graalvm.compiler.phases.common.FloatingReadPhase; import org.graalvm.compiler.phases.common.LoweringPhase; import org.graalvm.compiler.phases.schedule.SchedulePhase; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/IntegerDivRemConstantTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/IntegerDivRemConstantTest.java new file mode 100644 index 00000000000..633d1313c66 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/IntegerDivRemConstantTest.java @@ -0,0 +1,293 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, Arm Limited and affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + + +package org.graalvm.compiler.core.test; + +import org.junit.Test; + +public class IntegerDivRemConstantTest extends GraalCompilerTest { + public static int intDivPositiveConstant(int val) { + return val / 5; + } + + @Test + public void testIntDivPositiveConstant() { + test("intDivPositiveConstant", -10); + test("intDivPositiveConstant", 0); + test("intDivPositiveConstant", 4256); + test("intDivPositiveConstant", Integer.MAX_VALUE); + test("intDivPositiveConstant", Integer.MIN_VALUE); + } + + public static int intDivIntegerMax(int val) { + return val / Integer.MAX_VALUE; + } + + @Test + public void testIntDivIntegerMax() { + test("intDivIntegerMax", -10); + test("intDivIntegerMax", 0); + test("intDivIntegerMax", 4256); + test("intDivIntegerMax", Integer.MAX_VALUE); + test("intDivIntegerMax", Integer.MIN_VALUE); + } + + public static int intDivNegativeConstant(int val) { + return val / -1234; + } + + @Test + public void testIntDivNegativeConstant() { + test("intDivNegativeConstant", -123); + test("intDivNegativeConstant", 0); + test("intDivNegativeConstant", 123); + test("intDivNegativeConstant", Integer.MAX_VALUE); + test("intDivNegativeConstant", Integer.MIN_VALUE); + } + + public static int intDivIntegerMinOdd(int val) { + return val / (Integer.MIN_VALUE + 1); + } + + @Test + public void testIntDivIntegerMinOdd() { + test("intDivIntegerMinOdd", -123); + test("intDivIntegerMinOdd", 0); + test("intDivIntegerMinOdd", 123); + test("intDivIntegerMinOdd", Integer.MAX_VALUE); + test("intDivIntegerMinOdd", Integer.MIN_VALUE); + } + + public static long longDivPositiveConstant(long val) { + return val / 35170432; + } + + @Test + public void testLongDivPositiveConstant() { + test("longDivPositiveConstant", -1234L); + test("longDivPositiveConstant", 0L); + test("longDivPositiveConstant", 214423L); + test("longDivPositiveConstant", Long.MAX_VALUE); + test("longDivPositiveConstant", Long.MIN_VALUE); + } + + public static long longDivLongMax(long val) { + return val / Long.MAX_VALUE; + } + + @Test + public void testLongDivLongMax() { + test("longDivLongMax", -1234L); + test("longDivLongMax", 0L); + test("longDivLongMax", 214423L); + test("longDivLongMax", Long.MAX_VALUE); + test("longDivLongMax", Long.MIN_VALUE); + } + + public static long longDivNegativeConstant(long val) { + return val / -413; + } + + @Test + public void testLongDivNegativeConstant() { + test("longDivNegativeConstant", -43L); + test("longDivNegativeConstant", 0L); + test("longDivNegativeConstant", 147065L); + test("longDivNegativeConstant", Long.MAX_VALUE); + test("longDivNegativeConstant", Long.MIN_VALUE); + } + + public static long longDivLongMinOdd(long val) { + return val / (Long.MIN_VALUE + 1); + } + + @Test + public void testLongDivLongMinOdd() { + test("longDivLongMinOdd", -1234L); + test("longDivLongMinOdd", 0L); + test("longDivLongMinOdd", 214423L); + test("longDivLongMinOdd", Long.MAX_VALUE); + test("longDivLongMinOdd", Long.MIN_VALUE); + } + + public static int intRemPositiveConstant(int val) { + return val % 139968; + } + + @Test + public void testIntRemPositiveConstant() { + test("intRemPositiveConstant", -10); + test("intRemPositiveConstant", 0); + test("intRemPositiveConstant", 4256); + test("intRemPositiveConstant", Integer.MAX_VALUE); + test("intRemPositiveConstant", Integer.MIN_VALUE); + } + + public static int intRemNegativeConstant(int val) { + return val % -139968; + } + + @Test + public void testIntRemNegativeConstant() { + test("intRemNegativeConstant", -10); + test("intRemNegativeConstant", 0); + test("intRemNegativeConstant", 4256); + test("intRemNegativeConstant", Integer.MAX_VALUE); + test("intRemNegativeConstant", Integer.MIN_VALUE); + } + + @SuppressWarnings("divzero") + public static int intRemZero(int val) { + return val % 0; + } + + @Test + public void testIntRemZero() { + test("intRemZero", -10); + test("intRemZero", 0); + test("intRemZero", 4256); + test("intRemZero", Integer.MAX_VALUE); + test("intRemZero", Integer.MIN_VALUE); + } + + public static int intRemMax(int val) { + return val % Integer.MAX_VALUE; + } + + @Test + public void testIntRemMax() { + test("intRemMax", -10); + test("intRemMax", 0); + test("intRemMax", 4256); + test("intRemMax", Integer.MAX_VALUE); + test("intRemMax", Integer.MIN_VALUE); + } + + public static int intRemMin(int val) { + return val % Integer.MIN_VALUE; + } + + @Test + public void testIntRemMin() { + test("intRemMin", -10); + test("intRemMin", 0); + test("intRemMin", 4256); + test("intRemMin", Integer.MAX_VALUE); + test("intRemMin", Integer.MIN_VALUE); + } + + public static long longRemPositiveConstant(long val) { + return val % 35170432; + } + + public static int intRemPowerOf2(int val) { + return val % 4; + } + + @Test + public void testIntRemPowerOf2() { + test("intRemPowerOf2", -10); + test("intRemPowerOf2", 0); + test("intRemPowerOf2", 4256); + test("intRemPowerOf2", Integer.MAX_VALUE); + test("intRemPowerOf2", Integer.MIN_VALUE); + } + + @Test + public void testLongRemPositiveConstant() { + test("longRemPositiveConstant", -1234L); + test("longRemPositiveConstant", 0L); + test("longRemPositiveConstant", 214423L); + test("longRemPositiveConstant", Long.MAX_VALUE); + test("longRemPositiveConstant", Long.MIN_VALUE); + } + + public static long longRemNegativeConstant(long val) { + return val % -413; + } + + @Test + public void testLongRemNegativeConstant() { + test("longRemNegativeConstant", -43L); + test("longRemNegativeConstant", 0L); + test("longRemNegativeConstant", 147065L); + test("longRemNegativeConstant", Long.MAX_VALUE); + test("longRemNegativeConstant", Long.MIN_VALUE); + } + + @SuppressWarnings("divzero") + public static long longRemZero(long val) { + return val % 0; + } + + @Test + public void testLongRemZero() { + test("longRemZero", -43L); + test("longRemZero", 0L); + test("longRemZero", 147065L); + test("longRemZero", Long.MAX_VALUE); + test("longRemZero", Long.MIN_VALUE); + } + + public static long longRemMax(long val) { + return val % Long.MAX_VALUE; + } + + @Test + public void testLongRemMax() { + test("longRemMax", -43L); + test("longRemMax", 0L); + test("longRemMax", 147065L); + test("longRemMax", Long.MAX_VALUE); + test("longRemMax", Long.MIN_VALUE); + } + + public static long longRemMin(long val) { + return val % Long.MIN_VALUE; + } + + @Test + public void testLongRemMin() { + test("longRemMin", -43L); + test("longRemMin", 0L); + test("longRemMin", 147065L); + test("longRemMin", Long.MAX_VALUE); + test("longRemMin", Long.MIN_VALUE); + } + + public static long longRemPowerOf2(long val) { + return val % 4L; + } + + @Test + public void testLongRemPowerOf2() { + test("longRemPowerOf2", -43L); + test("longRemPowerOf2", 0L); + test("longRemPowerOf2", 147065L); + test("longRemPowerOf2", Long.MAX_VALUE); + test("longRemPowerOf2", Long.MIN_VALUE); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/MemoryScheduleTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/MemoryScheduleTest.java index 779da296377..a5fca846bab 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/MemoryScheduleTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/MemoryScheduleTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/NestedLoopTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/NestedLoopTest.java index 4954a4031ff..23420e121cf 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/NestedLoopTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/NestedLoopTest.java @@ -161,9 +161,9 @@ public class NestedLoopTest extends GraalCompilerTest { Assert.assertTrue(containsDirect(innerMostLoop, d, cfg)); Assert.assertTrue(contains(rootLoop, d, cfg)); Assert.assertTrue(contains(nestedLoop, d, cfg)); - Assert.assertEquals(rootExits, rootLoop.getExits().size()); - Assert.assertEquals(nestedExits, nestedLoop.getExits().size()); - Assert.assertEquals(innerExits, innerMostLoop.getExits().size()); + Assert.assertEquals(rootExits, rootLoop.getLoopExits().size()); + Assert.assertEquals(nestedExits, nestedLoop.getLoopExits().size()); + Assert.assertEquals(innerExits, innerMostLoop.getLoopExits().size()); debug.dump(DebugContext.BASIC_LEVEL, graph, "Graph"); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/SwitchCanonicalizerTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/SwitchCanonicalizerTest.java new file mode 100644 index 00000000000..5cf5db820a2 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/SwitchCanonicalizerTest.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.core.test; + +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.extended.IntegerSwitchNode; +import org.graalvm.compiler.phases.common.CanonicalizerPhase; +import org.junit.Test; + +public class SwitchCanonicalizerTest extends GraalCompilerTest { + + public int divByPowerOf2(int n) { + switch (n / 8) { + case Integer.MAX_VALUE / 8 + 1: + return hashCode(); + default: + return 1; + } + } + + @Test + public void testDivByPowerOf2() { + shouldFoldSwitch("divByPowerOf2"); + } + + public int divByNonPowerOf2(int n) { + switch (n / 7) { + case Integer.MAX_VALUE / 7 + 1: + return hashCode(); + default: + return 1; + } + } + + @Test + public void testDivByNonPowerOf2() { + shouldFoldSwitch("divByNonPowerOf2"); + } + + public int remByPowerOf2(int n) { + switch (n % 8) { + case 9: + return hashCode(); + default: + return 1; + } + } + + @Test + public void testRemByPowerOf2() { + shouldFoldSwitch("remByPowerOf2"); + } + + public int remByPowerOf2PositiveX(int n) { + int n0 = n > 0 ? 8 : 9; + switch (n0 % 8) { + case 9: + return hashCode(); + default: + return 1; + } + } + + @Test + public void testRemByPowerOf2PositiveX() { + shouldFoldSwitch("remByPowerOf2PositiveX"); + } + + public int remByPowerOf2NegativeX(int n) { + int n0 = n > 0 ? -8 : -9; + switch (n0 % 8) { + case 9: + return hashCode(); + default: + return 1; + } + } + + @Test + public void testRemByPowerOf2NegativeX() { + shouldFoldSwitch("remByPowerOf2NegativeX"); + } + + public int remByNonPowerOf2(int n) { + switch (n % 7) { + case 9: + return hashCode(); + default: + return 1; + } + } + + @Test + public void testRemByNonPowerOf2() { + shouldFoldSwitch("remByNonPowerOf2"); + } + + private void shouldFoldSwitch(String methodName) { + StructuredGraph graph = parseForCompile(getResolvedJavaMethod(methodName)); + new CanonicalizerPhase().apply(graph, getDefaultHighTierContext()); + assertFalse(graph.hasNode(IntegerSwitchNode.TYPE)); + } + +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/VerifyCallerSensitiveMethods.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/VerifyCallerSensitiveMethods.java index 045e67e9de5..91592b87f75 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/VerifyCallerSensitiveMethods.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/VerifyCallerSensitiveMethods.java @@ -24,7 +24,7 @@ package org.graalvm.compiler.core.test; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import java.lang.annotation.Annotation; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/VerifyDebugUsage.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/VerifyDebugUsage.java index 8cd9f1e8a30..846e5872285 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/VerifyDebugUsage.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/VerifyDebugUsage.java @@ -150,7 +150,8 @@ public class VerifyDebugUsage extends VerifyPhase { "org.graalvm.compiler.truffle.compiler.TruffleCompilerImpl.compilePEGraph", "org.graalvm.compiler.core.test.VerifyDebugUsageTest$ValidDumpUsagePhase.run", "org.graalvm.compiler.core.test.VerifyDebugUsageTest$InvalidConcatDumpUsagePhase.run", - "org.graalvm.compiler.core.test.VerifyDebugUsageTest$InvalidDumpUsagePhase.run")); + "org.graalvm.compiler.core.test.VerifyDebugUsageTest$InvalidDumpUsagePhase.run", + "org.graalvm.compiler.hotspot.SymbolicSnippetEncoder.verifySnippetEncodeDecode")); /** * The set of methods allowed to call a {@code Debug.dump(...)} method with the {@code level} @@ -165,7 +166,8 @@ public class VerifyDebugUsage extends VerifyPhase { "org.graalvm.compiler.core.GraalCompiler.emitFrontEnd", "org.graalvm.compiler.phases.BasePhase.dumpAfter", "org.graalvm.compiler.replacements.ReplacementsImpl$GraphMaker.makeGraph", - "org.graalvm.compiler.replacements.SnippetTemplate.instantiate")); + "org.graalvm.compiler.replacements.SnippetTemplate.instantiate", + "org.graalvm.compiler.hotspot.SymbolicSnippetEncoder.verifySnippetEncodeDecode")); private void verifyParameters(StructuredGraph callerGraph, MethodCallTargetNode debugCallTarget, List args, ResolvedJavaType stringType, int startArgIdx, int varArgsIndex) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/backend/BackendTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/backend/BackendTest.java index f54dd56be0b..c5bc306235c 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/backend/BackendTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/backend/BackendTest.java @@ -25,6 +25,7 @@ package org.graalvm.compiler.core.test.backend; import org.graalvm.compiler.core.GraalCompiler; +import org.graalvm.compiler.core.gen.LIRCompilerBackend; import org.graalvm.compiler.core.test.GraalCompilerTest; import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.lir.gen.LIRGenerationResult; @@ -52,7 +53,7 @@ public abstract class BackendTest extends GraalCompilerTest { throw debug.handle(e); } - LIRGenerationResult lirGen = GraalCompiler.emitLIR(getBackend(), graph, null, null, createLIRSuites(graph.getOptions())); + LIRGenerationResult lirGen = LIRCompilerBackend.emitLIR(getBackend(), graph, null, null, createLIRSuites(graph.getOptions())); return lirGen; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/deopt/RethrowDeoptMaterializeTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/deopt/RethrowDeoptMaterializeTest.java index fdbd44af78f..61274b808ff 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/deopt/RethrowDeoptMaterializeTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/deopt/RethrowDeoptMaterializeTest.java @@ -46,7 +46,7 @@ public final class RethrowDeoptMaterializeTest extends GraalCompilerTest { @SuppressWarnings("sync-override") @Override public final Throwable fillInStackTrace() { - return null; + return this; } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ea/EscapeAnalysisTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ea/EscapeAnalysisTest.java index 0d1f171ac36..7160944c1e2 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ea/EscapeAnalysisTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/ea/EscapeAnalysisTest.java @@ -293,7 +293,7 @@ public class EscapeAnalysisTest extends EATestBase { @SuppressWarnings("sync-override") @Override public final Throwable fillInStackTrace() { - return null; + return this; } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/GraalCompiler.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/GraalCompiler.java index 81b52bb29ba..740eb88e4d9 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/GraalCompiler.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/GraalCompiler.java @@ -24,42 +24,18 @@ package org.graalvm.compiler.core; -import java.util.Collection; -import java.util.List; - -import jdk.internal.vm.compiler.collections.EconomicSet; import org.graalvm.compiler.code.CompilationResult; -import org.graalvm.compiler.core.LIRGenerationPhase.LIRGenerationContext; -import org.graalvm.compiler.core.common.GraalOptions; import org.graalvm.compiler.core.common.PermanentBailoutException; import org.graalvm.compiler.core.common.RetryableBailoutException; -import org.graalvm.compiler.core.common.alloc.ComputeBlockOrder; -import org.graalvm.compiler.core.common.alloc.RegisterAllocationConfig; -import org.graalvm.compiler.core.common.cfg.AbstractBlockBase; import org.graalvm.compiler.core.common.util.CompilationAlarm; import org.graalvm.compiler.core.target.Backend; -import org.graalvm.compiler.debug.CounterKey; import org.graalvm.compiler.debug.DebugCloseable; import org.graalvm.compiler.debug.DebugContext; -import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.debug.MethodFilter; import org.graalvm.compiler.debug.TimerKey; -import org.graalvm.compiler.lir.LIR; -import org.graalvm.compiler.lir.alloc.OutOfRegistersException; -import org.graalvm.compiler.lir.asm.CompilationResultBuilder; import org.graalvm.compiler.lir.asm.CompilationResultBuilderFactory; -import org.graalvm.compiler.lir.framemap.FrameMap; -import org.graalvm.compiler.lir.framemap.FrameMapBuilder; -import org.graalvm.compiler.lir.gen.LIRGenerationResult; -import org.graalvm.compiler.lir.gen.LIRGeneratorTool; -import org.graalvm.compiler.lir.phases.AllocationPhase.AllocationContext; import org.graalvm.compiler.lir.phases.LIRSuites; -import org.graalvm.compiler.lir.phases.PostAllocationOptimizationPhase.PostAllocationOptimizationContext; -import org.graalvm.compiler.lir.phases.PreAllocationOptimizationPhase.PreAllocationOptimizationContext; import org.graalvm.compiler.nodes.StructuredGraph; -import org.graalvm.compiler.nodes.StructuredGraph.ScheduleResult; -import org.graalvm.compiler.nodes.cfg.Block; -import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; import org.graalvm.compiler.phases.OptimisticOptimizations; import org.graalvm.compiler.phases.PhaseSuite; import org.graalvm.compiler.phases.common.DeadCodeEliminationPhase; @@ -70,17 +46,8 @@ import org.graalvm.compiler.phases.tiers.Suites; import org.graalvm.compiler.phases.tiers.TargetProvider; import org.graalvm.compiler.phases.util.Providers; -import jdk.vm.ci.code.RegisterConfig; -import jdk.vm.ci.code.TargetDescription; -import jdk.vm.ci.code.site.ConstantReference; -import jdk.vm.ci.code.site.DataPatch; -import jdk.vm.ci.meta.Assumptions; -import jdk.vm.ci.meta.JavaConstant; -import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.ProfilingInfo; -import jdk.vm.ci.meta.ResolvedJavaField; import jdk.vm.ci.meta.ResolvedJavaMethod; -import jdk.vm.ci.meta.VMConstant; /** * Static methods for orchestrating the compilation of a {@linkplain StructuredGraph graph}. @@ -89,9 +56,6 @@ public class GraalCompiler { private static final TimerKey CompilerTimer = DebugContext.timer("GraalCompiler").doc("Time spent in compilation (excludes code installation)."); private static final TimerKey FrontEnd = DebugContext.timer("FrontEnd").doc("Time spent processing HIR."); - private static final TimerKey EmitLIR = DebugContext.timer("EmitLIR").doc("Time spent generating LIR from HIR."); - private static final TimerKey EmitCode = DebugContext.timer("EmitCode").doc("Time spent generating machine code from LIR."); - private static final TimerKey BackEnd = DebugContext.timer("BackEnd").doc("Time spent in EmitLIR and EmitCode."); /** * Encapsulates all the inputs to a {@linkplain GraalCompiler#compile(Request) compilation}. @@ -178,7 +142,7 @@ public class GraalCompiler { assert !r.graph.isFrozen(); try (DebugContext.Scope s0 = debug.scope("GraalCompiler", r.graph, r.providers.getCodeCache()); DebugCloseable a = CompilerTimer.start(debug)) { emitFrontEnd(r.providers, r.backend, r.graph, r.graphBuilderSuite, r.optimisticOpts, r.profilingInfo, r.suites); - emitBackEnd(r.graph, null, r.installedCodeOwner, r.backend, r.compilationResult, r.factory, null, r.lirSuites); + r.backend.emitBackEnd(r.graph, null, r.installedCodeOwner, r.compilationResult, r.factory, null, r.lirSuites); if (r.verifySourcePositions) { assert r.graph.verifySourcePositions(true); } @@ -275,85 +239,6 @@ public class GraalCompiler { } } - @SuppressWarnings("try") - public static void emitBackEnd(StructuredGraph graph, Object stub, ResolvedJavaMethod installedCodeOwner, Backend backend, T compilationResult, - CompilationResultBuilderFactory factory, RegisterConfig registerConfig, LIRSuites lirSuites) { - DebugContext debug = graph.getDebug(); - try (DebugContext.Scope s = debug.scope("BackEnd", graph.getLastSchedule()); DebugCloseable a = BackEnd.start(debug)) { - LIRGenerationResult lirGen = null; - lirGen = emitLIR(backend, graph, stub, registerConfig, lirSuites); - try (DebugContext.Scope s2 = debug.scope("CodeGen", lirGen, lirGen.getLIR())) { - int bytecodeSize = graph.method() == null ? 0 : graph.getBytecodeSize(); - compilationResult.setHasUnsafeAccess(graph.hasUnsafeAccess()); - emitCode(backend, graph.getAssumptions(), graph.method(), graph.getMethods(), graph.getFields(), bytecodeSize, lirGen, compilationResult, installedCodeOwner, factory); - } catch (Throwable e) { - throw debug.handle(e); - } - } catch (Throwable e) { - throw debug.handle(e); - } finally { - graph.checkCancellation(); - } - } - - @SuppressWarnings("try") - public static LIRGenerationResult emitLIR(Backend backend, StructuredGraph graph, Object stub, RegisterConfig registerConfig, LIRSuites lirSuites) { - String registerPressure = GraalOptions.RegisterPressure.getValue(graph.getOptions()); - String[] allocationRestrictedTo = registerPressure == null ? null : registerPressure.split(","); - try { - return emitLIR0(backend, graph, stub, registerConfig, lirSuites, allocationRestrictedTo); - } catch (OutOfRegistersException e) { - if (allocationRestrictedTo != null) { - allocationRestrictedTo = null; - return emitLIR0(backend, graph, stub, registerConfig, lirSuites, allocationRestrictedTo); - } - /* If the re-execution fails we convert the exception into a "hard" failure */ - throw new GraalError(e); - } finally { - graph.checkCancellation(); - } - } - - @SuppressWarnings("try") - private static LIRGenerationResult emitLIR0(Backend backend, StructuredGraph graph, Object stub, RegisterConfig registerConfig, LIRSuites lirSuites, - String[] allocationRestrictedTo) { - DebugContext debug = graph.getDebug(); - try (DebugContext.Scope ds = debug.scope("EmitLIR"); DebugCloseable a = EmitLIR.start(debug)) { - assert !graph.hasValueProxies(); - ScheduleResult schedule = graph.getLastSchedule(); - Block[] blocks = schedule.getCFG().getBlocks(); - Block startBlock = schedule.getCFG().getStartBlock(); - assert startBlock != null; - assert startBlock.getPredecessorCount() == 0; - - AbstractBlockBase[] codeEmittingOrder = ComputeBlockOrder.computeCodeEmittingOrder(blocks.length, startBlock); - AbstractBlockBase[] linearScanOrder = ComputeBlockOrder.computeLinearScanOrder(blocks.length, startBlock); - LIR lir = new LIR(schedule.getCFG(), linearScanOrder, codeEmittingOrder, graph.getOptions(), graph.getDebug()); - - FrameMapBuilder frameMapBuilder = backend.newFrameMapBuilder(registerConfig); - LIRGenerationResult lirGenRes = backend.newLIRGenerationResult(graph.compilationId(), lir, frameMapBuilder, graph, stub); - LIRGeneratorTool lirGen = backend.newLIRGenerator(lirGenRes); - NodeLIRBuilderTool nodeLirGen = backend.newNodeLIRBuilder(graph, lirGen); - - // LIR generation - LIRGenerationContext context = new LIRGenerationContext(lirGen, nodeLirGen, graph, schedule); - new LIRGenerationPhase().apply(backend.getTarget(), lirGenRes, context); - - try (DebugContext.Scope s = debug.scope("LIRStages", nodeLirGen, lirGenRes, lir)) { - // Dump LIR along with HIR (the LIR is looked up from context) - debug.dump(DebugContext.BASIC_LEVEL, graph.getLastSchedule(), "After LIR generation"); - LIRGenerationResult result = emitLowLevel(backend.getTarget(), lirGenRes, lirGen, lirSuites, backend.newRegisterAllocationConfig(registerConfig, allocationRestrictedTo)); - return result; - } catch (Throwable e) { - throw debug.handle(e); - } - } catch (Throwable e) { - throw debug.handle(e); - } finally { - graph.checkCancellation(); - } - } - protected static String getCompilationUnitName(StructuredGraph graph, T compilationResult) { if (compilationResult != null && compilationResult.getName() != null) { return compilationResult.getName(); @@ -364,70 +249,4 @@ public class GraalCompiler { } return method.format("%H.%n(%p)"); } - - public static LIRGenerationResult emitLowLevel(TargetDescription target, LIRGenerationResult lirGenRes, LIRGeneratorTool lirGen, LIRSuites lirSuites, - RegisterAllocationConfig registerAllocationConfig) { - DebugContext debug = lirGenRes.getLIR().getDebug(); - PreAllocationOptimizationContext preAllocOptContext = new PreAllocationOptimizationContext(lirGen); - lirSuites.getPreAllocationOptimizationStage().apply(target, lirGenRes, preAllocOptContext); - debug.dump(DebugContext.BASIC_LEVEL, lirGenRes.getLIR(), "After PreAllocationOptimizationStage"); - - AllocationContext allocContext = new AllocationContext(lirGen.getSpillMoveFactory(), registerAllocationConfig); - lirSuites.getAllocationStage().apply(target, lirGenRes, allocContext); - debug.dump(DebugContext.BASIC_LEVEL, lirGenRes.getLIR(), "After AllocationStage"); - - PostAllocationOptimizationContext postAllocOptContext = new PostAllocationOptimizationContext(lirGen); - lirSuites.getPostAllocationOptimizationStage().apply(target, lirGenRes, postAllocOptContext); - debug.dump(DebugContext.BASIC_LEVEL, lirGenRes.getLIR(), "After PostAllocationOptimizationStage"); - - return lirGenRes; - } - - @SuppressWarnings("try") - public static void emitCode(Backend backend, Assumptions assumptions, ResolvedJavaMethod rootMethod, Collection inlinedMethods, EconomicSet accessedFields, - int bytecodeSize, LIRGenerationResult lirGenRes, - CompilationResult compilationResult, ResolvedJavaMethod installedCodeOwner, CompilationResultBuilderFactory factory) { - DebugContext debug = lirGenRes.getLIR().getDebug(); - try (DebugCloseable a = EmitCode.start(debug)) { - FrameMap frameMap = lirGenRes.getFrameMap(); - CompilationResultBuilder crb = backend.newCompilationResultBuilder(lirGenRes, frameMap, compilationResult, factory); - backend.emitCode(crb, lirGenRes.getLIR(), installedCodeOwner); - if (assumptions != null && !assumptions.isEmpty()) { - compilationResult.setAssumptions(assumptions.toArray()); - } - if (rootMethod != null) { - compilationResult.setMethods(rootMethod, inlinedMethods); - compilationResult.setFields(accessedFields); - compilationResult.setBytecodeSize(bytecodeSize); - } - crb.finish(); - if (debug.isCountEnabled()) { - List ldp = compilationResult.getDataPatches(); - JavaKind[] kindValues = JavaKind.values(); - CounterKey[] dms = new CounterKey[kindValues.length]; - for (int i = 0; i < dms.length; i++) { - dms[i] = DebugContext.counter("DataPatches-%s", kindValues[i]); - } - - for (DataPatch dp : ldp) { - JavaKind kind = JavaKind.Illegal; - if (dp.reference instanceof ConstantReference) { - VMConstant constant = ((ConstantReference) dp.reference).getConstant(); - if (constant instanceof JavaConstant) { - kind = ((JavaConstant) constant).getJavaKind(); - } - } - dms[kind.ordinal()].add(debug, 1); - } - - DebugContext.counter("CompilationResults").increment(debug); - DebugContext.counter("CodeBytesEmitted").add(debug, compilationResult.getTargetCodeSize()); - DebugContext.counter("InfopointsEmitted").add(debug, compilationResult.getInfopoints().size()); - DebugContext.counter("DataPatches").add(debug, ldp.size()); - DebugContext.counter("ExceptionHandlersEmitted").add(debug, compilationResult.getExceptionHandlers().size()); - } - - debug.dump(DebugContext.BASIC_LEVEL, compilationResult, "After code generation"); - } - } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/LIRCompilerBackend.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/LIRCompilerBackend.java new file mode 100644 index 00000000000..0e5c82e5bfb --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/LIRCompilerBackend.java @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.core.gen; + +import java.util.Collection; +import java.util.List; + +import jdk.internal.vm.compiler.collections.EconomicSet; +import org.graalvm.compiler.code.CompilationResult; +import org.graalvm.compiler.core.LIRGenerationPhase; +import org.graalvm.compiler.core.LIRGenerationPhase.LIRGenerationContext; +import org.graalvm.compiler.core.common.GraalOptions; +import org.graalvm.compiler.core.common.alloc.ComputeBlockOrder; +import org.graalvm.compiler.core.common.alloc.RegisterAllocationConfig; +import org.graalvm.compiler.core.common.cfg.AbstractBlockBase; +import org.graalvm.compiler.core.target.Backend; +import org.graalvm.compiler.debug.CounterKey; +import org.graalvm.compiler.debug.DebugCloseable; +import org.graalvm.compiler.debug.DebugContext; +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.debug.TimerKey; +import org.graalvm.compiler.lir.LIR; +import org.graalvm.compiler.lir.alloc.OutOfRegistersException; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; +import org.graalvm.compiler.lir.asm.CompilationResultBuilderFactory; +import org.graalvm.compiler.lir.framemap.FrameMap; +import org.graalvm.compiler.lir.gen.LIRGenerationResult; +import org.graalvm.compiler.lir.gen.LIRGeneratorTool; +import org.graalvm.compiler.lir.phases.AllocationPhase.AllocationContext; +import org.graalvm.compiler.lir.phases.LIRSuites; +import org.graalvm.compiler.lir.phases.PostAllocationOptimizationPhase.PostAllocationOptimizationContext; +import org.graalvm.compiler.lir.phases.PreAllocationOptimizationPhase.PreAllocationOptimizationContext; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.StructuredGraph.ScheduleResult; +import org.graalvm.compiler.nodes.cfg.Block; +import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; + +import jdk.vm.ci.code.RegisterConfig; +import jdk.vm.ci.code.TargetDescription; +import jdk.vm.ci.code.site.ConstantReference; +import jdk.vm.ci.code.site.DataPatch; +import jdk.vm.ci.meta.Assumptions; +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.ResolvedJavaField; +import jdk.vm.ci.meta.ResolvedJavaMethod; +import jdk.vm.ci.meta.SpeculationLog; +import jdk.vm.ci.meta.VMConstant; + +public class LIRCompilerBackend { + private static final TimerKey EmitLIR = DebugContext.timer("EmitLIR").doc("Time spent generating LIR from HIR."); + private static final TimerKey EmitCode = DebugContext.timer("EmitCode").doc("Time spent generating machine code from LIR."); + private static final TimerKey BackEnd = DebugContext.timer("BackEnd").doc("Time spent in EmitLIR and EmitCode."); + + @SuppressWarnings("try") + public static void emitBackEnd(StructuredGraph graph, Object stub, ResolvedJavaMethod installedCodeOwner, Backend backend, T compilationResult, + CompilationResultBuilderFactory factory, RegisterConfig registerConfig, LIRSuites lirSuites) { + DebugContext debug = graph.getDebug(); + try (DebugContext.Scope s = debug.scope("BackEnd", graph.getLastSchedule()); DebugCloseable a = BackEnd.start(debug)) { + LIRGenerationResult lirGen = null; + lirGen = emitLIR(backend, graph, stub, registerConfig, lirSuites); + try (DebugContext.Scope s2 = debug.scope("CodeGen", lirGen, lirGen.getLIR())) { + int bytecodeSize = graph.method() == null ? 0 : graph.getBytecodeSize(); + compilationResult.setHasUnsafeAccess(graph.hasUnsafeAccess()); + emitCode(backend, + graph.getAssumptions(), + graph.method(), + graph.getMethods(), + graph.getFields(), + graph.getSpeculationLog(), + bytecodeSize, + lirGen, + compilationResult, + installedCodeOwner, + factory); + } catch (Throwable e) { + throw debug.handle(e); + } + } catch (Throwable e) { + throw debug.handle(e); + } finally { + graph.checkCancellation(); + } + } + + @SuppressWarnings("try") + public static LIRGenerationResult emitLIR(Backend backend, StructuredGraph graph, Object stub, RegisterConfig registerConfig, LIRSuites lirSuites) { + String registerPressure = GraalOptions.RegisterPressure.getValue(graph.getOptions()); + String[] allocationRestrictedTo = registerPressure == null ? null : registerPressure.split(","); + try { + return emitLIR0(backend, graph, stub, registerConfig, lirSuites, allocationRestrictedTo); + } catch (OutOfRegistersException e) { + if (allocationRestrictedTo != null) { + allocationRestrictedTo = null; + return emitLIR0(backend, graph, stub, registerConfig, lirSuites, allocationRestrictedTo); + } + /* If the re-execution fails we convert the exception into a "hard" failure */ + throw new GraalError(e); + } finally { + graph.checkCancellation(); + } + } + + @SuppressWarnings("try") + private static LIRGenerationResult emitLIR0(Backend backend, StructuredGraph graph, Object stub, RegisterConfig registerConfig, LIRSuites lirSuites, + String[] allocationRestrictedTo) { + DebugContext debug = graph.getDebug(); + try (DebugContext.Scope ds = debug.scope("EmitLIR"); DebugCloseable a = EmitLIR.start(debug)) { + assert !graph.hasValueProxies(); + + ScheduleResult schedule = graph.getLastSchedule(); + Block[] blocks = schedule.getCFG().getBlocks(); + Block startBlock = schedule.getCFG().getStartBlock(); + assert startBlock != null; + assert startBlock.getPredecessorCount() == 0; + + AbstractBlockBase[] codeEmittingOrder = ComputeBlockOrder.computeCodeEmittingOrder(blocks.length, startBlock); + AbstractBlockBase[] linearScanOrder = ComputeBlockOrder.computeLinearScanOrder(blocks.length, startBlock); + LIR lir = new LIR(schedule.getCFG(), linearScanOrder, codeEmittingOrder, graph.getOptions(), graph.getDebug()); + + LIRGenerationProvider lirBackend = (LIRGenerationProvider) backend; + LIRGenerationResult lirGenRes = lirBackend.newLIRGenerationResult(graph.compilationId(), lir, registerConfig, graph, stub); + LIRGeneratorTool lirGen = lirBackend.newLIRGenerator(lirGenRes); + NodeLIRBuilderTool nodeLirGen = lirBackend.newNodeLIRBuilder(graph, lirGen); + + // LIR generation + LIRGenerationContext context = new LIRGenerationContext(lirGen, nodeLirGen, graph, schedule); + new LIRGenerationPhase().apply(backend.getTarget(), lirGenRes, context); + + try (DebugContext.Scope s = debug.scope("LIRStages", nodeLirGen, lirGenRes, lir)) { + // Dump LIR along with HIR (the LIR is looked up from context) + debug.dump(DebugContext.BASIC_LEVEL, graph.getLastSchedule(), "After LIR generation"); + LIRGenerationResult result = emitLowLevel(backend.getTarget(), lirGenRes, lirGen, lirSuites, backend.newRegisterAllocationConfig(registerConfig, allocationRestrictedTo)); + return result; + } catch (Throwable e) { + throw debug.handle(e); + } + } catch (Throwable e) { + throw debug.handle(e); + } finally { + graph.checkCancellation(); + } + } + + private static LIRGenerationResult emitLowLevel(TargetDescription target, LIRGenerationResult lirGenRes, LIRGeneratorTool lirGen, LIRSuites lirSuites, + RegisterAllocationConfig registerAllocationConfig) { + DebugContext debug = lirGenRes.getLIR().getDebug(); + PreAllocationOptimizationContext preAllocOptContext = new PreAllocationOptimizationContext(lirGen); + lirSuites.getPreAllocationOptimizationStage().apply(target, lirGenRes, preAllocOptContext); + debug.dump(DebugContext.BASIC_LEVEL, lirGenRes.getLIR(), "After PreAllocationOptimizationStage"); + + AllocationContext allocContext = new AllocationContext(lirGen.getSpillMoveFactory(), registerAllocationConfig); + lirSuites.getAllocationStage().apply(target, lirGenRes, allocContext); + debug.dump(DebugContext.BASIC_LEVEL, lirGenRes.getLIR(), "After AllocationStage"); + + PostAllocationOptimizationContext postAllocOptContext = new PostAllocationOptimizationContext(lirGen); + lirSuites.getPostAllocationOptimizationStage().apply(target, lirGenRes, postAllocOptContext); + debug.dump(DebugContext.BASIC_LEVEL, lirGenRes.getLIR(), "After PostAllocationOptimizationStage"); + + return lirGenRes; + } + + @SuppressWarnings("try") + public static void emitCode(Backend backend, + Assumptions assumptions, + ResolvedJavaMethod rootMethod, + Collection inlinedMethods, + EconomicSet accessedFields, + SpeculationLog speculationLog, + int bytecodeSize, + LIRGenerationResult lirGenRes, + CompilationResult compilationResult, + ResolvedJavaMethod installedCodeOwner, + CompilationResultBuilderFactory factory) { + DebugContext debug = lirGenRes.getLIR().getDebug(); + try (DebugCloseable a = EmitCode.start(debug)) { + LIRGenerationProvider lirBackend = (LIRGenerationProvider) backend; + + FrameMap frameMap = lirGenRes.getFrameMap(); + CompilationResultBuilder crb = lirBackend.newCompilationResultBuilder(lirGenRes, frameMap, compilationResult, factory); + lirBackend.emitCode(crb, lirGenRes.getLIR(), installedCodeOwner); + if (assumptions != null && !assumptions.isEmpty()) { + compilationResult.setAssumptions(assumptions.toArray()); + } + if (rootMethod != null) { + compilationResult.setMethods(rootMethod, inlinedMethods); + compilationResult.setFields(accessedFields); + compilationResult.setBytecodeSize(bytecodeSize); + } + if (speculationLog != null) { + compilationResult.setSpeculationLog(speculationLog); + } + crb.finish(); + if (debug.isCountEnabled()) { + List ldp = compilationResult.getDataPatches(); + JavaKind[] kindValues = JavaKind.values(); + CounterKey[] dms = new CounterKey[kindValues.length]; + for (int i = 0; i < dms.length; i++) { + dms[i] = DebugContext.counter("DataPatches-%s", kindValues[i]); + } + + for (DataPatch dp : ldp) { + JavaKind kind = JavaKind.Illegal; + if (dp.reference instanceof ConstantReference) { + VMConstant constant = ((ConstantReference) dp.reference).getConstant(); + if (constant instanceof JavaConstant) { + kind = ((JavaConstant) constant).getJavaKind(); + } + } + dms[kind.ordinal()].add(debug, 1); + } + + DebugContext.counter("CompilationResults").increment(debug); + DebugContext.counter("CodeBytesEmitted").add(debug, compilationResult.getTargetCodeSize()); + DebugContext.counter("InfopointsEmitted").add(debug, compilationResult.getInfopoints().size()); + DebugContext.counter("DataPatches").add(debug, ldp.size()); + DebugContext.counter("ExceptionHandlersEmitted").add(debug, compilationResult.getExceptionHandlers().size()); + } + + debug.dump(DebugContext.BASIC_LEVEL, compilationResult, "After code generation"); + } + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/LIRGenerationProvider.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/LIRGenerationProvider.java new file mode 100644 index 00000000000..533f7f6087b --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/LIRGenerationProvider.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.core.gen; + +import org.graalvm.compiler.code.CompilationResult; +import org.graalvm.compiler.core.common.CompilationIdentifier; +import org.graalvm.compiler.lir.LIR; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; +import org.graalvm.compiler.lir.asm.CompilationResultBuilderFactory; +import org.graalvm.compiler.lir.framemap.FrameMap; +import org.graalvm.compiler.lir.gen.LIRGenerationResult; +import org.graalvm.compiler.lir.gen.LIRGeneratorTool; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; + +import jdk.vm.ci.code.RegisterConfig; +import jdk.vm.ci.meta.ResolvedJavaMethod; + +/** + * Provides compiler backend-specific generation helpers for the {@link LIRCompilerBackend}. + */ +public interface LIRGenerationProvider { + LIRGeneratorTool newLIRGenerator(LIRGenerationResult lirGenRes); + + LIRGenerationResult newLIRGenerationResult(CompilationIdentifier compilationId, LIR lir, RegisterConfig registerConfig, StructuredGraph graph, + Object stub); + + NodeLIRBuilderTool newNodeLIRBuilder(StructuredGraph graph, LIRGeneratorTool lirGen); + + /** + * Creates the object used to fill in the details of a given compilation result. + */ + CompilationResultBuilder newCompilationResultBuilder(LIRGenerationResult lirGenResult, FrameMap frameMap, CompilationResult compilationResult, + CompilationResultBuilderFactory factory); + + /** + * Emits the code for a given graph. + * + * @param installedCodeOwner the method the compiled code will be associated with once + * installed. This argument can be null. + */ + void emitCode(CompilationResultBuilder crb, LIR lir, ResolvedJavaMethod installedCodeOwner); +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/NodeLIRBuilder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/NodeLIRBuilder.java index 5f242953771..88344f6b6eb 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/NodeLIRBuilder.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/gen/NodeLIRBuilder.java @@ -768,4 +768,13 @@ public abstract class NodeLIRBuilder implements NodeLIRBuilderTool, LIRGeneratio public LIRGeneratorTool getLIRGeneratorTool() { return gen; } + + @Override + public void emitReadExceptionObject(ValueNode node) { + LIRGeneratorTool lirGenTool = getLIRGeneratorTool(); + Value returnRegister = lirGenTool.getRegisterConfig().getReturnRegister(node.getStackKind()).asValue( + LIRKind.fromJavaKind(lirGenTool.target().arch, node.getStackKind())); + lirGenTool.emitIncomingValues(new Value[]{returnRegister}); + setResult(node, lirGenTool.emitMove(returnRegister)); + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/phases/HighTier.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/phases/HighTier.java index de50932e699..b7f476b6df4 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/phases/HighTier.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/phases/HighTier.java @@ -47,7 +47,7 @@ import org.graalvm.compiler.options.OptionType; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.PhaseSuite; import org.graalvm.compiler.phases.common.CanonicalizerPhase; -import org.graalvm.compiler.phases.common.ConvertDeoptimizeToGuardPhase; +import org.graalvm.compiler.loop.phases.ConvertDeoptimizeToGuardPhase; import org.graalvm.compiler.phases.common.DeadCodeEliminationPhase; import org.graalvm.compiler.phases.common.IncrementalCanonicalizerPhase; import org.graalvm.compiler.phases.common.IterativeConditionalEliminationPhase; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/phases/LowTier.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/phases/LowTier.java index cae232c4c1c..d5cf5f82aa1 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/phases/LowTier.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/phases/LowTier.java @@ -39,6 +39,7 @@ import org.graalvm.compiler.phases.common.DeadCodeEliminationPhase; import org.graalvm.compiler.phases.common.ExpandLogicPhase; import org.graalvm.compiler.phases.common.FixReadsPhase; import org.graalvm.compiler.phases.common.LoweringPhase; +import org.graalvm.compiler.phases.common.OptimizeDivPhase; import org.graalvm.compiler.phases.common.ProfileCompiledMethodsPhase; import org.graalvm.compiler.phases.common.PropagateDeoptimizeProbabilityPhase; import org.graalvm.compiler.phases.common.UseTrappingNullChecksPhase; @@ -72,6 +73,8 @@ public class LowTier extends PhaseSuite { appendPhase(new LoweringPhase(canonicalizer, LoweringTool.StandardLoweringStage.LOW_TIER)); + appendPhase(new OptimizeDivPhase()); + appendPhase(new ExpandLogicPhase()); appendPhase(new FixReadsPhase(true, new SchedulePhase(GraalOptions.StressTestEarlyReads.getValue(options) ? SchedulingStrategy.EARLIEST : SchedulingStrategy.LATEST_OUT_OF_LOOPS))); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/target/Backend.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/target/Backend.java index 3b4d814dfa1..d8a652d583e 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/target/Backend.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.core/src/org/graalvm/compiler/core/target/Backend.java @@ -26,24 +26,17 @@ package org.graalvm.compiler.core.target; import java.util.ArrayList; -import jdk.internal.vm.compiler.collections.EconomicSet; -import org.graalvm.compiler.asm.Assembler; import org.graalvm.compiler.code.CompilationResult; import org.graalvm.compiler.core.common.CompilationIdentifier; import org.graalvm.compiler.core.common.LIRKind; import org.graalvm.compiler.core.common.alloc.RegisterAllocationConfig; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.core.common.spi.ForeignCallsProvider; +import org.graalvm.compiler.core.gen.LIRCompilerBackend; import org.graalvm.compiler.debug.DebugContext; -import org.graalvm.compiler.lir.LIR; -import org.graalvm.compiler.lir.asm.CompilationResultBuilder; import org.graalvm.compiler.lir.asm.CompilationResultBuilderFactory; -import org.graalvm.compiler.lir.framemap.FrameMap; -import org.graalvm.compiler.lir.framemap.FrameMapBuilder; -import org.graalvm.compiler.lir.gen.LIRGenerationResult; -import org.graalvm.compiler.lir.gen.LIRGeneratorTool; +import org.graalvm.compiler.lir.phases.LIRSuites; import org.graalvm.compiler.nodes.StructuredGraph; -import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.tiers.SuitesProvider; import org.graalvm.compiler.phases.tiers.TargetProvider; @@ -54,7 +47,6 @@ import jdk.vm.ci.code.CodeCacheProvider; import jdk.vm.ci.code.CompilationRequest; import jdk.vm.ci.code.CompiledCode; import jdk.vm.ci.code.InstalledCode; -import jdk.vm.ci.code.Register; import jdk.vm.ci.code.RegisterConfig; import jdk.vm.ci.code.TargetDescription; import jdk.vm.ci.code.ValueKindFactory; @@ -62,7 +54,6 @@ import jdk.vm.ci.meta.ConstantReflectionProvider; import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.meta.ResolvedJavaMethod; -import jdk.vm.ci.meta.SpeculationLog; /** * Represents a compiler backend for Graal. @@ -116,12 +107,6 @@ public abstract class Backend implements TargetProvider, ValueKindFactory translateToCallerRegisters(EconomicSet calleeRegisters); - /** * Gets the compilation id for a given {@link ResolvedJavaMethod}. Returns * {@code CompilationIdentifier#INVALID_COMPILATION_ID} in case there is no such id. @@ -306,6 +271,15 @@ public abstract class Backend implements TargetProvider, ValueKindFactory extends OptionKey { - private final OptionKey replacement; - DeprecatedOptionKey(OptionKey replacement) { - super(replacement.getDefaultValue()); - this.replacement = replacement; - } + /** + * Values for the {@link DebugOptions#PrintGraph} option denoting where graphs dumped as a + * result of the {@link DebugOptions#Dump} option are sent. + */ + public enum PrintGraphTarget { + /** + * Dump graphs to files. + */ + File, - @Override - protected void onValueUpdate(EconomicMap, Object> values, T oldValue, T newValue) { - // Ideally we'd use TTY here but it may not yet be initialized. - System.err.printf("Warning: the %s option is deprecated - use %s instead%n", getName(), replacement.getName()); - replacement.update(values, newValue); - } + /** + * Dump graphs to the network. The network destination is specified by the + * {@link DebugOptions#PrintGraphHost} and {@link DebugOptions#PrintGraphPort} options. If a + * network connection cannot be opened, dumping falls back to {@link #File} dumping. + */ + Network, + + /** + * Do not dump graphs. + */ + Disable; } // @formatter:off @@ -118,7 +127,7 @@ public class DebugOptions { public static final OptionKey LogVerbose = new OptionKey<>(false); @Option(help = "The directory where various Graal dump files are written.") - public static final OptionKey DumpPath = new OptionKey<>("dumps"); + public static final OptionKey DumpPath = new OptionKey<>("graal_dumps"); @Option(help = "Print the name of each dump file path as it's created.") public static final OptionKey ShowDumpFiles = new OptionKey<>(false); @@ -127,15 +136,30 @@ public class DebugOptions { @Option(help = "Enable dumping LIR, register allocation and code generation info to the C1Visualizer.", type = OptionType.Debug) public static final OptionKey PrintBackendCFG = new OptionKey<>(true); - @Option(help = "Enable dumping to the IdealGraphVisualizer.", type = OptionType.Debug) - public static final OptionKey PrintGraph = new OptionKey<>(true); - @Option(help = "Print graphs to files instead of sending them over the network.", type = OptionType.Debug) - public static final OptionKey PrintGraphFile = new OptionKey<>(false); + @Option(help = "file:doc-files/PrintGraphHelp.txt", type = OptionType.Debug) + public static final EnumOptionKey PrintGraph = new EnumOptionKey<>(PrintGraphTarget.File); + + @Option(help = "Setting to true sets PrintGraph=file, setting to false sets PrintGraph=network", type = OptionType.Debug) + public static final OptionKey PrintGraphFile = new OptionKey(true) { + @Override + protected void onValueUpdate(EconomicMap, Object> values, Boolean oldValue, Boolean newValue) { + PrintGraphTarget v = PrintGraph.getValueOrDefault(values); + if (newValue.booleanValue()) { + if (v != PrintGraphTarget.File) { + PrintGraph.update(values, PrintGraphTarget.File); + } + } else { + if (v != PrintGraphTarget.Network) { + PrintGraph.update(values, PrintGraphTarget.Network); + } + } + } + }; @Option(help = "Host part of the address to which graphs are dumped.", type = OptionType.Debug) public static final OptionKey PrintGraphHost = new OptionKey<>("127.0.0.1"); @Option(help = "Port part of the address to which graphs are dumped in binary format.", type = OptionType.Debug) - public static final OptionKey PrintBinaryGraphPort = new OptionKey<>(4445); + public static final OptionKey PrintGraphPort = new OptionKey<>(4445); @Option(help = "Schedule graphs as they are dumped.", type = OptionType.Debug) public static final OptionKey PrintGraphWithSchedule = new OptionKey<>(false); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/IgvDumpChannel.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/IgvDumpChannel.java index 53aa66597cb..0849c05b066 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/IgvDumpChannel.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/IgvDumpChannel.java @@ -24,6 +24,10 @@ package org.graalvm.compiler.debug; +import static org.graalvm.compiler.debug.DebugOptions.PrintGraphHost; +import static org.graalvm.compiler.debug.DebugOptions.PrintGraphPort; + +import java.io.File; import java.io.IOException; import java.io.InterruptedIOException; import java.net.InetSocketAddress; @@ -35,10 +39,12 @@ import java.nio.channels.WritableByteChannel; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.function.Supplier; -import static org.graalvm.compiler.debug.DebugOptions.PrintBinaryGraphPort; -import static org.graalvm.compiler.debug.DebugOptions.PrintGraphHost; + +import org.graalvm.compiler.debug.DebugOptions.PrintGraphTarget; import org.graalvm.compiler.options.OptionValues; +import jdk.vm.ci.common.NativeImageReinitialize; + final class IgvDumpChannel implements WritableByteChannel { private final Supplier pathProvider; private final OptionValues options; @@ -52,7 +58,8 @@ final class IgvDumpChannel implements WritableByteChannel { @Override public int write(ByteBuffer src) throws IOException { - return channel().write(src); + WritableByteChannel channel = channel(); + return channel == null ? 0 : channel.write(src); } @Override @@ -77,10 +84,13 @@ final class IgvDumpChannel implements WritableByteChannel { throw new IOException(); } if (sharedChannel == null) { - if (DebugOptions.PrintGraphFile.getValue(options)) { - sharedChannel = createFileChannel(pathProvider); - } else { + PrintGraphTarget target = DebugOptions.PrintGraph.getValue(options); + if (target == PrintGraphTarget.File) { + sharedChannel = createFileChannel(pathProvider, null); + } else if (target == PrintGraphTarget.Network) { sharedChannel = createNetworkChannel(pathProvider, options); + } else { + TTY.println("WARNING: Graph dumping requested but value of %s option is %s", DebugOptions.PrintGraph.getName(), PrintGraphTarget.Disable); } } return sharedChannel; @@ -88,10 +98,11 @@ final class IgvDumpChannel implements WritableByteChannel { private static WritableByteChannel createNetworkChannel(Supplier pathProvider, OptionValues options) throws IOException { String host = PrintGraphHost.getValue(options); - int port = PrintBinaryGraphPort.getValue(options); + int port = PrintGraphPort.getValue(options); try { WritableByteChannel channel = SocketChannel.open(new InetSocketAddress(host, port)); - TTY.println("Connected to the IGV on %s:%d", host, port); + String targetAnnouncement = String.format("Connected to the IGV on %s:%d", host, port); + maybeAnnounceTarget(targetAnnouncement); return channel; } catch (ClosedByInterruptException | InterruptedIOException e) { /* @@ -101,18 +112,39 @@ final class IgvDumpChannel implements WritableByteChannel { */ return null; } catch (IOException e) { - if (!DebugOptions.PrintGraphFile.hasBeenSet(options)) { - return createFileChannel(pathProvider); + String networkFailure = String.format("Could not connect to the IGV on %s:%d", host, port); + if (pathProvider != null) { + return createFileChannel(pathProvider, networkFailure); } else { - throw new IOException(String.format("Could not connect to the IGV on %s:%d", host, port), e); + throw new IOException(networkFailure, e); } } } - private static WritableByteChannel createFileChannel(Supplier pathProvider) throws IOException { + @NativeImageReinitialize private static String lastTargetAnnouncement; + + private static void maybeAnnounceTarget(String targetAnnouncement) { + if (!targetAnnouncement.equals(lastTargetAnnouncement)) { + // Ignore races - an extra announcement is ok + lastTargetAnnouncement = targetAnnouncement; + TTY.println(targetAnnouncement); + } + } + + private static WritableByteChannel createFileChannel(Supplier pathProvider, String networkFailure) throws IOException { Path path = pathProvider.get(); try { - return FileChannel.open(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE); + FileChannel channel = FileChannel.open(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE); + File dir = path.toFile(); + if (!dir.isDirectory()) { + dir = dir.getParentFile(); + } + if (networkFailure == null) { + maybeAnnounceTarget("Dumping IGV graphs in " + dir); + } else { + maybeAnnounceTarget(networkFailure + ". Dumping IGV graphs in " + dir); + } + return channel; } catch (IOException e) { throw new IOException(String.format("Failed to open %s to dump IGV graphs", path), e); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/PathUtilities.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/PathUtilities.java index db254152178..4917f7c9bde 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/PathUtilities.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/PathUtilities.java @@ -84,7 +84,7 @@ public class PathUtilities { private static final String ELLIPSIS = "..."; - static Path createUnique(OptionValues options, OptionKey baseNameOption, String id, String label, String ext, boolean createDirectory) throws IOException { + static Path createUnique(OptionValues options, OptionKey baseNameOption, String id, String label, String ext, boolean createMissingDirectory) throws IOException { String uniqueTag = ""; int dumpCounter = 1; String prefix; @@ -118,7 +118,7 @@ public class PathUtilities { Path dumpDir = DebugOptions.getDumpDirectory(options); Path result = Paths.get(dumpDir.toString(), fileName); try { - if (createDirectory) { + if (createMissingDirectory) { return Files.createDirectory(result); } else { try { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/doc-files/PrintGraphHelp.txt b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/doc-files/PrintGraphHelp.txt new file mode 100644 index 00000000000..cc5f9ea1b5d --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.debug/src/org/graalvm/compiler/debug/doc-files/PrintGraphHelp.txt @@ -0,0 +1,6 @@ +Where IdealGraphVisualizer graph dumps triggered by Dump or DumpOnError should be written. +The accepted values are: + File - Dump IGV graphs to the local file system (see DumpPath). + Network - Dump IGV graphs to the network destination specified by PrintGraphHost and PrintGraphPort. + If a network connection cannot be opened, dumping falls back to file dumping. + Disable - Do not dump IGV graphs. \ No newline at end of file diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph.test/src/org/graalvm/compiler/graph/test/graphio/GraphOutputTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph.test/src/org/graalvm/compiler/graph/test/graphio/GraphOutputTest.java new file mode 100644 index 00000000000..c9ccb191960 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph.test/src/org/graalvm/compiler/graph/test/graphio/GraphOutputTest.java @@ -0,0 +1,284 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.graph.test.graphio; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import org.graalvm.graphio.GraphOutput; +import org.graalvm.graphio.GraphStructure; +import org.junit.Test; + +public final class GraphOutputTest { + + @Test + @SuppressWarnings("static-method") + public void testWritableByteChannel() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + WritableByteChannel channel = Channels.newChannel(out); + ByteBuffer data = generateData(1 << 17); + GraphOutput graphOutput = GraphOutput.newBuilder(new MockGraphStructure()).protocolVersion(6, 0).embedded(true).build(channel); + try (GraphOutput closable = graphOutput) { + assertTrue(closable.isOpen()); + closable.write(data); + } + assertFalse(graphOutput.isOpen()); + assertArrayEquals(data.array(), out.toByteArray()); + } + + @Test + @SuppressWarnings("static-method") + public void testWriteDuringPrint() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + WritableByteChannel channel = Channels.newChannel(out); + class Action implements Runnable { + GraphOutput out; + + @Override + public void run() { + try { + ByteBuffer data = ByteBuffer.allocate(16); + data.limit(16); + out.write(data); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + } + Action action = new Action(); + try (GraphOutput graphOutput = GraphOutput.newBuilder(new MockGraphStructure(action)).protocolVersion(6, 0).build(channel)) { + action.out = graphOutput; + try { + graphOutput.print(new MockGraph(), Collections.emptyMap(), 0, "Mock Graph"); + fail("Expected IllegalStateException"); + } catch (IllegalStateException ise) { + // expected exception + } + } + } + + @Test + @SuppressWarnings("static-method") + public void testEmbeddedWritableByteChannel() throws IOException { + ByteArrayOutputStream expected = new ByteArrayOutputStream(); + WritableByteChannel expectedChannel = Channels.newChannel(expected); + Map properties = Collections.singletonMap("version.id", 1); + try (GraphOutput graphOutput = GraphOutput.newBuilder(new MockGraphStructure()).protocolVersion(6, 0).build(expectedChannel)) { + graphOutput.print(new MockGraph(), properties, 1, "Graph 1"); + graphOutput.write(ByteBuffer.allocate(0)); + graphOutput.print(new MockGraph(), properties, 2, "Graph 1"); + } + ByteArrayOutputStream embedded = new ByteArrayOutputStream(); + SharedWritableByteChannel embeddChannel = new SharedWritableByteChannel(Channels.newChannel(embedded)); + try { + try (GraphOutput baseOutput = GraphOutput.newBuilder(new MockGraphStructure()).protocolVersion(6, 0).build(embeddChannel)) { + try (GraphOutput embeddedOutput = GraphOutput.newBuilder(new MockGraphStructure()).protocolVersion(6, 0).embedded(true).build((WritableByteChannel) baseOutput)) { + embeddedOutput.print(new MockGraph(), properties, 1, "Graph 1"); + baseOutput.print(new MockGraph(), properties, 2, "Graph 1"); + } + } + } finally { + embeddChannel.realClose(); + } + assertArrayEquals(expected.toByteArray(), embedded.toByteArray()); + } + + private static ByteBuffer generateData(int size) { + ByteBuffer buffer = ByteBuffer.allocate(size); + for (int i = 0; i < size; i++) { + buffer.put(i, (byte) i); + } + buffer.limit(size); + return buffer; + } + + private static final class SharedWritableByteChannel implements WritableByteChannel { + + private final WritableByteChannel delegate; + + SharedWritableByteChannel(WritableByteChannel delegate) { + Objects.requireNonNull(delegate, "Delegate must be non null."); + this.delegate = delegate; + } + + @Override + public int write(ByteBuffer bb) throws IOException { + return delegate.write(bb); + } + + @Override + public boolean isOpen() { + return delegate.isOpen(); + } + + @Override + public void close() throws IOException { + } + + void realClose() throws IOException { + delegate.close(); + } + } + + private static final class MockGraphStructure implements GraphStructure { + + private final Runnable enterAction; + + MockGraphStructure() { + this.enterAction = null; + } + + MockGraphStructure(Runnable enterAction) { + this.enterAction = enterAction; + } + + @Override + public MockGraph graph(MockGraph currentGraph, Object obj) { + onEnter(); + return null; + } + + @Override + public Iterable nodes(MockGraph graph) { + onEnter(); + return Collections.emptySet(); + } + + @Override + public int nodesCount(MockGraph graph) { + onEnter(); + return 0; + } + + @Override + public int nodeId(Void node) { + onEnter(); + return 0; + } + + @Override + public boolean nodeHasPredecessor(Void node) { + onEnter(); + return false; + } + + @Override + public void nodeProperties(MockGraph graph, Void node, Map properties) { + onEnter(); + } + + @Override + public Void node(Object obj) { + onEnter(); + return null; + } + + @Override + public Void nodeClass(Object obj) { + onEnter(); + return null; + } + + @Override + public Void classForNode(Void node) { + onEnter(); + return null; + } + + @Override + public String nameTemplate(Void nodeClass) { + onEnter(); + return null; + } + + @Override + public Object nodeClassType(Void nodeClass) { + onEnter(); + return null; + } + + @Override + public Void portInputs(Void nodeClass) { + onEnter(); + return null; + } + + @Override + public Void portOutputs(Void nodeClass) { + onEnter(); + return null; + } + + @Override + public int portSize(Void port) { + onEnter(); + return 0; + } + + @Override + public boolean edgeDirect(Void port, int index) { + onEnter(); + return false; + } + + @Override + public String edgeName(Void port, int index) { + onEnter(); + return null; + } + + @Override + public Object edgeType(Void port, int index) { + onEnter(); + return null; + } + + @Override + public Collection edgeNodes(MockGraph graph, Void node, Void port, int index) { + onEnter(); + return null; + } + + private void onEnter() { + if (enterAction != null) { + enterAction.run(); + } + } + } + + private static final class MockGraph { + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/NodeClass.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/NodeClass.java index 190c31c2b26..b4e6aae8283 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/NodeClass.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.graph/src/org/graalvm/compiler/graph/NodeClass.java @@ -24,9 +24,7 @@ package org.graalvm.compiler.graph; -import static org.graalvm.compiler.core.common.Fields.translateInto; import static org.graalvm.compiler.debug.GraalError.shouldNotReachHere; -import static org.graalvm.compiler.graph.Edges.translateInto; import static org.graalvm.compiler.graph.Graph.isModificationCountsEnabled; import static org.graalvm.compiler.graph.InputEdges.translateInto; import static org.graalvm.compiler.graph.Node.WithAllEdges; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotBackend.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotBackend.java index 5ae51b83338..fb4717be2b6 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotBackend.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotBackend.java @@ -36,6 +36,7 @@ import static org.graalvm.compiler.core.common.GraalOptions.ZapStackOnMethodEntr import jdk.internal.vm.compiler.collections.EconomicSet; import org.graalvm.compiler.asm.Assembler; +import org.graalvm.compiler.asm.BranchTargetOutOfBoundsException; import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.asm.aarch64.AArch64Address; import org.graalvm.compiler.asm.aarch64.AArch64Assembler; @@ -47,6 +48,7 @@ import org.graalvm.compiler.core.common.CompilationIdentifier; import org.graalvm.compiler.core.common.LIRKind; import org.graalvm.compiler.core.common.alloc.RegisterAllocationConfig; import org.graalvm.compiler.core.common.spi.ForeignCallLinkage; +import org.graalvm.compiler.core.gen.LIRGenerationProvider; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; import org.graalvm.compiler.hotspot.HotSpotDataBuilder; import org.graalvm.compiler.hotspot.HotSpotGraalRuntimeProvider; @@ -86,21 +88,16 @@ import jdk.vm.ci.meta.ResolvedJavaMethod; /** * HotSpot AArch64 specific backend. */ -public class AArch64HotSpotBackend extends HotSpotHostBackend { +public class AArch64HotSpotBackend extends HotSpotHostBackend implements LIRGenerationProvider { public AArch64HotSpotBackend(GraalHotSpotVMConfig config, HotSpotGraalRuntimeProvider runtime, HotSpotProviders providers) { super(config, runtime, providers); } - @Override - public FrameMapBuilder newFrameMapBuilder(RegisterConfig registerConfig) { + private FrameMapBuilder newFrameMapBuilder(RegisterConfig registerConfig) { RegisterConfig registerConfigNonNull = registerConfig == null ? getCodeCache().getRegisterConfig() : registerConfig; - return new AArch64FrameMapBuilder(newFrameMap(registerConfigNonNull), getCodeCache(), registerConfigNonNull); - } - - @Override - public FrameMap newFrameMap(RegisterConfig registerConfig) { - return new AArch64FrameMap(getCodeCache(), registerConfig, this); + FrameMap frameMap = new AArch64FrameMap(getCodeCache(), registerConfigNonNull, this); + return new AArch64FrameMapBuilder(frameMap, getCodeCache(), registerConfigNonNull); } @Override @@ -109,8 +106,9 @@ public class AArch64HotSpotBackend extends HotSpotHostBackend { } @Override - public LIRGenerationResult newLIRGenerationResult(CompilationIdentifier compilationId, LIR lir, FrameMapBuilder frameMapBuilder, StructuredGraph graph, Object stub) { - return new HotSpotLIRGenerationResult(compilationId, lir, frameMapBuilder, makeCallingConvention(graph, (Stub) stub), stub, config.requiresReservedStackCheck(graph.getMethods())); + public LIRGenerationResult newLIRGenerationResult(CompilationIdentifier compilationId, LIR lir, RegisterConfig registerConfig, StructuredGraph graph, Object stub) { + return new HotSpotLIRGenerationResult(compilationId, lir, newFrameMapBuilder(registerConfig), makeCallingConvention(graph, (Stub) stub), stub, + config.requiresReservedStackCheck(graph.getMethods())); } @Override @@ -218,11 +216,6 @@ public class AArch64HotSpotBackend extends HotSpotHostBackend { } - @Override - protected Assembler createAssembler(FrameMap frameMap) { - return new AArch64MacroAssembler(getTarget()); - } - @Override public CompilationResultBuilder newCompilationResultBuilder(LIRGenerationResult lirGenRen, FrameMap frameMap, CompilationResult compilationResult, CompilationResultBuilderFactory factory) { HotSpotLIRGenerationResult gen = (HotSpotLIRGenerationResult) lirGenRen; @@ -230,7 +223,7 @@ public class AArch64HotSpotBackend extends HotSpotHostBackend { assert gen.getDeoptimizationRescueSlot() == null || frameMap.frameNeedsAllocating() : "method that can deoptimize must have a frame"; Stub stub = gen.getStub(); - Assembler masm = createAssembler(frameMap); + Assembler masm = new AArch64MacroAssembler(getTarget()); HotSpotFrameContext frameContext = new HotSpotFrameContext(stub != null); DataBuilder dataBuilder = new HotSpotDataBuilder(getCodeCache().getTarget()); @@ -252,11 +245,25 @@ public class AArch64HotSpotBackend extends HotSpotHostBackend { @Override public void emitCode(CompilationResultBuilder crb, LIR lir, ResolvedJavaMethod installedCodeOwner) { + Label verifiedStub = new Label(); + crb.buildLabelOffsets(lir); + try { + emitCode(crb, lir, installedCodeOwner, verifiedStub); + } catch (BranchTargetOutOfBoundsException e) { + // A branch estimation was wrong, now retry with conservative label ranges, this + // should always work + crb.setConservativeLabelRanges(); + crb.resetForEmittingCode(); + lir.resetLabels(); + verifiedStub.reset(); + emitCode(crb, lir, installedCodeOwner, verifiedStub); + } + } + + private void emitCode(CompilationResultBuilder crb, LIR lir, ResolvedJavaMethod installedCodeOwner, Label verifiedStub) { AArch64MacroAssembler masm = (AArch64MacroAssembler) crb.asm; FrameMap frameMap = crb.frameMap; RegisterConfig regConfig = frameMap.getRegisterConfig(); - Label verifiedStub = new Label(); - emitCodePrefix(crb, installedCodeOwner, masm, regConfig, verifiedStub); emitCodeBody(crb, lir, masm); emitCodeSuffix(crb, masm, frameMap); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotJumpToExceptionHandlerInCallerOp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotJumpToExceptionHandlerInCallerOp.java index 118cdcaa1bd..6725c0a7e62 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotJumpToExceptionHandlerInCallerOp.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.aarch64/src/org/graalvm/compiler/hotspot/aarch64/AArch64HotSpotJumpToExceptionHandlerInCallerOp.java @@ -37,7 +37,7 @@ import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.Opcode; import org.graalvm.compiler.lir.asm.CompilationResultBuilder; -import org.graalvm.compiler.serviceprovider.GraalServices; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import jdk.vm.ci.code.Register; import jdk.vm.ci.meta.AllocatableValue; @@ -71,7 +71,7 @@ public class AArch64HotSpotJumpToExceptionHandlerInCallerOp extends AArch64HotSp public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { leaveFrame(crb, masm, /* emitSafepoint */false, false); - if (GraalServices.JAVA_SPECIFICATION_VERSION < 8) { + if (JavaVersionUtil.JAVA_SPECIFICATION_VERSION < 8) { // Restore sp from fp if the exception PC is a method handle call site. try (ScratchRegister sc = masm.getScratchRegister()) { Register scratch = sc.getRegister(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/BinaryMathStubTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/BinaryMathStubTest.java new file mode 100644 index 00000000000..cc20c3963a9 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/BinaryMathStubTest.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.amd64.test; + +import static org.junit.Assume.assumeTrue; + +import java.util.ArrayList; +import java.util.List; + +import org.graalvm.compiler.api.test.Graal; +import org.graalvm.compiler.core.test.GraalCompilerTest; +import org.graalvm.compiler.runtime.RuntimeProvider; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import jdk.vm.ci.amd64.AMD64; +import jdk.vm.ci.code.Architecture; + +@RunWith(Parameterized.class) +public class BinaryMathStubTest extends GraalCompilerTest { + + @Parameterized.Parameters(name = "{0}") + public static List data() { + ArrayList ret = new ArrayList<>(); + ret.add(new Object[]{"pow"}); + return ret; + } + + private static final double[] inputs = {0.0D, Math.PI / 2, Math.PI, -1.0D, Double.MAX_VALUE, Double.MIN_VALUE, Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY}; + private final String stub; + + public BinaryMathStubTest(String stub) { + this.stub = stub; + } + + @Before + public void checkAMD64() { + Architecture arch = Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend().getTarget().arch; + assumeTrue("skipping AMD64 specific test", arch instanceof AMD64); + } + + public static double pow(double x, double y) { + return Math.pow(x, y); + } + + @Test + public void testStub() { + for (double x : inputs) { + for (double y : inputs) { + test(stub, x, y); + } + } + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/UnaryMathStubTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/UnaryMathStubTest.java new file mode 100644 index 00000000000..bf445a98d63 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64.test/src/org/graalvm/compiler/hotspot/amd64/test/UnaryMathStubTest.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.amd64.test; + +import static org.junit.Assume.assumeTrue; + +import java.util.ArrayList; +import java.util.List; + +import org.graalvm.compiler.api.test.Graal; +import org.graalvm.compiler.core.test.GraalCompilerTest; +import org.graalvm.compiler.runtime.RuntimeProvider; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import jdk.vm.ci.amd64.AMD64; +import jdk.vm.ci.code.Architecture; + +@RunWith(Parameterized.class) +public class UnaryMathStubTest extends GraalCompilerTest { + + @Parameterized.Parameters(name = "{0}") + public static List data() { + ArrayList ret = new ArrayList<>(); + ret.add(new Object[]{"sin"}); + ret.add(new Object[]{"cos"}); + ret.add(new Object[]{"tan"}); + ret.add(new Object[]{"exp"}); + ret.add(new Object[]{"log"}); + ret.add(new Object[]{"log10"}); + return ret; + } + + private static final double[] inputs = {0.0D, Math.PI / 2, Math.PI, -1.0D, Double.MAX_VALUE, Double.MIN_VALUE, Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY}; + private final String stub; + + public UnaryMathStubTest(String stub) { + this.stub = stub; + } + + @Before + public void checkAMD64() { + Architecture arch = Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend().getTarget().arch; + assumeTrue("skipping AMD64 specific test", arch instanceof AMD64); + } + + public static double sin(double value) { + return Math.sin(value); + } + + public static double cos(double value) { + return Math.cos(value); + } + + public static double tan(double value) { + return Math.tan(value); + } + + public static double exp(double value) { + return Math.exp(value); + } + + public static double log(double value) { + return Math.log(value); + } + + public static double log10(double value) { + return Math.log10(value); + } + + @Test + public void testStub() { + for (double input : inputs) { + test(stub, input); + } + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotAddressLowering.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotAddressLowering.java index 191f3a89d63..ef4423ead18 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotAddressLowering.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotAddressLowering.java @@ -199,7 +199,7 @@ public class AMD64HotSpotAddressLowering extends AMD64CompressAddressLowering { CountedLoopInfo countedLoopInfo = loop.counted(); IntegerStamp initStamp = (IntegerStamp) inductionVariable.initNode().stamp(NodeView.DEFAULT); if (initStamp.isPositive()) { - if (inductionVariable.isConstantExtremum()) { + if (inductionVariable.isConstantExtremum() && countedLoopInfo.counterNeverOverflows()) { long init = inductionVariable.constantInit(); long stride = inductionVariable.constantStride(); long extremum = inductionVariable.constantExtremum(); @@ -211,7 +211,9 @@ public class AMD64HotSpotAddressLowering extends AMD64CompressAddressLowering { } } } - if (countedLoopInfo.getCounter() == inductionVariable && inductionVariable.direction() == InductionVariable.Direction.Up && countedLoopInfo.getOverFlowGuard() != null) { + if (countedLoopInfo.getCounter() == inductionVariable && + inductionVariable.direction() == InductionVariable.Direction.Up && + (countedLoopInfo.getOverFlowGuard() != null || countedLoopInfo.counterNeverOverflows())) { return graph.unique(new ZeroExtendNode(input, INT_BITS, ADDRESS_BITS, true)); } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackend.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackend.java index 8cbb43b7a10..3602e6887b1 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackend.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackend.java @@ -43,7 +43,7 @@ import org.graalvm.compiler.core.amd64.AMD64NodeMatchRules; import org.graalvm.compiler.core.common.CompilationIdentifier; import org.graalvm.compiler.core.common.LIRKind; import org.graalvm.compiler.core.common.alloc.RegisterAllocationConfig; -import org.graalvm.compiler.core.target.Backend; +import org.graalvm.compiler.core.gen.LIRGenerationProvider; import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; import org.graalvm.compiler.hotspot.HotSpotDataBuilder; @@ -85,21 +85,16 @@ import jdk.vm.ci.meta.ResolvedJavaMethod; /** * HotSpot AMD64 specific backend. */ -public class AMD64HotSpotBackend extends HotSpotHostBackend { +public class AMD64HotSpotBackend extends HotSpotHostBackend implements LIRGenerationProvider { public AMD64HotSpotBackend(GraalHotSpotVMConfig config, HotSpotGraalRuntimeProvider runtime, HotSpotProviders providers) { super(config, runtime, providers); } - @Override - public FrameMapBuilder newFrameMapBuilder(RegisterConfig registerConfig) { + private FrameMapBuilder newFrameMapBuilder(RegisterConfig registerConfig) { RegisterConfig registerConfigNonNull = registerConfig == null ? getCodeCache().getRegisterConfig() : registerConfig; - return new AMD64FrameMapBuilder(newFrameMap(registerConfigNonNull), getCodeCache(), registerConfigNonNull); - } - - @Override - public FrameMap newFrameMap(RegisterConfig registerConfig) { - return new AMD64FrameMap(getCodeCache(), registerConfig, this); + FrameMap frameMap = new AMD64FrameMap(getCodeCache(), registerConfigNonNull, this); + return new AMD64FrameMapBuilder(frameMap, getCodeCache(), registerConfigNonNull); } @Override @@ -108,8 +103,9 @@ public class AMD64HotSpotBackend extends HotSpotHostBackend { } @Override - public LIRGenerationResult newLIRGenerationResult(CompilationIdentifier compilationId, LIR lir, FrameMapBuilder frameMapBuilder, StructuredGraph graph, Object stub) { - return new HotSpotLIRGenerationResult(compilationId, lir, frameMapBuilder, makeCallingConvention(graph, (Stub) stub), stub, config.requiresReservedStackCheck(graph.getMethods())); + public LIRGenerationResult newLIRGenerationResult(CompilationIdentifier compilationId, LIR lir, RegisterConfig registerConfig, StructuredGraph graph, Object stub) { + return new HotSpotLIRGenerationResult(compilationId, lir, newFrameMapBuilder(registerConfig), makeCallingConvention(graph, (Stub) stub), stub, + config.requiresReservedStackCheck(graph.getMethods())); } @Override @@ -195,11 +191,6 @@ public class AMD64HotSpotBackend extends HotSpotHostBackend { } } - @Override - protected Assembler createAssembler(FrameMap frameMap) { - return new AMD64MacroAssembler(getTarget()); - } - @Override public CompilationResultBuilder newCompilationResultBuilder(LIRGenerationResult lirGenRen, FrameMap frameMap, CompilationResult compilationResult, CompilationResultBuilderFactory factory) { // Omit the frame if the method: @@ -216,7 +207,7 @@ public class AMD64HotSpotBackend extends HotSpotHostBackend { boolean omitFrame = CanOmitFrame.getValue(options) && !frameMap.frameNeedsAllocating() && !lir.hasArgInCallerFrame() && !gen.hasForeignCall(); Stub stub = gen.getStub(); - Assembler masm = createAssembler(frameMap); + Assembler masm = new AMD64MacroAssembler(getTarget()); HotSpotFrameContext frameContext = new HotSpotFrameContext(stub != null, omitFrame); DataBuilder dataBuilder = new HotSpotDataBuilder(getCodeCache().getTarget()); CompilationResultBuilder crb = factory.createBuilder(getCodeCache(), getForeignCalls(), frameMap, masm, dataBuilder, frameContext, options, debug, compilationResult, Register.None); @@ -258,7 +249,7 @@ public class AMD64HotSpotBackend extends HotSpotHostBackend { /** * Emits the code prior to the verified entry point. * - * @param installedCodeOwner see {@link Backend#emitCode} + * @param installedCodeOwner see {@link LIRGenerationProvider#emitCode} */ public void emitCodePrefix(ResolvedJavaMethod installedCodeOwner, CompilationResultBuilder crb, AMD64MacroAssembler asm, RegisterConfig regConfig, Label verifiedEntry) { HotSpotProviders providers = getProviders(); @@ -309,14 +300,14 @@ public class AMD64HotSpotBackend extends HotSpotHostBackend { /** * Emits the code which starts at the verified entry point. * - * @param installedCodeOwner see {@link Backend#emitCode} + * @param installedCodeOwner see {@link LIRGenerationProvider#emitCode} */ public void emitCodeBody(ResolvedJavaMethod installedCodeOwner, CompilationResultBuilder crb, LIR lir) { crb.emit(lir); } /** - * @param installedCodeOwner see {@link Backend#emitCode} + * @param installedCodeOwner see {@link LIRGenerationProvider#emitCode} */ public void emitCodeSuffix(ResolvedJavaMethod installedCodeOwner, CompilationResultBuilder crb, AMD64MacroAssembler asm, FrameMap frameMap) { HotSpotProviders providers = getProviders(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackendFactory.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackendFactory.java index 92c5d6ded80..5921fef43ed 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackendFactory.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotBackendFactory.java @@ -25,7 +25,6 @@ package org.graalvm.compiler.hotspot.amd64; import static jdk.vm.ci.common.InitTimer.timer; -import static org.graalvm.compiler.hotspot.HotSpotBackend.Options.GraalArithmeticStubs; import java.util.ArrayList; import java.util.List; @@ -141,7 +140,7 @@ public class AMD64HotSpotBackendFactory implements HotSpotBackendFactory { replacements = createReplacements(options, p, snippetReflection, bytecodeProvider); } try (InitTimer rt = timer("create GraphBuilderPhase plugins")) { - plugins = createGraphBuilderPlugins(compilerConfiguration, config, options, target, constantReflection, foreignCalls, metaAccess, snippetReflection, replacements, wordTypes); + plugins = createGraphBuilderPlugins(compilerConfiguration, config, target, constantReflection, foreignCalls, metaAccess, snippetReflection, replacements, wordTypes); replacements.setGraphBuilderPlugins(plugins); } try (InitTimer rt = timer("create Suites provider")) { @@ -157,11 +156,11 @@ public class AMD64HotSpotBackendFactory implements HotSpotBackendFactory { } } - protected Plugins createGraphBuilderPlugins(CompilerConfiguration compilerConfiguration, GraalHotSpotVMConfig config, OptionValues options, TargetDescription target, + protected Plugins createGraphBuilderPlugins(CompilerConfiguration compilerConfiguration, GraalHotSpotVMConfig config, TargetDescription target, HotSpotConstantReflectionProvider constantReflection, HotSpotHostForeignCallsProvider foreignCalls, HotSpotMetaAccessProvider metaAccess, HotSpotSnippetReflectionProvider snippetReflection, HotSpotReplacementsImpl replacements, HotSpotWordTypes wordTypes) { Plugins plugins = HotSpotGraphBuilderPlugins.create(compilerConfiguration, config, wordTypes, metaAccess, constantReflection, snippetReflection, foreignCalls, replacements); - AMD64GraphBuilderPlugins.register(plugins, replacements.getDefaultReplacementBytecodeProvider(), (AMD64) target.arch, GraalArithmeticStubs.getValue(options), false); + AMD64GraphBuilderPlugins.register(plugins, replacements.getDefaultReplacementBytecodeProvider(), (AMD64) target.arch, false); return plugins; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotForeignCallsProvider.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotForeignCallsProvider.java index 58af7334c36..aadd591fde6 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotForeignCallsProvider.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotForeignCallsProvider.java @@ -30,6 +30,7 @@ import static jdk.vm.ci.hotspot.HotSpotCallingConventionType.NativeCall; import static jdk.vm.ci.meta.Value.ILLEGAL; import static org.graalvm.compiler.hotspot.HotSpotBackend.EXCEPTION_HANDLER; import static org.graalvm.compiler.hotspot.HotSpotBackend.EXCEPTION_HANDLER_IN_CALLER; +import static org.graalvm.compiler.hotspot.HotSpotBackend.Options.GraalArithmeticStubs; import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.JUMP_ADDRESS; import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Reexecutability.REEXECUTABLE; import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Reexecutability.REEXECUTABLE_ONLY_AFTER_EXCEPTION; @@ -38,10 +39,16 @@ import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Transition. import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Transition.LEAF_NOFP; import static org.graalvm.compiler.hotspot.replacements.CRC32CSubstitutions.UPDATE_BYTES_CRC32C; import static org.graalvm.compiler.hotspot.replacements.CRC32Substitutions.UPDATE_BYTES_CRC32; +import static org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode.BinaryOperation.POW; +import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.COS; +import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.EXP; +import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.LOG; +import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.LOG10; +import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.SIN; +import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.TAN; import static jdk.internal.vm.compiler.word.LocationIdentity.any; import org.graalvm.compiler.core.common.LIRKind; -import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkageImpl; import org.graalvm.compiler.hotspot.HotSpotGraalRuntimeProvider; @@ -62,14 +69,6 @@ import jdk.vm.ci.meta.Value; public class AMD64HotSpotForeignCallsProvider extends HotSpotHostForeignCallsProvider { - public static final ForeignCallDescriptor ARITHMETIC_SIN_STUB = new ForeignCallDescriptor("arithmeticSinStub", double.class, double.class); - public static final ForeignCallDescriptor ARITHMETIC_COS_STUB = new ForeignCallDescriptor("arithmeticCosStub", double.class, double.class); - public static final ForeignCallDescriptor ARITHMETIC_TAN_STUB = new ForeignCallDescriptor("arithmeticTanStub", double.class, double.class); - public static final ForeignCallDescriptor ARITHMETIC_EXP_STUB = new ForeignCallDescriptor("arithmeticExpStub", double.class, double.class); - public static final ForeignCallDescriptor ARITHMETIC_POW_STUB = new ForeignCallDescriptor("arithmeticPowStub", double.class, double.class, double.class); - public static final ForeignCallDescriptor ARITHMETIC_LOG_STUB = new ForeignCallDescriptor("arithmeticLogStub", double.class, double.class); - public static final ForeignCallDescriptor ARITHMETIC_LOG10_STUB = new ForeignCallDescriptor("arithmeticLog10Stub", double.class, double.class); - private final Value[] nativeABICallerSaveRegisters; public AMD64HotSpotForeignCallsProvider(HotSpotJVMCIRuntime jvmciRuntime, HotSpotGraalRuntimeProvider runtime, MetaAccessProvider metaAccess, CodeCacheProvider codeCache, @@ -93,14 +92,6 @@ public class AMD64HotSpotForeignCallsProvider extends HotSpotHostForeignCallsPro register(new HotSpotForeignCallLinkageImpl(EXCEPTION_HANDLER, 0L, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, exceptionCc, null, any())); register(new HotSpotForeignCallLinkageImpl(EXCEPTION_HANDLER_IN_CALLER, JUMP_ADDRESS, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, exceptionCc, null, any())); - link(new AMD64MathStub(ARITHMETIC_LOG_STUB, options, providers, registerStubCall(ARITHMETIC_LOG_STUB, LEAF, REEXECUTABLE, NO_LOCATIONS))); - link(new AMD64MathStub(ARITHMETIC_LOG10_STUB, options, providers, registerStubCall(ARITHMETIC_LOG10_STUB, LEAF, REEXECUTABLE, NO_LOCATIONS))); - link(new AMD64MathStub(ARITHMETIC_SIN_STUB, options, providers, registerStubCall(ARITHMETIC_SIN_STUB, LEAF, REEXECUTABLE, NO_LOCATIONS))); - link(new AMD64MathStub(ARITHMETIC_COS_STUB, options, providers, registerStubCall(ARITHMETIC_COS_STUB, LEAF, REEXECUTABLE, NO_LOCATIONS))); - link(new AMD64MathStub(ARITHMETIC_TAN_STUB, options, providers, registerStubCall(ARITHMETIC_TAN_STUB, LEAF, REEXECUTABLE, NO_LOCATIONS))); - link(new AMD64MathStub(ARITHMETIC_EXP_STUB, options, providers, registerStubCall(ARITHMETIC_EXP_STUB, LEAF, REEXECUTABLE, NO_LOCATIONS))); - link(new AMD64MathStub(ARITHMETIC_POW_STUB, options, providers, registerStubCall(ARITHMETIC_POW_STUB, LEAF, REEXECUTABLE, NO_LOCATIONS))); - if (config.useCRC32Intrinsics) { // This stub does callee saving registerForeignCall(UPDATE_BYTES_CRC32, config.updateBytesCRC32Stub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, any()); @@ -138,4 +129,19 @@ public class AMD64HotSpotForeignCallsProvider extends HotSpotHostForeignCallsPro return nativeABICallerSaveRegisters; } + @Override + protected void registerMathStubs(GraalHotSpotVMConfig hotSpotVMConfig, HotSpotProviders providers, OptionValues options) { + if (GraalArithmeticStubs.getValue(options)) { + link(new AMD64MathStub(SIN, options, providers, registerStubCall(SIN.foreignCallDescriptor, LEAF, REEXECUTABLE, NO_LOCATIONS))); + link(new AMD64MathStub(COS, options, providers, registerStubCall(COS.foreignCallDescriptor, LEAF, REEXECUTABLE, NO_LOCATIONS))); + link(new AMD64MathStub(TAN, options, providers, registerStubCall(TAN.foreignCallDescriptor, LEAF, REEXECUTABLE, NO_LOCATIONS))); + link(new AMD64MathStub(EXP, options, providers, registerStubCall(EXP.foreignCallDescriptor, LEAF, REEXECUTABLE, NO_LOCATIONS))); + link(new AMD64MathStub(LOG, options, providers, registerStubCall(LOG.foreignCallDescriptor, LEAF, REEXECUTABLE, NO_LOCATIONS))); + link(new AMD64MathStub(LOG10, options, providers, registerStubCall(LOG10.foreignCallDescriptor, LEAF, REEXECUTABLE, NO_LOCATIONS))); + link(new AMD64MathStub(POW, options, providers, registerStubCall(POW.foreignCallDescriptor, LEAF, REEXECUTABLE, NO_LOCATIONS))); + } else { + super.registerMathStubs(hotSpotVMConfig, providers, options); + } + } + } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotJumpToExceptionHandlerInCallerOp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotJumpToExceptionHandlerInCallerOp.java index ae9c4b11c43..6cc83312a46 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotJumpToExceptionHandlerInCallerOp.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotJumpToExceptionHandlerInCallerOp.java @@ -24,10 +24,10 @@ package org.graalvm.compiler.hotspot.amd64; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; import static jdk.vm.ci.amd64.AMD64.rbp; import static jdk.vm.ci.amd64.AMD64.rsp; import static jdk.vm.ci.code.ValueUtil.asRegister; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; import org.graalvm.compiler.asm.amd64.AMD64Address; import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; @@ -35,7 +35,7 @@ import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.Opcode; import org.graalvm.compiler.lir.asm.CompilationResultBuilder; -import org.graalvm.compiler.serviceprovider.GraalServices; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import jdk.vm.ci.code.Register; import jdk.vm.ci.meta.AllocatableValue; @@ -71,7 +71,7 @@ final class AMD64HotSpotJumpToExceptionHandlerInCallerOp extends AMD64HotSpotEpi // Discard the return address, thus completing restoration of caller frame masm.incrementq(rsp, 8); - if (GraalServices.JAVA_SPECIFICATION_VERSION < 8) { + if (JavaVersionUtil.JAVA_SPECIFICATION_VERSION < 8) { // Restore rsp from rbp if the exception PC is a method handle call site. AMD64Address dst = new AMD64Address(thread, isMethodHandleReturnOffset); masm.cmpl(dst, 0); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotLIRGenerator.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotLIRGenerator.java index b70c04f92a7..1f6b3e9dc7d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotLIRGenerator.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotLIRGenerator.java @@ -118,7 +118,7 @@ public class AMD64HotSpotLIRGenerator extends AMD64LIRGenerator implements HotSp } private AMD64HotSpotLIRGenerator(HotSpotProviders providers, GraalHotSpotVMConfig config, LIRGenerationResult lirGenRes, BackupSlotProvider backupSlotProvider) { - this(new AMD64HotSpotLIRKindTool(), new AMD64ArithmeticLIRGenerator(null, new AMD64HotSpotMaths()), new AMD64HotSpotMoveFactory(backupSlotProvider), providers, config, lirGenRes); + this(new AMD64HotSpotLIRKindTool(), new AMD64ArithmeticLIRGenerator(null), new AMD64HotSpotMoveFactory(backupSlotProvider), providers, config, lirGenRes); } protected AMD64HotSpotLIRGenerator(LIRKindTool lirKindTool, AMD64ArithmeticLIRGenerator arithmeticLIRGen, MoveFactory moveFactory, HotSpotProviders providers, GraalHotSpotVMConfig config, diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotLoweringProvider.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotLoweringProvider.java index d051468a598..2e269ef69ce 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotLoweringProvider.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotLoweringProvider.java @@ -25,15 +25,8 @@ package org.graalvm.compiler.hotspot.amd64; import static org.graalvm.compiler.hotspot.HotSpotBackend.Options.GraalArithmeticStubs; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_COS_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_EXP_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_LOG10_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_LOG_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_POW_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_SIN_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_TAN_STUB; -import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; +import org.graalvm.compiler.api.replacements.Snippet; import org.graalvm.compiler.core.common.spi.ForeignCallsProvider; import org.graalvm.compiler.debug.DebugHandlersFactory; import org.graalvm.compiler.graph.Node; @@ -44,21 +37,26 @@ import org.graalvm.compiler.hotspot.meta.HotSpotProviders; import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider; import org.graalvm.compiler.hotspot.nodes.profiling.ProfileNode; import org.graalvm.compiler.hotspot.replacements.profiling.ProbabilisticProfileSnippets; +import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.calc.FloatConvertNode; +import org.graalvm.compiler.nodes.extended.ForeignCallNode; import org.graalvm.compiler.nodes.spi.LoweringTool; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.replacements.amd64.AMD64ConvertSnippets; -import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode.BinaryOperation; +import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode; import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import jdk.vm.ci.code.TargetDescription; import jdk.vm.ci.hotspot.HotSpotConstantReflectionProvider; import jdk.vm.ci.meta.MetaAccessProvider; +import jdk.vm.ci.meta.ResolvedJavaMethod; public class AMD64HotSpotLoweringProvider extends DefaultHotSpotLoweringProvider { private AMD64ConvertSnippets.Templates convertSnippets; private ProbabilisticProfileSnippets.Templates profileSnippets; + private AMD64X87MathSnippets.Templates mathSnippets; public AMD64HotSpotLoweringProvider(HotSpotGraalRuntimeProvider runtime, MetaAccessProvider metaAccess, ForeignCallsProvider foreignCalls, HotSpotRegistersProvider registers, HotSpotConstantReflectionProvider constantReflection, TargetDescription target) { @@ -68,9 +66,10 @@ public class AMD64HotSpotLoweringProvider extends DefaultHotSpotLoweringProvider @Override public void initialize(OptionValues options, Iterable factories, HotSpotProviders providers, GraalHotSpotVMConfig config) { convertSnippets = new AMD64ConvertSnippets.Templates(options, factories, providers, providers.getSnippetReflection(), providers.getCodeCache().getTarget()); - profileSnippets = ProfileNode.Options.ProbabilisticProfiling.getValue(options) + profileSnippets = ProfileNode.Options.ProbabilisticProfiling.getValue(options) && !JavaVersionUtil.Java8OrEarlier ? new ProbabilisticProfileSnippets.Templates(options, factories, providers, providers.getCodeCache().getTarget()) : null; + mathSnippets = new AMD64X87MathSnippets.Templates(options, factories, providers, providers.getSnippetReflection(), providers.getCodeCache().getTarget()); super.initialize(options, factories, providers, config); } @@ -80,47 +79,49 @@ public class AMD64HotSpotLoweringProvider extends DefaultHotSpotLoweringProvider convertSnippets.lower((FloatConvertNode) n, tool); } else if (profileSnippets != null && n instanceof ProfileNode) { profileSnippets.lower((ProfileNode) n, tool); + } else if (n instanceof UnaryMathIntrinsicNode) { + lowerUnaryMath((UnaryMathIntrinsicNode) n, tool); } else { super.lower(n, tool); } } - @Override - protected ForeignCallDescriptor toForeignCall(UnaryOperation operation) { - if (GraalArithmeticStubs.getValue(runtime.getOptions())) { - switch (operation) { - case LOG: - return ARITHMETIC_LOG_STUB; - case LOG10: - return ARITHMETIC_LOG10_STUB; + private void lowerUnaryMath(UnaryMathIntrinsicNode math, LoweringTool tool) { + if (tool.getLoweringStage() == LoweringTool.StandardLoweringStage.HIGH_TIER) { + return; + } + StructuredGraph graph = math.graph(); + ResolvedJavaMethod method = graph.method(); + if (method != null) { + if (method.getAnnotation(Snippet.class) != null) { + // In the context of SnippetStub, i.e., Graal-generated stubs, use the LIR + // lowering to emit the stub assembly code instead of the Node lowering. + return; + } + } + if (!GraalArithmeticStubs.getValue(graph.getOptions())) { + switch (math.getOperation()) { case SIN: - return ARITHMETIC_SIN_STUB; case COS: - return ARITHMETIC_COS_STUB; case TAN: - return ARITHMETIC_TAN_STUB; - case EXP: - return ARITHMETIC_EXP_STUB; + // Math.sin(), .cos() and .tan() guarantee a value within 1 ULP of the exact + // result, but x87 trigonometric FPU instructions are only that accurate within + // [-pi/4, pi/4]. The snippets fall back to a foreign call to HotSpot stubs + // should the inputs outside of that interval. + mathSnippets.lower(math, tool); + return; + case LOG: + math.replaceAtUsages(graph.addOrUnique(new AMD64X87MathIntrinsicNode(math.getValue(), UnaryOperation.LOG))); + return; + case LOG10: + math.replaceAtUsages(graph.addOrUnique(new AMD64X87MathIntrinsicNode(math.getValue(), UnaryOperation.LOG10))); + return; } - } else if (operation == UnaryOperation.EXP) { - return operation.foreignCallDescriptor; } - // Lower only using LIRGenerator - return null; - } - @Override - protected ForeignCallDescriptor toForeignCall(BinaryOperation operation) { - if (GraalArithmeticStubs.getValue(runtime.getOptions())) { - switch (operation) { - case POW: - return ARITHMETIC_POW_STUB; - } - } else if (operation == BinaryOperation.POW) { - return operation.foreignCallDescriptor; - } - // Lower only using LIRGenerator - return null; + ForeignCallNode call = graph.add(new ForeignCallNode(foreignCalls, math.getOperation().foreignCallDescriptor, math.getValue())); + graph.addAfterFixed(tool.lastFixedNode(), call); + math.replaceAtUsages(call); } @Override diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotMaths.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotMaths.java deleted file mode 100644 index cde8e3b1cbf..00000000000 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64HotSpotMaths.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - - -package org.graalvm.compiler.hotspot.amd64; - -import static org.graalvm.compiler.hotspot.HotSpotBackend.Options.GraalArithmeticStubs; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.COS; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.LOG; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.LOG10; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.SIN; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.TAN; - -import org.graalvm.compiler.core.amd64.AMD64ArithmeticLIRGenerator; -import org.graalvm.compiler.core.common.LIRKind; -import org.graalvm.compiler.hotspot.HotSpotBackend.Options; -import org.graalvm.compiler.lir.Variable; -import org.graalvm.compiler.lir.gen.LIRGenerator; - -import jdk.vm.ci.meta.Value; - -/** - * Lowering of selected {@link Math} routines that depends on the value of - * {@link Options#GraalArithmeticStubs}. - */ -public class AMD64HotSpotMaths implements AMD64ArithmeticLIRGenerator.Maths { - - @Override - public Variable emitLog(LIRGenerator gen, Value input, boolean base10) { - if (GraalArithmeticStubs.getValue(gen.getResult().getLIR().getOptions())) { - return null; - } - Variable result = gen.newVariable(LIRKind.combine(input)); - gen.append(new AMD64HotSpotMathIntrinsicOp(base10 ? LOG10 : LOG, result, gen.asAllocatable(input))); - return result; - } - - @Override - public Variable emitCos(LIRGenerator gen, Value input) { - if (GraalArithmeticStubs.getValue(gen.getResult().getLIR().getOptions())) { - return null; - } - Variable result = gen.newVariable(LIRKind.combine(input)); - gen.append(new AMD64HotSpotMathIntrinsicOp(COS, result, gen.asAllocatable(input))); - return result; - } - - @Override - public Variable emitSin(LIRGenerator gen, Value input) { - if (GraalArithmeticStubs.getValue(gen.getResult().getLIR().getOptions())) { - return null; - } - Variable result = gen.newVariable(LIRKind.combine(input)); - gen.append(new AMD64HotSpotMathIntrinsicOp(SIN, result, gen.asAllocatable(input))); - return result; - } - - @Override - public Variable emitTan(LIRGenerator gen, Value input) { - if (GraalArithmeticStubs.getValue(gen.getResult().getLIR().getOptions())) { - return null; - } - Variable result = gen.newVariable(LIRKind.combine(input)); - gen.append(new AMD64HotSpotMathIntrinsicOp(TAN, result, gen.asAllocatable(input))); - return result; - } - -} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64MathStub.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64MathStub.java index dc99d9e465b..278b9d744da 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64MathStub.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64MathStub.java @@ -24,16 +24,8 @@ package org.graalvm.compiler.hotspot.amd64; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_COS_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_EXP_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_LOG10_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_LOG_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_POW_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_SIN_STUB; -import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotForeignCallsProvider.ARITHMETIC_TAN_STUB; - import org.graalvm.compiler.api.replacements.Snippet; -import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; +import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage; import org.graalvm.compiler.hotspot.meta.HotSpotProviders; import org.graalvm.compiler.hotspot.stubs.SnippetStub; @@ -48,33 +40,38 @@ import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOpera */ public class AMD64MathStub extends SnippetStub { - public AMD64MathStub(ForeignCallDescriptor descriptor, OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage) { - super(snippetName(descriptor), options, providers, linkage); + public AMD64MathStub(UnaryOperation operation, OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage) { + super(snippetName(operation), options, providers, linkage); } - private static String snippetName(ForeignCallDescriptor descriptor) { - if (descriptor == ARITHMETIC_LOG_STUB) { - return "log"; + public AMD64MathStub(BinaryOperation operation, OptionValues options, HotSpotProviders providers, HotSpotForeignCallLinkage linkage) { + super(snippetName(operation), options, providers, linkage); + } + + private static String snippetName(UnaryOperation operation) { + switch (operation) { + case SIN: + return "sin"; + case COS: + return "cos"; + case TAN: + return "tan"; + case EXP: + return "exp"; + case LOG: + return "log"; + case LOG10: + return "log10"; + default: + throw GraalError.shouldNotReachHere("Unknown operation " + operation); } - if (descriptor == ARITHMETIC_LOG10_STUB) { - return "log10"; - } - if (descriptor == ARITHMETIC_SIN_STUB) { - return "sin"; - } - if (descriptor == ARITHMETIC_COS_STUB) { - return "cos"; - } - if (descriptor == ARITHMETIC_TAN_STUB) { - return "tan"; - } - if (descriptor == ARITHMETIC_EXP_STUB) { - return "exp"; - } - if (descriptor == ARITHMETIC_POW_STUB) { + } + + private static String snippetName(BinaryOperation operation) { + if (operation == BinaryOperation.POW) { return "pow"; } - throw new InternalError("Unknown operation " + descriptor); + throw GraalError.shouldNotReachHere("Unknown operation " + operation); } @Snippet diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64X87MathIntrinsicNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64X87MathIntrinsicNode.java new file mode 100644 index 00000000000..114356d8998 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64X87MathIntrinsicNode.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.amd64; + +import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.COS; +import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.LOG; +import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.LOG10; +import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.SIN; +import static org.graalvm.compiler.hotspot.amd64.AMD64HotSpotMathIntrinsicOp.IntrinsicOpcode.TAN; +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_1; + +import org.graalvm.compiler.core.common.LIRKind; +import org.graalvm.compiler.core.common.type.FloatStamp; +import org.graalvm.compiler.core.common.type.PrimitiveStamp; +import org.graalvm.compiler.core.common.type.Stamp; +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.graph.spi.CanonicalizerTool; +import org.graalvm.compiler.lir.Variable; +import org.graalvm.compiler.lir.gen.LIRGeneratorTool; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.ConstantNode; +import org.graalvm.compiler.nodes.NodeView; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.calc.UnaryNode; +import org.graalvm.compiler.nodes.spi.LIRLowerable; +import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; +import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation; + +import jdk.vm.ci.meta.Value; + +@NodeInfo(nameTemplate = "X87MathIntrinsic#{p#operation/s}", cycles = CYCLES_64, size = SIZE_1) +public final class AMD64X87MathIntrinsicNode extends UnaryNode implements LIRLowerable { + + public static final NodeClass TYPE = NodeClass.create(AMD64X87MathIntrinsicNode.class); + protected final UnaryOperation operation; + + protected AMD64X87MathIntrinsicNode(ValueNode value, UnaryOperation op) { + super(TYPE, op.computeStamp(value.stamp(NodeView.DEFAULT)), value); + assert value.stamp(NodeView.DEFAULT) instanceof FloatStamp && PrimitiveStamp.getBits(value.stamp(NodeView.DEFAULT)) == 64; + this.operation = op; + } + + @Override + public Stamp foldStamp(Stamp valueStamp) { + return operation.computeStamp(valueStamp); + } + + @Override + public void generate(NodeLIRBuilderTool generator) { + LIRGeneratorTool gen = generator.getLIRGeneratorTool(); + Value input = generator.operand(getValue()); + Variable result = gen.newVariable(LIRKind.combine(input)); + + switch (operation) { + case SIN: + gen.append(new AMD64HotSpotMathIntrinsicOp(SIN, result, gen.asAllocatable(input))); + break; + case COS: + gen.append(new AMD64HotSpotMathIntrinsicOp(COS, result, gen.asAllocatable(input))); + break; + case TAN: + gen.append(new AMD64HotSpotMathIntrinsicOp(TAN, result, gen.asAllocatable(input))); + break; + case LOG: + gen.append(new AMD64HotSpotMathIntrinsicOp(LOG, result, gen.asAllocatable(input))); + break; + case LOG10: + gen.append(new AMD64HotSpotMathIntrinsicOp(LOG10, result, gen.asAllocatable(input))); + break; + default: + throw GraalError.shouldNotReachHere(); + } + generator.setResult(this, result); + } + + @Override + public ValueNode canonical(CanonicalizerTool tool, ValueNode forValue) { + if (forValue.isConstant()) { + return ConstantNode.forDouble(operation.compute(forValue.asJavaConstant().asDouble())); + } + return this; + } + + @NodeIntrinsic + public static native double compute(double value, @ConstantNodeParameter UnaryOperation op); + +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64X87MathSnippets.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64X87MathSnippets.java new file mode 100644 index 00000000000..1de9a703413 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.amd64/src/org/graalvm/compiler/hotspot/amd64/AMD64X87MathSnippets.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.amd64; + +import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER; + +import org.graalvm.compiler.api.replacements.Snippet; +import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; +import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; +import org.graalvm.compiler.debug.DebugHandlersFactory; +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.graph.Node.ConstantNodeParameter; +import org.graalvm.compiler.graph.Node.NodeIntrinsic; +import org.graalvm.compiler.nodes.extended.ForeignCallNode; +import org.graalvm.compiler.nodes.spi.LoweringTool; +import org.graalvm.compiler.options.OptionValues; +import org.graalvm.compiler.phases.util.Providers; +import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates; +import org.graalvm.compiler.replacements.SnippetTemplate.Arguments; +import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo; +import org.graalvm.compiler.replacements.Snippets; +import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode; +import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation; + +import jdk.vm.ci.code.TargetDescription; + +public class AMD64X87MathSnippets implements Snippets { + + private static final double PI_4 = Math.PI / 4; + + @Snippet + public static double sin(double input) { + if (Math.abs(input) < PI_4) { + return AMD64X87MathIntrinsicNode.compute(input, UnaryOperation.SIN); + } + return callDouble1(UnaryOperation.SIN.foreignCallDescriptor, input); + } + + @Snippet + public static double cos(double input) { + if (Math.abs(input) < PI_4) { + return AMD64X87MathIntrinsicNode.compute(input, UnaryOperation.COS); + } + return callDouble1(UnaryOperation.COS.foreignCallDescriptor, input); + } + + @Snippet + public static double tan(double input) { + if (Math.abs(input) < PI_4) { + return AMD64X87MathIntrinsicNode.compute(input, UnaryOperation.TAN); + } + return callDouble1(UnaryOperation.TAN.foreignCallDescriptor, input); + } + + @NodeIntrinsic(value = ForeignCallNode.class) + private static native double callDouble1(@ConstantNodeParameter ForeignCallDescriptor descriptor, double value); + + public static class Templates extends AbstractTemplates { + + private final SnippetInfo sin; + private final SnippetInfo cos; + private final SnippetInfo tan; + + public Templates(OptionValues options, Iterable factories, Providers providers, SnippetReflectionProvider snippetReflection, TargetDescription target) { + super(options, factories, providers, snippetReflection, target); + + sin = snippet(AMD64X87MathSnippets.class, "sin"); + cos = snippet(AMD64X87MathSnippets.class, "cos"); + tan = snippet(AMD64X87MathSnippets.class, "tan"); + } + + public void lower(UnaryMathIntrinsicNode mathIntrinsicNode, LoweringTool tool) { + SnippetInfo info; + + switch (mathIntrinsicNode.getOperation()) { + case SIN: + info = sin; + break; + case COS: + info = cos; + break; + case TAN: + info = tan; + break; + default: + throw GraalError.shouldNotReachHere("Snippet not found for math intrinsic " + mathIntrinsicNode.getOperation().name()); + } + + Arguments args = new Arguments(info, mathIntrinsicNode.graph().getGuardsStage(), tool.getLoweringStage()); + args.add("input", mathIntrinsicNode.getValue()); + template(mathIntrinsicNode, args).instantiate(providers.getMetaAccess(), mathIntrinsicNode, DEFAULT_REPLACER, tool, args); + mathIntrinsicNode.safeDelete(); + } + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.jdk9.test/src/org/graalvm/compiler/hotspot/jdk9/test/StringUTF16ToBytesGetCharsTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.jdk9.test/src/org/graalvm/compiler/hotspot/jdk9/test/StringUTF16ToBytesGetCharsTest.java new file mode 100644 index 00000000000..a70a0d2c54c --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.jdk9.test/src/org/graalvm/compiler/hotspot/jdk9/test/StringUTF16ToBytesGetCharsTest.java @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.jdk9.test; + +import static org.junit.Assume.assumeFalse; + +import jdk.vm.ci.code.InstalledCode; +import jdk.vm.ci.meta.ResolvedJavaMethod; +import org.graalvm.compiler.core.common.CompilationIdentifier; +import org.graalvm.compiler.hotspot.replacements.StringUTF16Substitutions; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.java.NewArrayNode; +import org.graalvm.compiler.replacements.arraycopy.ArrayCopyCallNode; +import org.graalvm.compiler.replacements.test.MethodSubstitutionTest; +import org.graalvm.compiler.test.AddExports; +import org.junit.Before; +import org.junit.Test; + +/** + * Test substitutions for (innate) methods StringUTF16.toBytes and StringUTF16.getChars provided by + * {@link StringUTF16Substitutions}. + */ +@AddExports({"java.base/java.lang"}) +public final class StringUTF16ToBytesGetCharsTest extends MethodSubstitutionTest { + + private static final int N = 1000; + private static final int N_OVERFLOW = 10; + + @Before + public void checkAMD64() { + assumeFalse(Java8OrEarlier); + } + + @Test + public void testStringUTF16ToBytes() throws ClassNotFoundException { + Class javaclass = Class.forName("java.lang.StringUTF16"); + + ResolvedJavaMethod caller = getResolvedJavaMethod(javaclass, "toBytes", char[].class, int.class, int.class); + StructuredGraph graph = getReplacements().getIntrinsicGraph(caller, CompilationIdentifier.INVALID_COMPILATION_ID, getDebugContext()); + assertInGraph(graph, NewArrayNode.class); + assertInGraph(graph, ArrayCopyCallNode.class); + + InstalledCode code = getCode(caller, graph); + + for (int srcOffset = 0; srcOffset < 2; srcOffset++) { + for (int i = 0; i < N; i++) { + int length = i2sz(i); + char[] src = fillUTF16Chars(new char[length]); + int copiedLength = Math.max(0, length - srcOffset); + int srcDelta = Math.min(srcOffset, copiedLength); + byte[] dst = (byte[]) invokeSafe(caller, null, src, srcDelta, copiedLength); + assert dst.length == copiedLength * 2; + byte[] dst2 = (byte[]) executeVarargsSafe(code, src, srcDelta, copiedLength); + assertDeepEquals(dst, dst2); + } + } + for (int srcOff = 0; srcOff < N_OVERFLOW; ++srcOff) { + for (int len = 0; len < N_OVERFLOW; ++len) { + char[] src = fillUTF16Chars(new char[N_OVERFLOW]); + test(caller, null, src, srcOff, len); + } + } + } + + @Test + public void testStringUTF16getChars() throws ClassNotFoundException { + Class javaclass = Class.forName("java.lang.StringUTF16"); + + ResolvedJavaMethod caller = getResolvedJavaMethod(javaclass, "getChars", byte[].class, int.class, int.class, char[].class, int.class); + StructuredGraph graph = getReplacements().getIntrinsicGraph(caller, CompilationIdentifier.INVALID_COMPILATION_ID, getDebugContext()); + assertInGraph(graph, ArrayCopyCallNode.class); + + InstalledCode code = getCode(caller, graph); + + for (int dstOffset = 0; dstOffset < 2; dstOffset++) { + for (int srcOffset = 0; srcOffset < 2; srcOffset++) { + for (int i = 0; i < N; i++) { + int length = i2sz(i); + byte[] src = fillUTF16Bytes(new byte[length * 2]); + char[] dst = new char[length]; + int copiedLength = Math.max(0, length - Math.max(dstOffset, srcOffset)); + int dstDelta = Math.min(dstOffset, copiedLength); + int srcDelta = Math.min(srcOffset, copiedLength); + invokeSafe(caller, null, src, srcDelta, srcDelta + copiedLength, dst, dstDelta); + char[] dst2 = new char[length]; + executeVarargsSafe(code, src, srcDelta, srcDelta + copiedLength, dst2, dstDelta); + assertDeepEquals(dst, dst2); + } + } + } + for (int srcOff = 0; srcOff < N_OVERFLOW; ++srcOff) { + for (int dstOff = 0; dstOff < N_OVERFLOW; ++dstOff) { + for (int len = 0; len < N_OVERFLOW; ++len) { + byte[] src = fillUTF16Bytes(new byte[N_OVERFLOW]); + char[] dst = new char[N_OVERFLOW]; + test(caller, null, src, srcOff, len, dst, dstOff); + } + } + } + } + + private static char[] fillUTF16Chars(char[] v) { + for (int ch = 0, i = 0; i < v.length; i++, ch += 0x101) { + v[i] = (char) ch; + } + return v; + } + + private static byte[] fillUTF16Bytes(byte[] v) { + for (int ch = 1, i = 0; i < v.length; i += 2, ch++) { + v[i] = (byte) (ch - 1); + v[i + 1] = (byte) ch; + } + return v; + } + + private static int i2sz(int i) { + return i * 3; + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.lir.test/src/org/graalvm/compiler/hotspot/lir/test/BenchmarkCounterOverflowTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.lir.test/src/org/graalvm/compiler/hotspot/lir/test/BenchmarkCounterOverflowTest.java index 8cfc7ed4ffd..cbac73211ab 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.lir.test/src/org/graalvm/compiler/hotspot/lir/test/BenchmarkCounterOverflowTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.lir.test/src/org/graalvm/compiler/hotspot/lir/test/BenchmarkCounterOverflowTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,9 +27,12 @@ package org.graalvm.compiler.hotspot.lir.test; import static org.graalvm.compiler.test.SubprocessUtil.getVMCommandLine; import static org.graalvm.compiler.test.SubprocessUtil.withoutDebuggerArguments; +import java.io.File; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.graalvm.compiler.api.directives.GraalDirectives; import org.graalvm.compiler.core.common.LIRKind; @@ -121,6 +124,7 @@ public class BenchmarkCounterOverflowTest extends LIRTest { Assert.assertNotEquals("Expected non-zero exit status", 0, proc.exitCode); Iterator it = proc.output.iterator(); + boolean foundProblematicFrame = false; while (it.hasNext()) { String line = it.next(); if (line.contains("Problematic frame:")) { @@ -130,11 +134,34 @@ public class BenchmarkCounterOverflowTest extends LIRTest { } line = it.next(); if (line.contains(BenchmarkCounterOverflowTest.class.getName() + ".test")) { - return; + foundProblematicFrame = true; + break; } Assert.fail("Unexpected stack trace: " + line); } } - Assert.fail(String.format("Could not find method in output:%n%s", proc)); + // find and delete hserr file + while (it.hasNext()) { + String line = it.next(); + if (line.contains("An error report file with more information is saved as:")) { + if (!it.hasNext()) { + // no more line + break; + } + line = it.next(); + Pattern pattern = Pattern.compile("^# (.*hs_err_pid.*log)$"); + Matcher matcher = pattern.matcher(line); + if (matcher.matches()) { + File hserrFile = new File(matcher.group(1)); + if (hserrFile.exists()) { + if (VERBOSE) { + System.out.println("Deleting error report file:" + hserrFile.getAbsolutePath()); + } + hserrFile.delete(); + } + } + } + } + Assert.assertTrue(String.format("Could not find method in output:%n%s", proc), foundProblematicFrame); } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotBackend.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotBackend.java index 7ba9e4328de..cb9740b5b8f 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotBackend.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotBackend.java @@ -63,6 +63,7 @@ import org.graalvm.compiler.code.DataSection.Data; import org.graalvm.compiler.core.common.CompilationIdentifier; import org.graalvm.compiler.core.common.alloc.RegisterAllocationConfig; import org.graalvm.compiler.core.common.cfg.AbstractBlockBase; +import org.graalvm.compiler.core.gen.LIRGenerationProvider; import org.graalvm.compiler.core.sparc.SPARCNodeMatchRules; import org.graalvm.compiler.debug.CounterKey; import org.graalvm.compiler.debug.DebugContext; @@ -109,7 +110,7 @@ import jdk.vm.ci.meta.ResolvedJavaMethod; /** * HotSpot SPARC specific backend. */ -public class SPARCHotSpotBackend extends HotSpotHostBackend { +public class SPARCHotSpotBackend extends HotSpotHostBackend implements LIRGenerationProvider { private static final SizeEstimateStatistics CONSTANT_ESTIMATED_STATS = new SizeEstimateStatistics("ESTIMATE"); private static final SizeEstimateStatistics CONSTANT_ACTUAL_STATS = new SizeEstimateStatistics("ACTUAL"); @@ -134,15 +135,10 @@ public class SPARCHotSpotBackend extends HotSpotHostBackend { } } - @Override - public FrameMapBuilder newFrameMapBuilder(RegisterConfig registerConfig) { + private FrameMapBuilder newFrameMapBuilder(RegisterConfig registerConfig) { RegisterConfig registerConfigNonNull = registerConfig == null ? getCodeCache().getRegisterConfig() : registerConfig; - return new SPARCFrameMapBuilder(newFrameMap(registerConfigNonNull), getCodeCache(), registerConfigNonNull); - } - - @Override - public FrameMap newFrameMap(RegisterConfig registerConfig) { - return new SPARCFrameMap(getCodeCache(), registerConfig, this); + FrameMap frameMap = new SPARCFrameMap(getCodeCache(), registerConfigNonNull, this); + return new SPARCFrameMapBuilder(frameMap, getCodeCache(), registerConfigNonNull); } @Override @@ -151,8 +147,9 @@ public class SPARCHotSpotBackend extends HotSpotHostBackend { } @Override - public LIRGenerationResult newLIRGenerationResult(CompilationIdentifier compilationId, LIR lir, FrameMapBuilder frameMapBuilder, StructuredGraph graph, Object stub) { - return new HotSpotLIRGenerationResult(compilationId, lir, frameMapBuilder, makeCallingConvention(graph, (Stub) stub), stub, config.requiresReservedStackCheck(graph.getMethods())); + public LIRGenerationResult newLIRGenerationResult(CompilationIdentifier compilationId, LIR lir, RegisterConfig registerConfig, StructuredGraph graph, Object stub) { + return new HotSpotLIRGenerationResult(compilationId, lir, newFrameMapBuilder(registerConfig), makeCallingConvention(graph, (Stub) stub), stub, + config.requiresReservedStackCheck(graph.getMethods())); } @Override @@ -226,11 +223,6 @@ public class SPARCHotSpotBackend extends HotSpotHostBackend { } } - @Override - protected Assembler createAssembler(FrameMap frameMap) { - return new SPARCMacroAssembler(getTarget()); - } - @Override public CompilationResultBuilder newCompilationResultBuilder(LIRGenerationResult lirGenRes, FrameMap frameMap, CompilationResult compilationResult, CompilationResultBuilderFactory factory) { HotSpotLIRGenerationResult gen = (HotSpotLIRGenerationResult) lirGenRes; @@ -238,7 +230,7 @@ public class SPARCHotSpotBackend extends HotSpotHostBackend { assert gen.getDeoptimizationRescueSlot() == null || frameMap.frameNeedsAllocating() : "method that can deoptimize must have a frame"; Stub stub = gen.getStub(); - Assembler masm = createAssembler(frameMap); + Assembler masm = new SPARCMacroAssembler(getTarget()); // On SPARC we always use stack frames. HotSpotFrameContext frameContext = new HotSpotFrameContext(stub != null); DataBuilder dataBuilder = new HotSpotDataBuilder(getCodeCache().getTarget()); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotMove.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotMove.java index 8a712d92d36..11085a5d0b2 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotMove.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.sparc/src/org/graalvm/compiler/hotspot/sparc/SPARCHotSpotMove.java @@ -24,20 +24,21 @@ package org.graalvm.compiler.hotspot.sparc; +import static jdk.vm.ci.code.ValueUtil.asRegister; import static org.graalvm.compiler.asm.sparc.SPARCAssembler.BPR; import static org.graalvm.compiler.asm.sparc.SPARCAssembler.Annul.ANNUL; +import static org.graalvm.compiler.asm.sparc.SPARCAssembler.Annul.NOT_ANNUL; +import static org.graalvm.compiler.asm.sparc.SPARCAssembler.BranchPredict.PREDICT_NOT_TAKEN; import static org.graalvm.compiler.asm.sparc.SPARCAssembler.BranchPredict.PREDICT_TAKEN; +import static org.graalvm.compiler.asm.sparc.SPARCAssembler.RCondition.Rc_nz; import static org.graalvm.compiler.asm.sparc.SPARCAssembler.RCondition.Rc_z; import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.ILLEGAL; import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK; import static org.graalvm.compiler.lir.sparc.SPARCMove.loadFromConstantTable; -import static jdk.vm.ci.code.ValueUtil.asRegister; import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.asm.sparc.SPARCAddress; -import org.graalvm.compiler.asm.sparc.SPARCAssembler.CC; -import org.graalvm.compiler.asm.sparc.SPARCAssembler.ConditionFlag; import org.graalvm.compiler.asm.sparc.SPARCMacroAssembler; import org.graalvm.compiler.asm.sparc.SPARCMacroAssembler.ScratchRegister; import org.graalvm.compiler.core.common.CompressEncoding; @@ -53,6 +54,7 @@ import jdk.vm.ci.code.ValueUtil; import jdk.vm.ci.hotspot.HotSpotConstant; import jdk.vm.ci.meta.AllocatableValue; import jdk.vm.ci.meta.Constant; +import jdk.vm.ci.sparc.SPARC; public class SPARCHotSpotMove { @@ -150,17 +152,33 @@ public class SPARCHotSpotMove { if (encoding.hasBase()) { Register baseReg = asRegister(baseRegister); if (!nonNull) { - masm.cmp(inputRegister, baseReg); - masm.movcc(ConditionFlag.Equal, CC.Xcc, baseReg, resReg); - masm.sub(resReg, baseReg, resReg); + Label done = new Label(); + if (inputRegister.equals(resReg)) { + BPR.emit(masm, Rc_nz, ANNUL, PREDICT_TAKEN, inputRegister, done); + masm.sub(inputRegister, baseReg, resReg); + masm.bind(done); + if (encoding.getShift() != 0) { + masm.srlx(resReg, encoding.getShift(), resReg); + } + } else { + BPR.emit(masm, Rc_z, NOT_ANNUL, PREDICT_NOT_TAKEN, inputRegister, done); + masm.mov(SPARC.g0, resReg); + masm.sub(inputRegister, baseReg, resReg); + if (encoding.getShift() != 0) { + masm.srlx(resReg, encoding.getShift(), resReg); + } + masm.bind(done); + } } else { masm.sub(inputRegister, baseReg, resReg); - } - if (encoding.getShift() != 0) { - masm.srlx(resReg, encoding.getShift(), resReg); + if (encoding.getShift() != 0) { + masm.srlx(resReg, encoding.getShift(), resReg); + } } } else { - masm.srlx(inputRegister, encoding.getShift(), resReg); + if (encoding.getShift() != 0) { + masm.srlx(inputRegister, encoding.getShift(), resReg); + } } } } @@ -196,7 +214,7 @@ public class SPARCHotSpotMove { public static void emitUncompressCode(SPARCMacroAssembler masm, Register inputRegister, Register resReg, Register baseReg, int shift, boolean nonNull) { Register secondaryInput; if (shift != 0) { - masm.sll(inputRegister, shift, resReg); + masm.sllx(inputRegister, shift, resReg); secondaryInput = resReg; } else { secondaryInput = inputRegister; @@ -207,7 +225,7 @@ public class SPARCHotSpotMove { masm.add(secondaryInput, baseReg, resReg); } else { Label done = new Label(); - BPR.emit(masm, Rc_z, ANNUL, PREDICT_TAKEN, secondaryInput, done); + BPR.emit(masm, Rc_nz, ANNUL, PREDICT_TAKEN, secondaryInput, done); masm.add(baseReg, secondaryInput, resReg); masm.bind(done); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CheckGraalIntrinsics.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CheckGraalIntrinsics.java index fc67b021c03..4433a1b380b 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CheckGraalIntrinsics.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CheckGraalIntrinsics.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -47,7 +47,7 @@ import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin; import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins; import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins.Binding; import org.graalvm.compiler.runtime.RuntimeProvider; -import org.graalvm.compiler.serviceprovider.GraalServices; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import org.graalvm.compiler.test.GraalTest; import org.junit.Test; @@ -257,7 +257,8 @@ public class CheckGraalIntrinsics extends GraalTest { "java/lang/reflect/Array.newArray(Ljava/lang/Class;I)Ljava/lang/Object;", // HotSpot MacroAssembler-based intrinsic "sun/nio/cs/ISO_8859_1$Encoder.encodeISOArray([CI[BII)I", - // Stub based intrinsics but implementation seems complex in C2 + // We have implemented implCompressMultiBlock0 on JDK9+. Does it worth + // backporting as corresponding HotSpot stubs are only generated on SPARC? "sun/security/provider/DigestBase.implCompressMultiBlock([BII)I"); // See JDK-8207146. @@ -271,18 +272,12 @@ public class CheckGraalIntrinsics extends GraalTest { "jdk/jfr/internal/JVM.getClassId(Ljava/lang/Class;)J"); add(toBeInvestigated, - // Some logic and a stub call - "com/sun/crypto/provider/CounterMode.implCrypt([BII[BI)I", - // Stub and very little logic - "com/sun/crypto/provider/GHASH.processBlocks([BII[J[J)V", // HotSpot MacroAssembler-based intrinsic "java/lang/Math.fma(DDD)D", // HotSpot MacroAssembler-based intrinsic "java/lang/Math.fma(FFF)F", // Just check if the argument is a compile time constant "java/lang/invoke/MethodHandleImpl.isCompileConstant(Ljava/lang/Object;)Z", - // Some logic and a runtime call - "java/util/ArraysSupport.vectorizedMismatch(Ljava/lang/Object;JLjava/lang/Object;JII)I", // Only used as a marker for vectorization? "java/util/stream/Streams$RangeIntSpliterator.forEachRemaining(Ljava/util/function/IntConsumer;)V", // Only implemented on non-AMD64 platforms (some logic and runtime call) @@ -295,9 +290,7 @@ public class CheckGraalIntrinsics extends GraalTest { // Control flow, deopts, and a cast "jdk/internal/util/Preconditions.checkIndex(IILjava/util/function/BiFunction;)I", // HotSpot MacroAssembler-based intrinsic - "sun/nio/cs/ISO_8859_1$Encoder.implEncodeISOArray([CI[BII)I", - // Runtime call and some complex compiler logic - "sun/security/provider/DigestBase.implCompressMultiBlock0([BII)I"); + "sun/nio/cs/ISO_8859_1$Encoder.implEncodeISOArray([CI[BII)I"); /* * Per default, all these operations are mapped to some generic method for which we @@ -344,27 +337,45 @@ public class CheckGraalIntrinsics extends GraalTest { // Compact string support - HotSpot MacroAssembler-based intrinsic or complex C2 logic. add(toBeInvestigated, "java/lang/StringCoding.hasNegatives([BII)Z", - "java/lang/StringCoding.implEncodeISOArray([BI[BII)I", + "java/lang/StringCoding.implEncodeISOArray([BI[BII)I"); + add(ignore, + // handled through an intrinsic for String.equals itself + "java/lang/StringLatin1.equals([B[B)Z", + + // handled by an intrinsic for StringLatin1.indexOf([BI[BII)I "java/lang/StringLatin1.indexOf([B[B)I", - "java/lang/StringUTF16.getChar([BI)C", - "java/lang/StringUTF16.getChars([BII[CI)V", + + // handled through an intrinsic for String.equals itself + "java/lang/StringUTF16.equals([B[B)Z", + + // handled by an intrinsic for StringUTF16.indexOfUnsafe "java/lang/StringUTF16.indexOf([BI[BII)I", "java/lang/StringUTF16.indexOf([B[B)I", + + // handled by an intrinsic for StringUTF16.indexOfCharUnsafe "java/lang/StringUTF16.indexOfChar([BIII)I", + + // handled by an intrinsic for StringUTF16.indexOfLatin1Unsafe "java/lang/StringUTF16.indexOfLatin1([BI[BII)I", - "java/lang/StringUTF16.indexOfLatin1([B[B)I", - "java/lang/StringUTF16.putChar([BII)V", - "java/lang/StringUTF16.toBytes([CII)[B"); - // These are handled through an intrinsic for String.equals itself - add(ignore, - "java/lang/StringLatin1.equals([B[B)Z", - "java/lang/StringUTF16.equals([B[B)Z"); + "java/lang/StringUTF16.indexOfLatin1([B[B)I"); + + if (!config.useAESCTRIntrinsics) { + add(ignore, + "com/sun/crypto/provider/CounterMode.implCrypt([BII[BI)I"); + } + if (!config.useGHASHIntrinsics()) { + add(ignore, + "com/sun/crypto/provider/GHASH.processBlocks([BII[J[J)V"); + } + if (!(config.useSHA1Intrinsics() || config.useSHA256Intrinsics() || config.useSHA512Intrinsics())) { + add(ignore, + "sun/security/provider/DigestBase.implCompressMultiBlock0([BII)I"); + } } if (isJDK10OrHigher()) { add(toBeInvestigated, - "java/lang/Math.multiplyHigh(JJ)J", - "jdk/internal/util/ArraysSupport.vectorizedMismatch(Ljava/lang/Object;JLjava/lang/Object;JII)I"); + "java/lang/Math.multiplyHigh(JJ)J"); } if (isJDK11OrHigher()) { @@ -401,10 +412,16 @@ public class CheckGraalIntrinsics extends GraalTest { // Can we implement these on non-AMD64 platforms? C2 seems to. add(toBeInvestigated, "java/lang/String.compareTo(Ljava/lang/String;)I", + "java/lang/StringLatin1.indexOf([B[B)I", "java/lang/StringLatin1.inflate([BI[BII)V", "java/lang/StringLatin1.inflate([BI[CII)V", "java/lang/StringUTF16.compress([BI[BII)I", "java/lang/StringUTF16.compress([CI[BII)I", + "java/lang/StringUTF16.indexOf([BI[BII)I", + "java/lang/StringUTF16.indexOf([B[B)I", + "java/lang/StringUTF16.indexOfChar([BIII)I", + "java/lang/StringUTF16.indexOfLatin1([BI[BII)I", + "java/lang/StringUTF16.indexOfLatin1([B[B)I", "jdk/internal/misc/Unsafe.compareAndExchangeByte(Ljava/lang/Object;JBB)B", "jdk/internal/misc/Unsafe.compareAndExchangeShort(Ljava/lang/Object;JSS)S", "jdk/internal/misc/Unsafe.compareAndSetByte(Ljava/lang/Object;JBB)Z", @@ -433,7 +450,9 @@ public class CheckGraalIntrinsics extends GraalTest { "jdk/internal/misc/Unsafe.getAndSet" + oopName + "(Ljava/lang/Object;JLjava/lang/Object;)Ljava/lang/Object;"); } add(toBeInvestigated, + "com/sun/crypto/provider/CounterMode.implCrypt([BII[BI)I", "java/lang/Thread.onSpinWait()V", + "java/util/ArraysSupport.vectorizedMismatch(Ljava/lang/Object;JLjava/lang/Object;JII)I", "jdk/internal/misc/Unsafe.getCharUnaligned(Ljava/lang/Object;J)C", "jdk/internal/misc/Unsafe.getIntUnaligned(Ljava/lang/Object;J)I", "jdk/internal/misc/Unsafe.getLongUnaligned(Ljava/lang/Object;J)J", @@ -443,6 +462,10 @@ public class CheckGraalIntrinsics extends GraalTest { "jdk/internal/misc/Unsafe.putLongUnaligned(Ljava/lang/Object;JJ)V", "jdk/internal/misc/Unsafe.putShortUnaligned(Ljava/lang/Object;JS)V"); } + if (isJDK10OrHigher()) { + add(toBeInvestigated, + "jdk/internal/util/ArraysSupport.vectorizedMismatch(Ljava/lang/Object;JLjava/lang/Object;JII)I"); + } } /* @@ -535,23 +558,23 @@ public class CheckGraalIntrinsics extends GraalTest { } private static boolean isJDK9OrHigher() { - return GraalServices.JAVA_SPECIFICATION_VERSION >= 9; + return JavaVersionUtil.JAVA_SPECIFICATION_VERSION >= 9; } private static boolean isJDK10OrHigher() { - return GraalServices.JAVA_SPECIFICATION_VERSION >= 10; + return JavaVersionUtil.JAVA_SPECIFICATION_VERSION >= 10; } private static boolean isJDK11OrHigher() { - return GraalServices.JAVA_SPECIFICATION_VERSION >= 11; + return JavaVersionUtil.JAVA_SPECIFICATION_VERSION >= 11; } private static boolean isJDK12OrHigher() { - return GraalServices.JAVA_SPECIFICATION_VERSION >= 12; + return JavaVersionUtil.JAVA_SPECIFICATION_VERSION >= 12; } private static boolean isJDK13OrHigher() { - return GraalServices.JAVA_SPECIFICATION_VERSION >= 13; + return JavaVersionUtil.JAVA_SPECIFICATION_VERSION >= 13; } public interface Refiner { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CompileTheWorld.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CompileTheWorld.java index 9f42f9ed6d8..516811509a1 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CompileTheWorld.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/CompileTheWorld.java @@ -31,7 +31,7 @@ import static org.graalvm.compiler.core.GraalCompilerOptions.CompilationFailureA import static org.graalvm.compiler.core.test.ReflectionOptionDescriptors.extractEntries; import static org.graalvm.compiler.debug.MemUseTrackerKey.getCurrentThreadAllocatedBytes; import static org.graalvm.compiler.hotspot.test.CompileTheWorld.Options.DESCRIPTORS; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import java.io.Closeable; import java.io.File; @@ -90,7 +90,7 @@ import org.graalvm.compiler.options.OptionDescriptors; import org.graalvm.compiler.options.OptionKey; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.options.OptionsParser; -import org.graalvm.compiler.serviceprovider.GraalServices; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import jdk.vm.ci.hotspot.HotSpotCodeCacheProvider; import jdk.vm.ci.hotspot.HotSpotCompilationRequest; @@ -110,7 +110,7 @@ public final class CompileTheWorld { /** * Magic token to denote that JDK classes are to be compiled. If - * {@link GraalServices#Java8OrEarlier}, then the classes in {@code rt.jar} are compiled. + * {@link JavaVersionUtil#Java8OrEarlier}, then the classes in {@code rt.jar} are compiled. * Otherwise the classes in the Java runtime image are compiled. */ public static final String SUN_BOOT_CLASS_PATH = "sun.boot.class.path"; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/EliminateRedundantInitializationPhaseTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/EliminateRedundantInitializationPhaseTest.java new file mode 100644 index 00000000000..9884669b43c --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/EliminateRedundantInitializationPhaseTest.java @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.test; + +import org.graalvm.compiler.core.test.GraalCompilerTest; +import org.graalvm.compiler.hotspot.meta.HotSpotClassInitializationPlugin; +import org.graalvm.compiler.hotspot.nodes.aot.InitializeKlassNode; +import org.graalvm.compiler.hotspot.phases.aot.EliminateRedundantInitializationPhase; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions; +import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.Plugins; +import org.graalvm.compiler.phases.tiers.HighTierContext; +import org.junit.Assert; +import org.junit.Test; + +public class EliminateRedundantInitializationPhaseTest extends GraalCompilerTest { + @Override + protected Plugins getDefaultGraphBuilderPlugins() { + Plugins plugins = super.getDefaultGraphBuilderPlugins(); + plugins.setClassInitializationPlugin(new HotSpotClassInitializationPlugin()); + return plugins; + } + + public static class X { + public static int x; + public static int y; + public static int z; + } + + public static class Y extends X { + public static int a; + public static int b; + } + + public static void assignFields() { + X.x = 1; + X.y = 2; + X.z = 3; + } + + public static void assignFieldsConditionally(boolean choice) { + X.x = 1; + if (choice) { + X.y = 2; + } else { + X.z = 3; + } + } + + public static void assignFieldsSubclassDominates() { + Y.a = 1; + X.x = 2; + X.y = 3; + X.z = 4; + } + + public static void assignFieldsConditionallySubclassDominates(boolean choice) { + Y.a = 1; + if (choice) { + X.x = 2; + } else { + X.y = 3; + } + Y.z = 4; + } + + public static void assignFieldsSubclassPostdominates() { + X.x = 1; + Y.a = 2; + } + + public static void assignFieldsConditionallySubclassPostdominates(boolean choice) { + X.x = 1; + if (choice) { + X.y = 2; + } else { + X.z = 3; + } + Y.a = 4; + } + + public static void assignFieldsConditionallyMixed(boolean choice) { + X.x = 1; + if (choice) { + Y.a = 2; + } else { + X.z = 3; + } + Y.b = 4; + } + + public static void assignFieldsInLoop() { + X.x = 1; + for (int i = 0; i < 10; i++) { + X.y += X.z; + } + } + + public static void assignFieldsInBranches(boolean choice) { + if (choice) { + X.x = 1; + } else { + X.y = 2; + } + X.z = 3; + } + + public static void assignFieldsInBranchesMixed(boolean choice) { + if (choice) { + X.x = 1; + } else { + Y.a = 2; + } + X.z = 3; + } + + static class SomeClass { + @BytecodeParserNeverInline + static void method() { + } + + @BytecodeParserForceInline + static void inlinedMethod() { + } + } + + public static void invokestatic() { + SomeClass.method(); + } + + public static void invokestaticInlined() { + SomeClass.inlinedMethod(); + } + + private void test(String name, int initNodesAfterParse, int initNodesAfterOpt) { + StructuredGraph graph = parseEager(name, AllowAssumptions.NO); + Assert.assertEquals(initNodesAfterParse, graph.getNodes().filter(InitializeKlassNode.class).count()); + HighTierContext highTierContext = getDefaultHighTierContext(); + new EliminateRedundantInitializationPhase().apply(graph, highTierContext); + Assert.assertEquals(initNodesAfterOpt, graph.getNodes().filter(InitializeKlassNode.class).count()); + } + + @Test + public void test1() { + test("assignFields", 3, 1); + } + + @Test + public void test2() { + test("assignFieldsConditionally", 3, 1); + } + + @Test + public void test3() { + test("assignFieldsSubclassDominates", 4, 1); + } + + @Test + public void test4() { + test("assignFieldsConditionallySubclassDominates", 4, 1); + } + + @Test + public void test5() { + test("assignFieldsSubclassPostdominates", 2, 2); + } + + @Test + public void test6() { + test("assignFieldsConditionallySubclassPostdominates", 4, 2); + } + + @Test + public void test7() { + test("assignFieldsConditionallyMixed", 4, 3); + } + + @Test + public void test8() { + test("assignFieldsInLoop", 4, 1); + } + + @Test + public void test9() { + test("assignFieldsInBranches", 3, 2); + } + + @Test + public void test10() { + test("assignFieldsInBranchesMixed", 3, 2); + } + + @Test + public void test11() { + test("invokestatic", 1, 0); + } + + @Test + public void test12() { + test("invokestaticInlined", 1, 1); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/GraalOSRTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/GraalOSRTest.java index 5f164c02505..4094916a181 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/GraalOSRTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/GraalOSRTest.java @@ -53,6 +53,11 @@ public class GraalOSRTest extends GraalOSRTestBase { testOSR(getInitialOptions(), "testNonReduceLoop"); } + @Test + public void testOSR04() { + testOSR(getInitialOptions(), "testDeoptAfterCountedLoop"); + } + static int limit = 10000; public static int sideEffect; @@ -100,4 +105,14 @@ public class GraalOSRTest extends GraalOSRTestBase { GraalDirectives.controlFlowAnchor(); return ret; } + + public static ReturnValue testDeoptAfterCountedLoop() { + long ret = 0; + for (int i = 0; GraalDirectives.injectBranchProbability(1, i < limit * limit); i++) { + GraalDirectives.blackhole(i); + ret = GraalDirectives.opaque(i); + } + GraalDirectives.controlFlowAnchor(); + return ret + 1 == limit * limit ? ReturnValue.SUCCESS : ReturnValue.FAILURE; + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/JVMCIInfopointErrorTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/JVMCIInfopointErrorTest.java index dee9f2ddaea..bb62ff5b9ea 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/JVMCIInfopointErrorTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/JVMCIInfopointErrorTest.java @@ -24,6 +24,7 @@ package org.graalvm.compiler.hotspot.test; +import static org.graalvm.compiler.debug.DebugOptions.DumpOnError; import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK; import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_IGNORED; @@ -52,6 +53,7 @@ import org.graalvm.compiler.nodes.DeoptimizingFixedWithNextNode; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.spi.LIRLowerable; import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; +import org.graalvm.compiler.options.OptionValues; import org.junit.Test; import jdk.vm.ci.code.BytecodeFrame; @@ -139,6 +141,14 @@ public class JVMCIInfopointErrorTest extends GraalCompilerTest { test(getDebugContext(), spec); } + /** + * Avoids dumping during tests which are expected to fail. + */ + private void testNoDump(TestSpec spec) { + OptionValues options = new OptionValues(getInitialOptions(), DumpOnError, false); + test(getDebugContext(options, null, null), spec); + } + private void test(DebugContext debug, TestSpec spec) { ResolvedJavaMethod method = getResolvedJavaMethod("testMethod"); @@ -154,7 +164,7 @@ public class JVMCIInfopointErrorTest extends GraalCompilerTest { @Test(expected = Error.class) public void testInvalidShortOop() { - test((tool, state, safepoint) -> { + testNoDump((tool, state, safepoint) -> { PlatformKind kind = tool.target().arch.getPlatformKind(JavaKind.Short); LIRKind lirKind = LIRKind.reference(kind); @@ -167,7 +177,7 @@ public class JVMCIInfopointErrorTest extends GraalCompilerTest { @Test(expected = Error.class) public void testInvalidShortDerivedOop() { - test((tool, state, safepoint) -> { + testNoDump((tool, state, safepoint) -> { Variable baseOop = tool.newVariable(LIRKind.fromJavaKind(tool.target().arch, JavaKind.Object)); tool.append(new ValueDef(baseOop)); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/OptionsInFileTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/OptionsInFileTest.java index 1e239520cae..41ffb9cd23b 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/OptionsInFileTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/OptionsInFileTest.java @@ -37,6 +37,7 @@ import java.io.PrintStream; import java.util.List; import org.graalvm.compiler.core.test.GraalCompilerTest; +import org.graalvm.compiler.debug.DebugOptions.PrintGraphTarget; import org.graalvm.compiler.test.SubprocessUtil; import org.graalvm.compiler.test.SubprocessUtil.Subprocess; import org.junit.Assert; @@ -54,12 +55,12 @@ public class OptionsInFileTest extends GraalCompilerTest { try { Assert.assertFalse(methodFilterValue.equals(MethodFilter.getDefaultValue())); Assert.assertFalse(debugFilterValue.equals(Dump.getDefaultValue())); - Assert.assertTrue(PrintGraph.getDefaultValue()); + Assert.assertEquals(PrintGraphTarget.File, PrintGraph.getDefaultValue()); try (PrintStream out = new PrintStream(new FileOutputStream(optionsFile))) { out.println(MethodFilter.getName() + "=" + methodFilterValue); out.println(Dump.getName() + "=" + debugFilterValue); - out.println(PrintGraph.getName() + " = false"); + out.println(PrintGraph.getName() + " = Network"); } List vmArgs = withoutDebuggerArguments(getVMCommandLine()); @@ -70,7 +71,7 @@ public class OptionsInFileTest extends GraalCompilerTest { String[] expected = { "graal.MethodFilter := \"a very unlikely method name\"", "graal.Dump := \"a very unlikely debug scope\"", - "graal.PrintGraph := false"}; + "graal.PrintGraph := Network"}; for (String line : proc.output) { for (int i = 0; i < expected.length; i++) { if (expected[i] != null && line.contains(expected[i])) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/ReservedStackAccessTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/ReservedStackAccessTest.java index bfa9439e97c..b3e6898d2fa 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/ReservedStackAccessTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/ReservedStackAccessTest.java @@ -73,6 +73,10 @@ public class ReservedStackAccessTest extends HotSpotGraalCompilerTest { vmArgs.add("-XX:+UseJVMCICompiler"); vmArgs.add("-Dgraal.Inline=false"); vmArgs.add("-XX:CompileCommand=exclude,java/util/concurrent/locks/AbstractOwnableSynchronizer.setExclusiveOwnerThread"); + + // Avoid SOE in HotSpotJVMCIRuntime.adjustCompilationLevel + vmArgs.add("-Dgraal.CompileGraalWithC1Only=false"); + Subprocess proc = SubprocessUtil.java(vmArgs, ReservedStackAccessTest.class.getName()); boolean passed = false; for (String line : proc.output) { @@ -81,9 +85,7 @@ public class ReservedStackAccessTest extends HotSpotGraalCompilerTest { } } if (!passed) { - for (String line : proc.output) { - System.err.println("" + line); - } + System.err.println(proc); } assertTrue(passed); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/WriteBarrierAdditionTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/WriteBarrierAdditionTest.java index ce0f79acbe6..04b4d2b2d8d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/WriteBarrierAdditionTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/WriteBarrierAdditionTest.java @@ -32,10 +32,10 @@ import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase; -import org.graalvm.compiler.hotspot.nodes.G1PostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PreWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1ReferentFieldReadBarrier; -import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1PostWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1PreWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1ReferentFieldReadBarrier; +import org.graalvm.compiler.hotspot.gc.shared.SerialWriteBarrier; import org.graalvm.compiler.hotspot.phases.WriteBarrierAdditionPhase; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/WriteBarrierVerificationTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/WriteBarrierVerificationTest.java index 9069c9b5f4e..4d9404ac073 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/WriteBarrierVerificationTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/WriteBarrierVerificationTest.java @@ -32,12 +32,12 @@ import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.debug.DebugContext.Scope; import org.graalvm.compiler.debug.DebugDumpScope; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; -import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePreWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PreWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.SerialArrayRangeWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1ArrayRangePostWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1ArrayRangePreWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1PostWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1PreWriteBarrier; +import org.graalvm.compiler.hotspot.gc.shared.SerialArrayRangeWriteBarrier; +import org.graalvm.compiler.hotspot.gc.shared.SerialWriteBarrier; import org.graalvm.compiler.hotspot.phases.WriteBarrierAdditionPhase; import org.graalvm.compiler.hotspot.phases.WriteBarrierVerificationPhase; import org.graalvm.compiler.nodes.AbstractBeginNode; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/CompilationTask.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/CompilationTask.java index 0f11edc7f9e..30ebffa4449 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/CompilationTask.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/CompilationTask.java @@ -368,8 +368,14 @@ public class CompilationTask { installedCode = null; Object[] context = {new DebugDumpScope(getIdString(), true), codeCache, getMethod(), compResult}; try (DebugContext.Scope s = debug.scope("CodeInstall", context)) { - installedCode = (HotSpotInstalledCode) backend.createInstalledCode(debug, getRequest().getMethod(), getRequest(), compResult, - getRequest().getMethod().getSpeculationLog(), null, installAsDefault, context); + HotSpotCompilationRequest request = getRequest(); + installedCode = (HotSpotInstalledCode) backend.createInstalledCode(debug, + request.getMethod(), + request, + compResult, + null, + installAsDefault, + context); } catch (Throwable e) { throw debug.handle(e); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/GraalHotSpotVMConfig.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/GraalHotSpotVMConfig.java index dde381ea648..8f8b3766822 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/GraalHotSpotVMConfig.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/GraalHotSpotVMConfig.java @@ -92,6 +92,7 @@ public class GraalHotSpotVMConfig extends GraalHotSpotVMConfigBase { public final boolean useBiasedLocking = getFlag("UseBiasedLocking", Boolean.class); public final boolean usePopCountInstruction = getFlag("UsePopCountInstruction", Boolean.class); public final boolean useAESIntrinsics = getFlag("UseAESIntrinsics", Boolean.class); + public final boolean useAESCTRIntrinsics = getFlag("UseAESCTRIntrinsics", Boolean.class, false); public final boolean useCRC32Intrinsics = getFlag("UseCRC32Intrinsics", Boolean.class); public final boolean useCRC32CIntrinsics = versioned.useCRC32CIntrinsics; public final boolean threadLocalHandshakes = getFlag("ThreadLocalHandshakes", Boolean.class, false); @@ -100,10 +101,12 @@ public class GraalHotSpotVMConfig extends GraalHotSpotVMConfigBase { private final boolean useSHA1Intrinsics = getFlag("UseSHA1Intrinsics", Boolean.class); private final boolean useSHA256Intrinsics = getFlag("UseSHA256Intrinsics", Boolean.class); private final boolean useSHA512Intrinsics = getFlag("UseSHA512Intrinsics", Boolean.class); + private final boolean useGHASHIntrinsics = getFlag("UseGHASHIntrinsics", Boolean.class, false); private final boolean useMontgomeryMultiplyIntrinsic = getFlag("UseMontgomeryMultiplyIntrinsic", Boolean.class, false); private final boolean useMontgomerySquareIntrinsic = getFlag("UseMontgomerySquareIntrinsic", Boolean.class, false); private final boolean useMulAddIntrinsic = getFlag("UseMulAddIntrinsic", Boolean.class, false); private final boolean useSquareToLenIntrinsic = getFlag("UseSquareToLenIntrinsic", Boolean.class, false); + public final boolean useVectorizedMismatchIntrinsic = getFlag("UseVectorizedMismatchIntrinsic", Boolean.class, false); /* * These are methods because in some JDKs the flags are visible but the stubs themselves haven't @@ -114,15 +117,19 @@ public class GraalHotSpotVMConfig extends GraalHotSpotVMConfigBase { } public boolean useSHA1Intrinsics() { - return useSHA1Intrinsics && sha1ImplCompress != 0; + return useSHA1Intrinsics && sha1ImplCompress != 0 && sha1ImplCompressMultiBlock != 0; } public boolean useSHA256Intrinsics() { - return useSHA256Intrinsics && sha256ImplCompress != 0; + return useSHA256Intrinsics && sha256ImplCompress != 0 && sha256ImplCompressMultiBlock != 0; } public boolean useSHA512Intrinsics() { - return useSHA512Intrinsics && sha512ImplCompress != 0; + return useSHA512Intrinsics && sha512ImplCompress != 0 && sha512ImplCompressMultiBlock != 0; + } + + public boolean useGHASHIntrinsics() { + return useGHASHIntrinsics && ghashProcessBlocks != 0; } public boolean useMontgomeryMultiplyIntrinsic() { @@ -309,7 +316,17 @@ public class GraalHotSpotVMConfig extends GraalHotSpotVMConfigBase { public final int jvmAccWrittenFlags = getConstant("JVM_ACC_WRITTEN_FLAGS", Integer.class); public final int jvmAccSynthetic = getConstant("JVM_ACC_SYNTHETIC", Integer.class); - public final int jvmciCompileStateCanPostOnExceptionsOffset = getFieldOffset("JVMCIEnv::_jvmti_can_post_on_exceptions", Integer.class, "jbyte", Integer.MIN_VALUE); + public final int jvmciCompileStateCanPostOnExceptionsOffset = getJvmciCompileStateCanPostOnExceptionsOffset(); + + // Integer.MIN_VALUE if not available + private int getJvmciCompileStateCanPostOnExceptionsOffset() { + int offset = getFieldOffset("JVMCICompileState::_jvmti_can_post_on_exceptions", Integer.class, "jbyte", Integer.MIN_VALUE); + if (offset == Integer.MIN_VALUE) { + // JDK 12 + offset = getFieldOffset("JVMCIEnv::_jvmti_can_post_on_exceptions", Integer.class, "jbyte", Integer.MIN_VALUE); + } + return offset; + } public final int threadTlabOffset = getFieldOffset("Thread::_tlab", Integer.class, "ThreadLocalAllocBuffer"); public final int javaThreadAnchorOffset = getFieldOffset("JavaThread::_anchor", Integer.class, "JavaFrameAnchor"); @@ -372,12 +389,29 @@ public class GraalHotSpotVMConfig extends GraalHotSpotVMConfigBase { public final int pendingExceptionOffset = getFieldOffset("ThreadShadow::_pending_exception", Integer.class, "oop"); public final int pendingDeoptimizationOffset = getFieldOffset("JavaThread::_pending_deoptimization", Integer.class, "int"); - public final int pendingFailedSpeculationOffset = getFieldOffset("JavaThread::_pending_failed_speculation", Integer.class, "long"); public final int pendingTransferToInterpreterOffset = getFieldOffset("JavaThread::_pending_transfer_to_interpreter", Integer.class, "bool"); private final int javaFrameAnchorLastJavaSpOffset = getFieldOffset("JavaFrameAnchor::_last_Java_sp", Integer.class, "intptr_t*"); private final int javaFrameAnchorLastJavaPcOffset = getFieldOffset("JavaFrameAnchor::_last_Java_pc", Integer.class, "address"); + public final int pendingFailedSpeculationOffset; + { + String name = "JavaThread::_pending_failed_speculation"; + int offset = -1; + try { + offset = getFieldOffset(name, Integer.class, "jlong"); + } catch (JVMCIError e) { + try { + offset = getFieldOffset(name, Integer.class, "long"); + } catch (JVMCIError e2) { + } + } + if (offset == -1) { + throw new JVMCIError("cannot get offset of field " + name + " with type long or jlong"); + } + pendingFailedSpeculationOffset = offset; + } + public int threadLastJavaSpOffset() { return javaThreadAnchorOffset + javaFrameAnchorLastJavaSpOffset; } @@ -602,11 +636,11 @@ public class GraalHotSpotVMConfig extends GraalHotSpotVMConfigBase { public final long crcTableAddress = getFieldValue("StubRoutines::_crc_table_adr", Long.class, "address"); public final long sha1ImplCompress = getFieldValue("StubRoutines::_sha1_implCompress", Long.class, "address", 0L); - public final long sha1ImplCompressMB = getFieldValue("StubRoutines::_sha1_implCompressMB", Long.class, "address", 0L); + public final long sha1ImplCompressMultiBlock = getFieldValue("StubRoutines::_sha1_implCompressMB", Long.class, "address", 0L); public final long sha256ImplCompress = getFieldValue("StubRoutines::_sha256_implCompress", Long.class, "address", 0L); - public final long sha256ImplCompressMB = getFieldValue("StubRoutines::_sha256_implCompressMB", Long.class, "address", 0L); + public final long sha256ImplCompressMultiBlock = getFieldValue("StubRoutines::_sha256_implCompressMB", Long.class, "address", 0L); public final long sha512ImplCompress = getFieldValue("StubRoutines::_sha512_implCompress", Long.class, "address", 0L); - public final long sha512ImplCompressMB = getFieldValue("StubRoutines::_sha512_implCompressMB", Long.class, "address", 0L); + public final long sha512ImplCompressMultiBlock = getFieldValue("StubRoutines::_sha512_implCompressMB", Long.class, "address", 0L); public final long multiplyToLen = getFieldValue("StubRoutines::_multiplyToLen", Long.class, "address", longRequiredOnAMD64); public final long counterModeAESCrypt = getFieldValue("StubRoutines::_counterMode_AESCrypt", Long.class, "address", 0L); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotBackend.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotBackend.java index 562da1fa46b..dba7f718cbf 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotBackend.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotBackend.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -46,6 +46,7 @@ import org.graalvm.compiler.hotspot.nodes.aot.ResolveConstantStubCall; import org.graalvm.compiler.hotspot.replacements.AESCryptSubstitutions; import org.graalvm.compiler.hotspot.replacements.BigIntegerSubstitutions; import org.graalvm.compiler.hotspot.replacements.CipherBlockChainingSubstitutions; +import org.graalvm.compiler.hotspot.replacements.DigestBaseSubstitutions; import org.graalvm.compiler.hotspot.replacements.SHA2Substitutions; import org.graalvm.compiler.hotspot.replacements.SHA5Substitutions; import org.graalvm.compiler.hotspot.replacements.SHASubstitutions; @@ -71,6 +72,7 @@ import org.graalvm.compiler.options.OptionKey; import org.graalvm.compiler.options.OptionType; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.tiers.SuitesProvider; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import org.graalvm.compiler.word.Word; import jdk.internal.vm.compiler.word.Pointer; @@ -95,7 +97,7 @@ public abstract class HotSpotBackend extends Backend implements FrameMap.Referen public static class Options { // @formatter:off @Option(help = "Use Graal arithmetic stubs instead of HotSpot stubs where possible") - public static final OptionKey GraalArithmeticStubs = new OptionKey<>(false); // GR-8276 + public static final OptionKey GraalArithmeticStubs = new OptionKey<>(JavaVersionUtil.JAVA_SPECIFICATION_VERSION >= 9); @Option(help = "Enables instruction profiling on assembler level. Valid values are a comma separated list of supported instructions." + " Compare with subclasses of Assembler.InstructionCounter.", type = OptionType.Debug) public static final OptionKey ASMInstructionProfiling = new OptionKey<>(null); @@ -261,6 +263,36 @@ public abstract class HotSpotBackend extends Backend implements FrameMap.Referen @NodeIntrinsic(ForeignCallNode.class) private static native void sha5ImplCompressStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word bufAddr, Object state); + /** + * @see DigestBaseSubstitutions#implCompressMultiBlock0 + */ + public static final ForeignCallDescriptor SHA_IMPL_COMPRESS_MB = new ForeignCallDescriptor("shaImplCompressMB", int.class, Word.class, Object.class, int.class, int.class); + + public static int shaImplCompressMBStub(Word bufAddr, Object stateAddr, int ofs, int limit) { + return shaImplCompressMBStub(HotSpotBackend.SHA_IMPL_COMPRESS_MB, bufAddr, stateAddr, ofs, limit); + } + + @NodeIntrinsic(ForeignCallNode.class) + private static native int shaImplCompressMBStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word bufAddr, Object state, int ofs, int limit); + + public static final ForeignCallDescriptor SHA2_IMPL_COMPRESS_MB = new ForeignCallDescriptor("sha2ImplCompressMB", int.class, Word.class, Object.class, int.class, int.class); + + public static int sha2ImplCompressMBStub(Word bufAddr, Object stateAddr, int ofs, int limit) { + return sha2ImplCompressMBStub(HotSpotBackend.SHA2_IMPL_COMPRESS_MB, bufAddr, stateAddr, ofs, limit); + } + + @NodeIntrinsic(ForeignCallNode.class) + private static native int sha2ImplCompressMBStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word bufAddr, Object state, int ofs, int limit); + + public static final ForeignCallDescriptor SHA5_IMPL_COMPRESS_MB = new ForeignCallDescriptor("sha5ImplCompressMB", int.class, Word.class, Object.class, int.class, int.class); + + public static int sha5ImplCompressMBStub(Word bufAddr, Object stateAddr, int ofs, int limit) { + return sha5ImplCompressMBStub(HotSpotBackend.SHA5_IMPL_COMPRESS_MB, bufAddr, stateAddr, ofs, limit); + } + + @NodeIntrinsic(ForeignCallNode.class) + private static native int sha5ImplCompressMBStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word bufAddr, Object state, int ofs, int limit); + public static void unsafeArraycopy(Word srcAddr, Word dstAddr, Word size) { unsafeArraycopyStub(UNSAFE_ARRAYCOPY, srcAddr, dstAddr, size); } @@ -268,6 +300,37 @@ public abstract class HotSpotBackend extends Backend implements FrameMap.Referen @NodeIntrinsic(ForeignCallNode.class) private static native void unsafeArraycopyStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word srcAddr, Word dstAddr, Word size); + /** + * Descriptor for {@code StubRoutines::_ghash_processBlocks}. + */ + public static final ForeignCallDescriptor GHASH_PROCESS_BLOCKS = new ForeignCallDescriptor("ghashProcessBlocks", void.class, Word.class, Word.class, Word.class, int.class); + + /** + * Descriptor for {@code StubRoutines::_counterMode_AESCrypt}. + */ + public static final ForeignCallDescriptor COUNTERMODE_IMPL_CRYPT = new ForeignCallDescriptor("counterModeAESCrypt", int.class, Word.class, Word.class, Word.class, Word.class, int.class, + Word.class, Word.class); + + public static int counterModeAESCrypt(Word srcAddr, Word dstAddr, Word kPtr, Word cntPtr, int len, Word encCntPtr, Word used) { + return counterModeAESCrypt(COUNTERMODE_IMPL_CRYPT, srcAddr, dstAddr, kPtr, cntPtr, len, encCntPtr, used); + } + + @NodeIntrinsic(ForeignCallNode.class) + private static native int counterModeAESCrypt(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word srcAddr, Word dstAddr, Word kPtr, Word cntPtr, int len, Word encCntPtr, + Word used); + + /** + * Descriptor for {@code StubRoutines::_vectorizedMismatch}. + */ + public static final ForeignCallDescriptor VECTORIZED_MISMATCHED = new ForeignCallDescriptor("vectorizedMismatch", int.class, Word.class, Word.class, int.class, int.class); + + public static int vectorizedMismatch(Word aAddr, Word bAddr, int length, int log2ArrayIndexScale) { + return vectorizedMismatchStub(VECTORIZED_MISMATCHED, aAddr, bAddr, length, log2ArrayIndexScale); + } + + @NodeIntrinsic(ForeignCallNode.class) + private static native int vectorizedMismatchStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word aAddr, Word bAddr, int length, int log2ArrayIndexScale); + /** * @see VMErrorNode */ @@ -388,6 +451,13 @@ public abstract class HotSpotBackend extends Backend implements FrameMap.Referen return translateToCallerRegisters(destroyedRegisters); } + /** + * Translates a set of registers from the callee's perspective to the caller's perspective. This + * is needed for architectures where input/output registers are renamed during a call (e.g. + * register windows on SPARC). Registers which are not visible by the caller are removed. + */ + protected abstract EconomicSet translateToCallerRegisters(EconomicSet calleeRegisters); + /** * Updates a given stub with respect to the registers it destroys. *

diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotCompiledCodeBuilder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotCompiledCodeBuilder.java index ce8e7ac4056..6481fdb9c98 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotCompiledCodeBuilder.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotCompiledCodeBuilder.java @@ -88,7 +88,7 @@ public class HotSpotCompiledCodeBuilder { ResolvedJavaMethod[] methods = compResult.getMethods(); - List annotations = compResult.getAnnotations(); + List annotations = compResult.getCodeAnnotations(); Comment[] comments = new Comment[annotations.size()]; if (!annotations.isEmpty()) { for (int i = 0; i < comments.length; i++) { @@ -129,16 +129,16 @@ public class HotSpotCompiledCodeBuilder { boolean hasUnsafeAccess = compResult.hasUnsafeAccess(); int id; - long jvmciEnv; + long jvmciCompileState; if (compRequest != null) { id = compRequest.getId(); - jvmciEnv = compRequest.getJvmciEnv(); + jvmciCompileState = compRequest.getJvmciEnv(); } else { id = hsMethod.allocateCompileId(entryBCI); - jvmciEnv = 0L; + jvmciCompileState = 0L; } return new HotSpotCompiledNmethod(name, targetCode, targetCodeSize, sites, assumptions, methods, comments, dataSection, dataSectionAlignment, dataSectionPatches, isImmutablePIC, - totalFrameSize, customStackArea, hsMethod, entryBCI, id, jvmciEnv, hasUnsafeAccess); + totalFrameSize, customStackArea, hsMethod, entryBCI, id, jvmciCompileState, hasUnsafeAccess); } else { return new HotSpotCompiledCode(name, targetCode, targetCodeSize, sites, assumptions, methods, comments, dataSection, dataSectionAlignment, dataSectionPatches, isImmutablePIC, totalFrameSize, customStackArea); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotDataBuilder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotDataBuilder.java index d5f504fd70b..587590ee493 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotDataBuilder.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotDataBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -50,12 +50,6 @@ public class HotSpotDataBuilder extends DataBuilder { this.target = target; } - @Override - public boolean needDetailedPatchingInformation() { - /* The HotSpot VM finds operands that need patching by decoding the instruction. */ - return false; - } - @Override public Data createDataItem(Constant constant) { if (JavaConstant.isNull(constant)) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalCompilerFactory.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalCompilerFactory.java index 37f02408d4f..c6734b607c1 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalCompilerFactory.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalCompilerFactory.java @@ -25,6 +25,7 @@ package org.graalvm.compiler.hotspot; import static jdk.vm.ci.common.InitTimer.timer; +import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; import static org.graalvm.compiler.hotspot.HotSpotGraalOptionValues.GRAAL_OPTION_PROPERTY_PREFIX; import java.io.PrintStream; @@ -71,7 +72,7 @@ public final class HotSpotGraalCompilerFactory extends HotSpotJVMCICompilerFacto public void onSelection() { JVMCIVersionCheck.check(false); assert options == null : "cannot select " + getClass() + " service more than once"; - options = HotSpotGraalOptionValues.HOTSPOT_OPTIONS; + options = HotSpotGraalOptionValues.defaultOptions(); initializeGraalCompilePolicyFields(options); isGraalPredicate = compileGraalWithC1Only ? new IsGraalPredicate() : null; /* @@ -80,6 +81,10 @@ public final class HotSpotGraalCompilerFactory extends HotSpotJVMCICompilerFacto */ adjustCompilationLevelInternal(Object.class, "hashCode", "()I", CompilationLevel.FullOptimization); adjustCompilationLevelInternal(Object.class, "hashCode", "()I", CompilationLevel.Simple); + if (IS_BUILDING_NATIVE_IMAGE) { + // Triggers initialization of all option descriptors + Options.CompileGraalWithC1Only.getName(); + } } private static void initializeGraalCompilePolicyFields(OptionValues options) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalJVMCIServiceLocator.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalJVMCIServiceLocator.java index 1279bdb1535..975c6ac607c 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalJVMCIServiceLocator.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalJVMCIServiceLocator.java @@ -26,6 +26,7 @@ package org.graalvm.compiler.hotspot; import org.graalvm.compiler.serviceprovider.ServiceProvider; +import jdk.vm.ci.common.NativeImageReinitialize; import jdk.vm.ci.hotspot.HotSpotVMEventListener; import jdk.vm.ci.runtime.JVMCICompilerFactory; import jdk.vm.ci.services.JVMCIServiceLocator; @@ -51,7 +52,7 @@ public final class HotSpotGraalJVMCIServiceLocator extends JVMCIServiceLocator { return null; } - private HotSpotGraalRuntime graalRuntime; + @NativeImageReinitialize private HotSpotGraalRuntime graalRuntime; /** * Notifies this object of the compiler created via {@link HotSpotGraalJVMCIServiceLocator}. diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalOptionValues.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalOptionValues.java index 3b3b12135b8..20a01bd7c89 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalOptionValues.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalOptionValues.java @@ -40,9 +40,10 @@ import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.options.OptionsParser; import jdk.vm.ci.common.InitTimer; +import jdk.vm.ci.common.NativeImageReinitialize; /** - * The {@link #HOTSPOT_OPTIONS} value contains the options values initialized in a HotSpot VM. The + * The {@link #defaultOptions()} method returns the options values initialized in a HotSpot VM. The * values are set via system properties with the {@value #GRAAL_OPTION_PROPERTY_PREFIX} prefix. */ public class HotSpotGraalOptionValues { @@ -71,7 +72,21 @@ public class HotSpotGraalOptionValues { return GRAAL_OPTION_PROPERTY_PREFIX + value.getName() + "=" + value.getValue(options); } - public static final OptionValues HOTSPOT_OPTIONS = initializeOptions(); + @NativeImageReinitialize private static volatile OptionValues hotspotOptions; + + public static OptionValues defaultOptions() { + OptionValues res = hotspotOptions; + if (res == null) { + synchronized (HotSpotGraalOptionValues.class) { + res = hotspotOptions; + if (res == null) { + res = initializeOptions(); + hotspotOptions = res; + } + } + } + return res; + } /** * Global options. The values for these options are initialized by parsing the file denoted by diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalRuntime.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalRuntime.java index 62d7ded960b..fac77606ce9 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalRuntime.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotGraalRuntime.java @@ -95,6 +95,8 @@ import jdk.vm.ci.runtime.JVMCIBackend; */ public final class HotSpotGraalRuntime implements HotSpotGraalRuntimeProvider { + private static final boolean IS_AOT = Boolean.getBoolean("com.oracle.graalvm.isaot"); + private static boolean checkArrayIndexScaleInvariants(MetaAccessProvider metaAccess) { assert metaAccess.getArrayIndexScale(JavaKind.Byte) == 1; assert metaAccess.getArrayIndexScale(JavaKind.Boolean) == 1; @@ -159,9 +161,13 @@ public final class HotSpotGraalRuntime implements HotSpotGraalRuntimeProvider { compilerConfigurationName = compilerConfigurationFactory.getName(); compiler = new HotSpotGraalCompiler(jvmciRuntime, this, options); - management = GraalServices.loadSingle(HotSpotGraalManagementRegistration.class, false); - if (management != null) { - management.initialize(this); + if (IS_AOT) { + management = null; + } else { + management = GraalServices.loadSingle(HotSpotGraalManagementRegistration.class, false); + if (management != null) { + management.initialize(this); + } } BackendMap backendMap = compilerConfigurationFactory.createBackendMap(); @@ -571,7 +577,7 @@ public final class HotSpotGraalRuntime implements HotSpotGraalRuntimeProvider { EconomicMap, Object> extra = EconomicMap.create(); extra.put(DebugOptions.Dump, filter); extra.put(DebugOptions.PrintGraphHost, host); - extra.put(DebugOptions.PrintBinaryGraphPort, port); + extra.put(DebugOptions.PrintGraphPort, port); OptionValues compileOptions = new OptionValues(getOptions(), extra); compiler.compileMethod(new HotSpotCompilationRequest(hotSpotMethod, -1, 0L), false, compileOptions); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotReplacementsImpl.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotReplacementsImpl.java index 6d68f073414..8f5a70d0149 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotReplacementsImpl.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotReplacementsImpl.java @@ -24,17 +24,34 @@ package org.graalvm.compiler.hotspot; +import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; +import static org.graalvm.compiler.replacements.ReplacementsImpl.Options.UseEncodedSnippets; + +import java.util.Set; + +import jdk.internal.vm.compiler.collections.EconomicSet; import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; import org.graalvm.compiler.bytecode.BytecodeProvider; +import org.graalvm.compiler.core.common.CompilationIdentifier; +import org.graalvm.compiler.debug.DebugContext; +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.graph.NodeSourcePosition; import org.graalvm.compiler.hotspot.meta.HotSpotWordOperationPlugin; import org.graalvm.compiler.hotspot.word.HotSpotOperation; +import org.graalvm.compiler.nodes.Invoke; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderPlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.MethodSubstitutionPlugin; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.util.Providers; import org.graalvm.compiler.printer.GraalDebugHandlersFactory; import org.graalvm.compiler.replacements.ReplacementsImpl; import jdk.vm.ci.code.TargetDescription; +import jdk.vm.ci.common.NativeImageReinitialize; import jdk.vm.ci.meta.ResolvedJavaMethod; /** @@ -46,21 +63,149 @@ public class HotSpotReplacementsImpl extends ReplacementsImpl { super(options, new GraalDebugHandlersFactory(snippetReflection), providers, snippetReflection, bytecodeProvider, target); } + protected HotSpotReplacementsImpl(HotSpotReplacementsImpl replacements, Providers providers) { + super(replacements.options, new GraalDebugHandlersFactory(replacements.snippetReflection), providers, replacements.snippetReflection, + replacements.getDefaultReplacementBytecodeProvider(), replacements.target); + } + @Override public Class getIntrinsifyingPlugin(ResolvedJavaMethod method) { return method.getAnnotation(HotSpotOperation.class) != null ? HotSpotWordOperationPlugin.class : super.getIntrinsifyingPlugin(method); } + public void registerMethodSubstitution(ResolvedJavaMethod method, ResolvedJavaMethod original) { + if (!IS_IN_NATIVE_IMAGE) { + if (IS_BUILDING_NATIVE_IMAGE || UseEncodedSnippets.getValue(options)) { + synchronized (HotSpotReplacementsImpl.class) { + if (snippetEncoder == null) { + snippetEncoder = new SymbolicSnippetEncoder(this); + } + snippetEncoder.registerMethodSubstitution(method, original); + } + } + } + } + + @Override + public StructuredGraph getIntrinsicGraph(ResolvedJavaMethod method, CompilationIdentifier compilationId, DebugContext debug) { + if (IS_IN_NATIVE_IMAGE) { + HotSpotReplacementsImpl replacements = (HotSpotReplacementsImpl) providers.getReplacements(); + InvocationPlugin plugin = replacements.getGraphBuilderPlugins().getInvocationPlugins().lookupInvocation(method); + if (plugin instanceof MethodSubstitutionPlugin) { + MethodSubstitutionPlugin msp = (MethodSubstitutionPlugin) plugin; + return replacements.getMethodSubstitution(msp, method); + } + return null; + } + return super.getIntrinsicGraph(method, compilationId, debug); + } + + @Override + public void notifyNotInlined(GraphBuilderContext b, ResolvedJavaMethod method, Invoke invoke) { + if (b.parsingIntrinsic() && snippetEncoder != null) { + if (getIntrinsifyingPlugin(method) != null) { + snippetEncoder.addDelayedInvocationPluginMethod(method); + return; + } + } + super.notifyNotInlined(b, method, invoke); + } + + // When assertions are enabled, these fields are used to ensure all snippets are + // registered during Graal initialization which in turn ensures that native image + // building will not miss any snippets. + @NativeImageReinitialize private EconomicSet registeredSnippets = EconomicSet.create(); private boolean snippetRegistrationClosed; @Override public void registerSnippet(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean trackNodeSourcePosition) { - assert !snippetRegistrationClosed; - super.registerSnippet(method, original, receiver, trackNodeSourcePosition); + if (!IS_IN_NATIVE_IMAGE) { + assert !snippetRegistrationClosed : "Cannot register snippet after registration is closed: " + method.format("%H.%n(%p)"); + assert registeredSnippets.add(method) : "Cannot register snippet twice: " + method.format("%H.%n(%p)"); + if (IS_BUILDING_NATIVE_IMAGE || UseEncodedSnippets.getValue(options)) { + synchronized (HotSpotReplacementsImpl.class) { + if (snippetEncoder == null) { + snippetEncoder = new SymbolicSnippetEncoder(this); + } + snippetEncoder.registerSnippet(method, original, receiver, trackNodeSourcePosition); + } + } + } } @Override public void closeSnippetRegistration() { snippetRegistrationClosed = true; } + + static SymbolicSnippetEncoder.EncodedSnippets getEncodedSnippets() { + return encodedSnippets; + } + + public Set getSnippetMethods() { + if (snippetEncoder != null) { + return snippetEncoder.getSnippetMethods(); + } + return null; + } + + static void setEncodedSnippets(SymbolicSnippetEncoder.EncodedSnippets encodedSnippets) { + HotSpotReplacementsImpl.encodedSnippets = encodedSnippets; + } + + public boolean encode() { + SymbolicSnippetEncoder encoder = HotSpotReplacementsImpl.snippetEncoder; + if (encoder != null) { + return encoder.encode(); + } + return false; + } + + private static volatile SymbolicSnippetEncoder.EncodedSnippets encodedSnippets; + + @NativeImageReinitialize static SymbolicSnippetEncoder snippetEncoder; + + @Override + public StructuredGraph getSnippet(ResolvedJavaMethod method, ResolvedJavaMethod recursiveEntry, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) { + StructuredGraph graph = getEncodedSnippet(method, args); + if (graph != null) { + return graph; + } + + assert !IS_IN_NATIVE_IMAGE : "should be using encoded snippets"; + return super.getSnippet(method, recursiveEntry, args, trackNodeSourcePosition, replaceePosition); + } + + public StructuredGraph getEncodedSnippet(ResolvedJavaMethod method, Object[] args) { + if (IS_IN_NATIVE_IMAGE || UseEncodedSnippets.getValue(options)) { + synchronized (HotSpotReplacementsImpl.class) { + if (!IS_IN_NATIVE_IMAGE && UseEncodedSnippets.getValue(options)) { + snippetEncoder.encode(); + } + + if (getEncodedSnippets() == null) { + throw GraalError.shouldNotReachHere("encoded snippets not found"); + } + StructuredGraph graph = getEncodedSnippets().getEncodedSnippet(method, this, args); + if (graph == null) { + throw GraalError.shouldNotReachHere("snippet not found: " + method.format("%H.%n(%p)")); + } + return graph; + } + } else if (registeredSnippets != null) { + assert registeredSnippets.contains(method) : "Asking for snippet method that was never registered: " + method.format("%H.%n(%p)"); + } + return null; + } + + public StructuredGraph getMethodSubstitution(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original) { + if (IS_IN_NATIVE_IMAGE || UseEncodedSnippets.getValue(options)) { + if (getEncodedSnippets() == null) { + throw GraalError.shouldNotReachHere("encoded snippets not found"); + } + return getEncodedSnippets().getMethodSubstitutionGraph(plugin, original, this); + } + return null; + } + } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotTTYStreamProvider.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotTTYStreamProvider.java index 31aee425fb5..31fda80bd9d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotTTYStreamProvider.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/HotSpotTTYStreamProvider.java @@ -25,7 +25,7 @@ package org.graalvm.compiler.hotspot; import static org.graalvm.compiler.hotspot.HotSpotGraalOptionValues.GRAAL_OPTION_PROPERTY_PREFIX; -import static org.graalvm.compiler.hotspot.HotSpotGraalOptionValues.HOTSPOT_OPTIONS; +import static org.graalvm.compiler.hotspot.HotSpotGraalOptionValues.defaultOptions; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -59,7 +59,7 @@ public class HotSpotTTYStreamProvider implements TTYStreamProvider { @Override public PrintStream getStream() { - return Options.LogFile.getStream(HOTSPOT_OPTIONS); + return Options.LogFile.getStream(defaultOptions()); } /** diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/JVMCIVersionCheck.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/JVMCIVersionCheck.java index 42aab4eef91..46695a57f04 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/JVMCIVersionCheck.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/JVMCIVersionCheck.java @@ -38,8 +38,9 @@ import java.util.Formatter; */ class JVMCIVersionCheck { + // 0.55 introduces new HotSpotSpeculationLog API private static final int JVMCI8_MIN_MAJOR_VERSION = 0; - private static final int JVMCI8_MIN_MINOR_VERSION = 46; + private static final int JVMCI8_MIN_MINOR_VERSION = 55; private static void failVersionCheck(boolean exit, String reason, Object... args) { Formatter errorMessage = new Formatter().format(reason, args); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/SymbolicSnippetEncoder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/SymbolicSnippetEncoder.java new file mode 100644 index 00000000000..fc48a86bf05 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/SymbolicSnippetEncoder.java @@ -0,0 +1,1034 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot; + +import static jdk.vm.ci.runtime.JVMCI.getRuntime; +import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; +import static org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin.InlineInfo.createIntrinsicInlineInfo; +import static org.graalvm.compiler.replacements.ReplacementsImpl.Options.UseEncodedSnippets; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.compiler.api.replacements.MethodSubstitution; +import org.graalvm.compiler.api.replacements.Snippet; +import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; +import org.graalvm.compiler.api.runtime.GraalJVMCICompiler; +import org.graalvm.compiler.api.runtime.GraalRuntime; +import org.graalvm.compiler.bytecode.BytecodeProvider; +import org.graalvm.compiler.core.common.spi.ConstantFieldProvider; +import org.graalvm.compiler.core.common.type.AbstractObjectStamp; +import org.graalvm.compiler.core.common.type.Stamp; +import org.graalvm.compiler.core.common.type.StampPair; +import org.graalvm.compiler.core.common.type.SymbolicJVMCIReference; +import org.graalvm.compiler.debug.DebugContext; +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.graph.Node; +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.graph.NodeMap; +import org.graalvm.compiler.graph.NodeSourcePosition; +import org.graalvm.compiler.hotspot.meta.HotSpotForeignCallsProvider; +import org.graalvm.compiler.hotspot.meta.HotSpotProviders; +import org.graalvm.compiler.java.BytecodeParser; +import org.graalvm.compiler.java.GraphBuilderPhase; +import org.graalvm.compiler.nodeinfo.Verbosity; +import org.graalvm.compiler.nodes.CallTargetNode; +import org.graalvm.compiler.nodes.ConstantNode; +import org.graalvm.compiler.nodes.EncodedGraph; +import org.graalvm.compiler.nodes.FrameState; +import org.graalvm.compiler.nodes.FullInfopointNode; +import org.graalvm.compiler.nodes.GraphEncoder; +import org.graalvm.compiler.nodes.ParameterNode; +import org.graalvm.compiler.nodes.ProxyNode; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.cfg.Block; +import org.graalvm.compiler.nodes.graphbuilderconf.GeneratedInvocationPlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration; +import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; +import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext; +import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins; +import org.graalvm.compiler.nodes.graphbuilderconf.MethodSubstitutionPlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.NodePlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.ParameterPlugin; +import org.graalvm.compiler.nodes.java.AccessFieldNode; +import org.graalvm.compiler.nodes.java.MethodCallTargetNode; +import org.graalvm.compiler.nodes.spi.DelegatingReplacements; +import org.graalvm.compiler.nodes.spi.StampProvider; +import org.graalvm.compiler.nodes.virtual.VirtualObjectNode; +import org.graalvm.compiler.options.OptionValues; +import org.graalvm.compiler.phases.OptimisticOptimizations; +import org.graalvm.compiler.phases.schedule.SchedulePhase; +import org.graalvm.compiler.phases.util.Providers; +import org.graalvm.compiler.replacements.ConstantBindingParameterPlugin; +import org.graalvm.compiler.replacements.PEGraphDecoder; +import org.graalvm.compiler.replacements.ReplacementsImpl; +import org.graalvm.compiler.replacements.SnippetCounter; +import org.graalvm.compiler.replacements.SnippetIntegerHistogram; + +import jdk.vm.ci.code.Architecture; +import jdk.vm.ci.code.TargetDescription; +import jdk.vm.ci.hotspot.HotSpotJVMCIRuntime; +import jdk.vm.ci.hotspot.HotSpotResolvedJavaField; +import jdk.vm.ci.hotspot.HotSpotResolvedJavaMethod; +import jdk.vm.ci.hotspot.HotSpotResolvedJavaType; +import jdk.vm.ci.meta.Constant; +import jdk.vm.ci.meta.ConstantReflectionProvider; +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.JavaType; +import jdk.vm.ci.meta.MemoryAccessProvider; +import jdk.vm.ci.meta.MetaAccessProvider; +import jdk.vm.ci.meta.MethodHandleAccessProvider; +import jdk.vm.ci.meta.ResolvedJavaField; +import jdk.vm.ci.meta.ResolvedJavaMethod; +import jdk.vm.ci.meta.ResolvedJavaType; +import jdk.vm.ci.meta.UnresolvedJavaField; +import jdk.vm.ci.meta.UnresolvedJavaMethod; +import jdk.vm.ci.meta.UnresolvedJavaType; + +/** + * This class performs graph encoding using {@link GraphEncoder} but also converts JVMCI type and + * method references into a symbolic form that can be resolved at graph decode time using + * {@link SymbolicJVMCIReference}. + */ +public class SymbolicSnippetEncoder extends DelegatingReplacements { + + /** + * This is a customized HotSpotReplacementsImpl intended only for parsing snippets and method + * substitutions for graph encoding. + */ + private final HotSpotSnippetReplacementsImpl replacements; + + /** + * The set of all snippet methods that have been encoded. + */ + private final Set snippetMethods = Collections.synchronizedSet(new HashSet<>()); + + /** + * A mapping from the method substitution method to the original method name. The string key and + * values are produced using {@link #methodKey(ResolvedJavaMethod)}. + */ + private final Map originalMethods = new ConcurrentHashMap<>(); + + /** + * The current count of graphs encoded. Used to detect when new graphs have been enqueued for + * encoding. + */ + int encodedGraphs = 0; + + /** + * All the graphs parsed so far. + */ + private Map preparedSnippetGraphs = new HashMap<>(); + + /** + * The invocation plugins which were delayed during graph preparation. + */ + private Set delayedInvocationPluginMethods = new HashSet<>(); + + void addDelayedInvocationPluginMethod(ResolvedJavaMethod method) { + delayedInvocationPluginMethods.add(method); + } + + Set getSnippetMethods() { + return snippetMethods; + } + + protected class SnippetInlineInvokePlugin implements InlineInvokePlugin { + + @Override + public InlineInfo shouldInlineInvoke(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode[] args) { + if (method.getAnnotation(Fold.class) != null) { + delayedInvocationPluginMethods.add(method); + return InlineInfo.DO_NOT_INLINE_NO_EXCEPTION; + } + + if (getIntrinsifyingPlugin(method) != null) { + delayedInvocationPluginMethods.add(method); + return InlineInfo.DO_NOT_INLINE_NO_EXCEPTION; + } + + // Force inlining when parsing replacements + return createIntrinsicInlineInfo(method, null, getDefaultReplacementBytecodeProvider()); + } + + @Override + public void notifyAfterInline(ResolvedJavaMethod methodToInline) { + assert methodToInline.getAnnotation(Fold.class) == null : methodToInline; + } + } + + public static class SnippetInvocationPlugins extends InvocationPlugins { + + SnippetInvocationPlugins(InvocationPlugins invocationPlugins) { + super(invocationPlugins); + } + + @Override + public InvocationPlugin lookupInvocation(ResolvedJavaMethod method) { + if (method.getAnnotation(Fold.class) != null) { + return null; + } + return super.lookupInvocation(method); + } + } + + /** + * This plugin disables the snippet counter machinery. + */ + private class SnippetCounterPlugin implements NodePlugin { + String snippetCounterName = 'L' + SnippetCounter.class.getName().replace('.', '/') + ';'; + String snippetIntegerHistogramName = 'L' + SnippetIntegerHistogram.class.getName().replace('.', '/') + ';'; + + @Override + public boolean handleLoadField(GraphBuilderContext b, ValueNode object, ResolvedJavaField field) { + if (field.getName().equals("group") && field.getDeclaringClass().getName().equals(snippetCounterName)) { + b.addPush(JavaKind.Object, ConstantNode.forConstant(JavaConstant.NULL_POINTER, b.getMetaAccess())); + return true; + } + if (field.getType().getName().equals(snippetCounterName)) { + b.addPush(JavaKind.Object, ConstantNode.forConstant(replacements.snippetReflection.forObject(SnippetCounter.DISABLED_COUNTER), b.getMetaAccess())); + return true; + } + + if (field.getType().getName().equals(snippetIntegerHistogramName)) { + b.addPush(JavaKind.Object, ConstantNode.forConstant(replacements.snippetReflection.forObject(SnippetIntegerHistogram.DISABLED_COUNTER), b.getMetaAccess())); + return true; + } + return false; + } + } + + /** + * Generate a String name for a method including all type information. Used as a symbolic key + * for lookup. + */ + private static String methodKey(ResolvedJavaMethod method) { + return method.format("%f %H.%n(%P)"); + } + + SymbolicSnippetEncoder(HotSpotReplacementsImpl replacements) { + super(replacements); + + GraphBuilderConfiguration.Plugins plugins = replacements.getGraphBuilderPlugins(); + SnippetInvocationPlugins invocationPlugins = new SnippetInvocationPlugins(plugins.getInvocationPlugins()); + GraphBuilderConfiguration.Plugins copy = new GraphBuilderConfiguration.Plugins(plugins, invocationPlugins); + copy.clearInlineInvokePlugins(); + copy.appendInlineInvokePlugin(new SnippetInlineInvokePlugin()); + copy.appendNodePlugin(new SnippetCounterPlugin()); + HotSpotProviders providers = (HotSpotProviders) replacements.getProviders().copyWith(new HotSpotSubstrateConstantReflectionProvider(replacements.getProviders().getConstantReflection())); + this.replacements = new HotSpotSnippetReplacementsImpl(replacements, providers.copyWith(copy)); + this.replacements.setGraphBuilderPlugins(copy); + } + + @Override + public GraphBuilderConfiguration.Plugins getGraphBuilderPlugins() { + return replacements.getGraphBuilderPlugins(); + } + + /** + * Compiles the snippet and stores the graph. + */ + public void registerMethodSubstitution(ResolvedJavaMethod method, ResolvedJavaMethod original) { + assert method.getAnnotation(MethodSubstitution.class) != null : "MethodSubstitution must be annotated with @" + MethodSubstitution.class.getSimpleName(); + buildGraph(method, original, null, false, false); + snippetMethods.add(method); + } + + static class EncodedSnippets { + private byte[] snippetEncoding; + private Object[] snippetObjects; + private NodeClass[] snippetNodeClasses; + private Map snippetStartOffsets; + private Map originalMethods; + + EncodedSnippets(byte[] snippetEncoding, Object[] snippetObjects, NodeClass[] snippetNodeClasses, Map snippetStartOffsets, Map originalMethods) { + this.snippetEncoding = snippetEncoding; + this.snippetObjects = snippetObjects; + this.snippetNodeClasses = snippetNodeClasses; + this.snippetStartOffsets = snippetStartOffsets; + this.originalMethods = originalMethods; + } + + public StructuredGraph getMethodSubstitutionGraph(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original, ReplacementsImpl replacements) { + Integer startOffset = snippetStartOffsets.get(plugin.toString()); + if (startOffset == null) { + throw GraalError.shouldNotReachHere("plugin graph not found: " + plugin); + } + + return decodeGraph(original, null, startOffset, replacements); + } + + @SuppressWarnings("try") + private StructuredGraph decodeGraph(ResolvedJavaMethod method, Object[] args, int startOffset, ReplacementsImpl replacements) { + OptionValues options = replacements.getOptions(); + SnippetReflectionProvider snippetReflection = replacements.snippetReflection; + ParameterPlugin parameterPlugin = null; + Providers providers = replacements.getProviders(); + if (args != null) { + parameterPlugin = new ConstantBindingParameterPlugin(args, providers.getMetaAccess(), snippetReflection); + } + + EncodedGraph encodedGraph = new SymbolicEncodedGraph(snippetEncoding, startOffset, snippetObjects, snippetNodeClasses, method.getDeclaringClass(), + originalMethods.get(methodKey(method))); + try (DebugContext debug = replacements.openDebugContext("SVMSnippet_", method)) { + StructuredGraph result = new StructuredGraph.Builder(options, debug).method(method).setIsSubstitution(true).build(); + PEGraphDecoder graphDecoder = new PEGraphDecoder( + providers.getCodeCache().getTarget().arch, + result, + providers.getMetaAccess(), + providers.getConstantReflection(), + providers.getConstantFieldProvider(), + providers.getStampProvider(), + null, // loopExplosionPlugin + replacements.getGraphBuilderPlugins().getInvocationPlugins(), + new InlineInvokePlugin[0], + parameterPlugin, + null, // nodePlugins + null, // callInlinedMethod + null // sourceLanguagePositionProvider + ) { + @Override + protected EncodedGraph lookupEncodedGraph(ResolvedJavaMethod lookupMethod, + ResolvedJavaMethod originalMethod, + BytecodeProvider intrinsicBytecodeProvider, + boolean isSubstitution, + boolean trackNodeSourcePosition) { + if (lookupMethod.equals(method)) { + return encodedGraph; + } else { + throw GraalError.shouldNotReachHere(method.format("%H.%n(%p)")); + } + } + }; + + graphDecoder.decode(method, result.isSubstitution(), encodedGraph.trackNodeSourcePosition()); + + assert result.verify(); + return result; + } + } + + StructuredGraph getEncodedSnippet(ResolvedJavaMethod method, ReplacementsImpl replacements, Object[] args) { + Integer startOffset = null; + if (snippetStartOffsets != null) { + startOffset = snippetStartOffsets.get(methodKey(method)); + } + if (startOffset == null) { + if (IS_IN_NATIVE_IMAGE) { + throw GraalError.shouldNotReachHere("snippet not found: " + method.format("%H.%n(%p)")); + } else { + return null; + } + } + + SymbolicEncodedGraph encodedGraph = new SymbolicEncodedGraph(snippetEncoding, startOffset, snippetObjects, snippetNodeClasses, method.getDeclaringClass(), + originalMethods.get(methodKey(method))); + return decodeSnippetGraph(encodedGraph, method, replacements, args, HotSpotJVMCIRuntime.runtime().getHostJVMCIBackend().getTarget().arch); + } + + } + + private StructuredGraph buildGraph(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean requireInlining, boolean trackNodeSourcePosition) { + assert method.hasBytecodes() : "Snippet must not be abstract or native"; + Object[] args = null; + if (receiver != null) { + args = new Object[method.getSignature().getParameterCount(true)]; + args[0] = receiver; + } + try (DebugContext debug = openDebugContext("Snippet_", method)) { + StructuredGraph graph = replacements.makeGraph(debug, replacements.getDefaultReplacementBytecodeProvider(), method, args, original, trackNodeSourcePosition, null); + + // Check if all methods which should be inlined are really inlined. + for (MethodCallTargetNode callTarget : graph.getNodes(MethodCallTargetNode.TYPE)) { + ResolvedJavaMethod callee = callTarget.targetMethod(); + if (requireInlining && !delayedInvocationPluginMethods.contains(callee) && !Objects.equals(callee, original)) { + throw GraalError.shouldNotReachHere("method " + callee.format("%H.%n") + " not inlined in snippet " + method.getName() + " (maybe not final?)"); + } + } + assert verifySnippetEncodeDecode(method, original, trackNodeSourcePosition, graph); + debug.dump(DebugContext.VERBOSE_LEVEL, graph, "After buildGraph"); + return graph; + } + } + + @SuppressWarnings("try") + static StructuredGraph decodeSnippetGraph(SymbolicEncodedGraph encodedGraph, ResolvedJavaMethod method, ReplacementsImpl replacements, Object[] args, Architecture architecture) { + Providers providers = replacements.getProviders(); + ParameterPlugin parameterPlugin = null; + if (args != null) { + parameterPlugin = new ConstantBindingParameterPlugin(args, providers.getMetaAccess(), replacements.snippetReflection); + } + + try (DebugContext debug = replacements.openDebugContext("SVMSnippet_", method)) { + // @formatter:off + StructuredGraph result = new StructuredGraph.Builder(replacements.getOptions(), debug) + .method(method) + .trackNodeSourcePosition(encodedGraph.trackNodeSourcePosition()) + .setIsSubstitution(true) + .build(); + // @formatter:on + try (DebugContext.Scope scope = debug.scope("DecodeSnippetGraph", result)) { + PEGraphDecoder graphDecoder = new PEGraphDecoder( + architecture, + result, + providers.getMetaAccess(), + providers.getConstantReflection(), + providers.getConstantFieldProvider(), + providers.getStampProvider(), + null, + replacements.getGraphBuilderPlugins().getInvocationPlugins(), + new InlineInvokePlugin[0], + parameterPlugin, + null, + null, + null) { + @Override + protected EncodedGraph lookupEncodedGraph( + ResolvedJavaMethod lookupMethod, + ResolvedJavaMethod originalMethod, + BytecodeProvider intrinsicBytecodeProvider, + boolean isSubstitution, + boolean track) { + if (lookupMethod.equals(method)) { + assert !track || encodedGraph.trackNodeSourcePosition(); + return encodedGraph; + } else { + throw GraalError.shouldNotReachHere(method.format("%H.%n(%p)")); + } + } + }; + + graphDecoder.decode(method, result.isSubstitution(), encodedGraph.trackNodeSourcePosition()); + debug.dump(DebugContext.VERBOSE_LEVEL, result, "After decoding"); + + assert result.verify(); + return result; + } catch (Throwable t) { + throw debug.handle(t); + } + } + } + + @SuppressWarnings("try") + private boolean verifySnippetEncodeDecode(ResolvedJavaMethod method, ResolvedJavaMethod original, boolean trackNodeSourcePosition, StructuredGraph structuredGraph) { + // Verify the encoding and decoding process + EncodedGraph encodedGraph = GraphEncoder.encodeSingleGraph(structuredGraph, HotSpotJVMCIRuntime.runtime().getHostJVMCIBackend().getTarget().arch); + + Architecture arch = HotSpotJVMCIRuntime.runtime().getHostJVMCIBackend().getTarget().arch; + + try (DebugContext debug = replacements.openDebugContext("VerifySnippetEncodeDecode_", method)) { + HotSpotProviders originalProvider = (HotSpotProviders) replacements.getProviders(); + + SnippetReflectionProvider snippetReflection = originalProvider.getSnippetReflection(); + SymbolicSnippetEncoder.HotSpotSubstrateConstantReflectionProvider constantReflection = new SymbolicSnippetEncoder.HotSpotSubstrateConstantReflectionProvider( + originalProvider.getConstantReflection()); + HotSpotProviders newProviders = new HotSpotProviders(originalProvider.getMetaAccess(), originalProvider.getCodeCache(), constantReflection, + originalProvider.getConstantFieldProvider(), originalProvider.getForeignCalls(), originalProvider.getLowerer(), null, originalProvider.getSuites(), + originalProvider.getRegisters(), snippetReflection, originalProvider.getWordTypes(), originalProvider.getGraphBuilderPlugins()); + HotSpotSnippetReplacementsImpl filteringReplacements = new HotSpotSnippetReplacementsImpl(getOptions(), newProviders, snippetReflection, + originalProvider.getReplacements().getDefaultReplacementBytecodeProvider(), + originalProvider.getCodeCache().getTarget()); + filteringReplacements.setGraphBuilderPlugins(originalProvider.getReplacements().getGraphBuilderPlugins()); + try (DebugContext.Scope scaope = debug.scope("VerifySnippetEncodeDecode", structuredGraph)) { + for (int i = 0; i < encodedGraph.getNumObjects(); i++) { + filterSnippetObject(encodedGraph.getObject(i)); + } + StructuredGraph snippet = filteringReplacements.makeGraph(debug, filteringReplacements.getDefaultReplacementBytecodeProvider(), method, null, original, + trackNodeSourcePosition, null); + SymbolicEncodedGraph symbolicGraph = new SymbolicEncodedGraph(encodedGraph, method.getDeclaringClass(), original != null ? methodKey(original) : null); + StructuredGraph decodedSnippet = decodeSnippetGraph(symbolicGraph, method, replacements, null, arch); + String snippetString = getCanonicalGraphString(snippet, true, false); + String decodedSnippetString = getCanonicalGraphString(decodedSnippet, true, false); + if (snippetString.equals(decodedSnippetString)) { + debug.log("Snippet decode for %s produces exactly same graph", method); + debug.dump(DebugContext.INFO_LEVEL, decodedSnippet, "Decoded snippet graph for %s", method); + } else { + debug.log("Snippet decode for %s produces different graph", method); + debug.log("%s", compareGraphStrings(snippet, snippetString, decodedSnippet, decodedSnippetString)); + debug.dump(DebugContext.INFO_LEVEL, snippet, "Snippet graph for %s", method); + debug.dump(DebugContext.INFO_LEVEL, structuredGraph, "Encoded snippet graph for %s", method); + debug.dump(DebugContext.INFO_LEVEL, decodedSnippet, "Decoded snippet graph for %s", method); + } + } catch (Throwable t) { + throw debug.handle(t); + } + } + return true; + } + + /** + * If there are new graphs waiting to be encoded, reencode all the graphs and return the result. + */ + @SuppressWarnings("try") + synchronized EncodedSnippets maybeEncodeSnippets() { + Map graphs = this.preparedSnippetGraphs; + if (encodedGraphs != graphs.size()) { + DebugContext debug = openDebugContext("SnippetEncoder", null); + try (DebugContext.Scope scope = debug.scope("SnippetSupportEncode")) { + encodedGraphs = graphs.size(); + for (StructuredGraph graph : graphs.values()) { + for (Node node : graph.getNodes()) { + node.setNodeSourcePosition(null); + } + } + return encodeSnippets(debug); + } + } + return null; + } + + @Override + public void registerSnippet(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean trackNodeSourcePosition) { + if (IS_BUILDING_NATIVE_IMAGE || UseEncodedSnippets.getValue(getOptions())) { + assert method.getAnnotation(Snippet.class) != null : "Snippet must be annotated with @" + Snippet.class.getSimpleName(); + String key = methodKey(method); + if (!preparedSnippetGraphs.containsKey(key)) { + if (original != null) { + originalMethods.put(key, methodKey(original)); + } + StructuredGraph snippet = buildGraph(method, original, receiver, true, trackNodeSourcePosition); + snippetMethods.add(method); + preparedSnippetGraphs.put(key, snippet); + } + } + } + + EncodedSnippets encodeSnippets(DebugContext debug) { + GraphEncoder encoder = new GraphEncoder(HotSpotJVMCIRuntime.runtime().getHostJVMCIBackend().getTarget().arch, debug); + for (StructuredGraph graph : preparedSnippetGraphs.values()) { + encoder.prepare(graph); + } + encoder.finishPrepare(); + + byte[] snippetEncoding; + Object[] snippetObjects; + NodeClass[] snippetNodeClasses; + Map snippetStartOffsets; + + snippetStartOffsets = new HashMap<>(); + for (Map.Entry entry : preparedSnippetGraphs.entrySet()) { + snippetStartOffsets.put(entry.getKey(), encoder.encode(entry.getValue())); + } + snippetEncoding = encoder.getEncoding(); + snippetObjects = encoder.getObjects(); + snippetNodeClasses = encoder.getNodeClasses(); + for (int i = 0; i < snippetObjects.length; i++) { + Object o = filterSnippetObject(snippetObjects[i]); + debug.log("snippetObjects[%d] = %s -> %s", i, o != null ? o.getClass().getSimpleName() : null, o); + snippetObjects[i] = o; + } + debug.log("Encoded %d snippet preparedSnippetGraphs using %d bytes with %d objects", snippetStartOffsets.size(), snippetEncoding.length, snippetObjects.length); + return new EncodedSnippets(snippetEncoding, snippetObjects, snippetNodeClasses, snippetStartOffsets, originalMethods); + } + + /** + * Encode any outstanding graphs and return true if any work was done. + */ + @SuppressWarnings("try") + public boolean encode() { + EncodedSnippets encodedSnippets = maybeEncodeSnippets(); + if (encodedSnippets != null) { + HotSpotReplacementsImpl.setEncodedSnippets(encodedSnippets); + return true; + } + return false; + } + + private DebugContext openDebugContext(String idPrefix, ResolvedJavaMethod method) { + return replacements.openDebugContext(idPrefix, method); + } + + static class SymbolicEncodedGraph extends EncodedGraph { + + private final ResolvedJavaType accessingClass; + private final String originalMethod; + + SymbolicEncodedGraph(byte[] encoding, int startOffset, Object[] objects, NodeClass[] types, ResolvedJavaType accessingClass, String originalMethod) { + super(encoding, startOffset, objects, types, null, null, null, false, false); + this.accessingClass = accessingClass; + this.originalMethod = originalMethod; + } + + SymbolicEncodedGraph(EncodedGraph encodedGraph, ResolvedJavaType declaringClass, String originalMethod) { + this(encodedGraph.getEncoding(), encodedGraph.getStartOffset(), encodedGraph.getObjects(), encodedGraph.getNodeClasses(), declaringClass, originalMethod); + } + + @Override + public Object getObject(int i) { + Object o = objects[i]; + if (o instanceof SymbolicJVMCIReference) { + objects[i] = o = ((SymbolicJVMCIReference) o).resolve(accessingClass); + } else if (o instanceof UnresolvedJavaType) { + objects[i] = o = ((UnresolvedJavaType) o).resolve(accessingClass); + } else if (o instanceof UnresolvedJavaMethod) { + throw new InternalError(o.toString()); + } else if (o instanceof UnresolvedJavaField) { + objects[i] = o = ((UnresolvedJavaField) o).resolve(accessingClass); + } else if (o instanceof GraalCapability) { + objects[i] = o = ((GraalCapability) o).resolve(((GraalJVMCICompiler) getRuntime().getCompiler()).getGraalRuntime()); + } + return o; + } + + @Override + public boolean isCallToOriginal(ResolvedJavaMethod callTarget) { + if (originalMethod != null && originalMethod.equals(methodKey(callTarget))) { + return true; + } + return super.isCallToOriginal(callTarget); + } + } + + /** + * Symbolic reference to an object which can be retrieved from + * {@link GraalRuntime#getCapability(Class)}. + */ + static class GraalCapability { + final Class capabilityClass; + + GraalCapability(Class capabilityClass) { + this.capabilityClass = capabilityClass; + } + + public Object resolve(GraalRuntime runtime) { + Object capability = runtime.getCapability(this.capabilityClass); + if (capability != null) { + assert capability.getClass() == capabilityClass; + return capability; + } + throw new InternalError(this.capabilityClass.getName()); + } + } + + static class SymbolicResolvedJavaMethod implements SymbolicJVMCIReference { + final UnresolvedJavaType type; + final String methodName; + final String signature; + + SymbolicResolvedJavaMethod(HotSpotResolvedJavaMethod method) { + this.type = UnresolvedJavaType.create(method.getDeclaringClass().getName()); + this.methodName = method.getName(); + this.signature = method.getSignature().toMethodDescriptor(); + } + + @Override + public String toString() { + return "SymbolicResolvedJavaMethod{" + + "declaringType='" + type.getName() + '\'' + + ", methodName='" + methodName + '\'' + + ", signature='" + signature + '\'' + + '}'; + } + + @Override + public ResolvedJavaMethod resolve(ResolvedJavaType accessingClass) { + ResolvedJavaType resolvedType = type.resolve(accessingClass); + for (ResolvedJavaMethod method : methodName.equals("") ? resolvedType.getDeclaredConstructors() : resolvedType.getDeclaredMethods()) { + if (method.getName().equals(methodName) && method.getSignature().toMethodDescriptor().equals(signature)) { + return method; + } + } + throw new InternalError("Could not resolve " + this + " in context of " + accessingClass.toJavaName()); + } + } + + static class SymbolicResolvedJavaField implements SymbolicJVMCIReference { + final UnresolvedJavaType declaringType; + final String name; + final UnresolvedJavaType signature; + private final boolean isStatic; + + SymbolicResolvedJavaField(HotSpotResolvedJavaField field) { + this.declaringType = UnresolvedJavaType.create(field.getDeclaringClass().getName()); + this.name = field.getName(); + this.signature = UnresolvedJavaType.create(field.getType().getName()); + this.isStatic = field.isStatic(); + } + + @Override + public ResolvedJavaField resolve(ResolvedJavaType accessingClass) { + ResolvedJavaType resolvedType = declaringType.resolve(accessingClass); + ResolvedJavaType resolvedFieldType = signature.resolve(accessingClass); + ResolvedJavaField[] fields = isStatic ? resolvedType.getStaticFields() : resolvedType.getInstanceFields(true); + for (ResolvedJavaField field : fields) { + if (field.getName().equals(name)) { + if (field.getType().equals(resolvedFieldType)) { + return field; + } + } + } + throw new InternalError("Could not resolve " + this + " in context of " + accessingClass.toJavaName()); + } + + @Override + public String toString() { + return "SymbolicResolvedJavaField{" + + signature.getName() + ' ' + + declaringType.getName() + '.' + + name + + '}'; + } + } + + static class SymbolicStampPair implements SymbolicJVMCIReference { + Object trustedStamp; + Object uncheckdStamp; + + SymbolicStampPair(StampPair stamp) { + this.trustedStamp = maybeMakeSymbolic(stamp.getTrustedStamp()); + this.uncheckdStamp = maybeMakeSymbolic(stamp.getUncheckedStamp()); + } + + @Override + public StampPair resolve(ResolvedJavaType accessingClass) { + return StampPair.create(resolveStamp(accessingClass, trustedStamp), resolveStamp(accessingClass, uncheckdStamp)); + } + } + + private static Object maybeMakeSymbolic(Stamp trustedStamp) { + if (trustedStamp != null) { + SymbolicJVMCIReference symbolicJVMCIReference = trustedStamp.makeSymbolic(); + if (symbolicJVMCIReference != null) { + return symbolicJVMCIReference; + } + } + return trustedStamp; + } + + private static Stamp resolveStamp(ResolvedJavaType accessingClass, Object stamp) { + if (stamp == null) { + return null; + } + if (stamp instanceof Stamp) { + return (Stamp) stamp; + } + return (Stamp) ((SymbolicJVMCIReference) stamp).resolve(accessingClass); + } + + public static class HotSpotSubstrateConstantReflectionProvider implements ConstantReflectionProvider { + + private final ConstantReflectionProvider constantReflection; + + HotSpotSubstrateConstantReflectionProvider(ConstantReflectionProvider constantReflection) { + this.constantReflection = constantReflection; + } + + HashSet safeConstants = new HashSet<>(); + + @Override + public Boolean constantEquals(Constant x, Constant y) { + return constantReflection.constantEquals(x, y); + } + + @Override + public Integer readArrayLength(JavaConstant array) { + return constantReflection.readArrayLength(array); + } + + @Override + public JavaConstant readArrayElement(JavaConstant array, int index) { + return constantReflection.readArrayElement(array, index); + } + + @Override + public JavaConstant readFieldValue(ResolvedJavaField field, JavaConstant receiver) { + JavaConstant javaConstant = constantReflection.readFieldValue(field, receiver); + if (!safeConstants.contains(receiver) && !field.getDeclaringClass().getName().contains("graalvm") && !field.getDeclaringClass().getName().contains("jdk/vm/ci/") && + !field.getName().equals("TYPE")) { + // Only permit constant reflection on compiler classes. This is necessary primarily + // because of the boxing snippets which are compiled as snippets but are really just + // regular JDK java sources that are being compiled like a snippet. These shouldn't + // permit constant folding during graph preparation as that embeds constants from + // the runtime into a compiler graph. + return null; + } + if (javaConstant.getJavaKind() == JavaKind.Object) { + safeConstants.add(javaConstant); + } + return javaConstant; + } + + @Override + public JavaConstant boxPrimitive(JavaConstant source) { + return constantReflection.boxPrimitive(source); + } + + @Override + public JavaConstant unboxPrimitive(JavaConstant source) { + return constantReflection.unboxPrimitive(source); + } + + @Override + public JavaConstant forString(String value) { + return constantReflection.forString(value); + } + + @Override + public ResolvedJavaType asJavaType(Constant constant) { + return constantReflection.asJavaType(constant); + } + + @Override + public MethodHandleAccessProvider getMethodHandleAccess() { + return constantReflection.getMethodHandleAccess(); + } + + @Override + public MemoryAccessProvider getMemoryAccessProvider() { + return constantReflection.getMemoryAccessProvider(); + } + + @Override + public JavaConstant asJavaClass(ResolvedJavaType type) { + return constantReflection.asJavaClass(type); + } + + @Override + public Constant asObjectHub(ResolvedJavaType type) { + return constantReflection.asObjectHub(type); + } + } + + /** + * Objects embedded in encoded graphs might need to converted into a symbolic form so convert + * the object or pass it through. + */ + static Object filterSnippetObject(Object o) { + if (o instanceof HotSpotResolvedJavaMethod) { + return new SymbolicResolvedJavaMethod((HotSpotResolvedJavaMethod) o); + } else if (o instanceof HotSpotResolvedJavaField) { + return new SymbolicResolvedJavaField((HotSpotResolvedJavaField) o); + } else if (o instanceof HotSpotResolvedJavaType) { + return UnresolvedJavaType.create(((HotSpotResolvedJavaType) o).getName()); + } else if (o instanceof NodeSourcePosition) { + // Filter these out for now. These can't easily be handled because these positions + // description snippet methods which might not be available in the runtime. + return null; + } else if (o instanceof HotSpotForeignCallsProvider || o instanceof GraalHotSpotVMConfig) { + return new GraalCapability(o.getClass()); + } else if (o instanceof Stamp) { + SymbolicJVMCIReference ref = ((Stamp) o).makeSymbolic(); + if (ref != null) { + return ref; + } + return o; + } else if (o instanceof StampPair) { + if (((StampPair) o).getTrustedStamp() instanceof AbstractObjectStamp) { + return new SymbolicStampPair((StampPair) o); + } + } + return o; + } + + static String compareGraphStrings(StructuredGraph expectedGraph, String expectedString, StructuredGraph actualGraph, String actualString) { + if (!expectedString.equals(actualString)) { + String[] expectedLines = expectedString.split("\n"); + String[] actualLines = actualString.split("\n"); + int diffIndex = -1; + int limit = Math.min(actualLines.length, expectedLines.length); + String marker = " <<<"; + for (int i = 0; i < limit; i++) { + if (!expectedLines[i].equals(actualLines[i])) { + diffIndex = i; + break; + } + } + if (diffIndex == -1) { + // Prefix is the same so add some space after the prefix + diffIndex = limit; + if (actualLines.length == limit) { + actualLines = Arrays.copyOf(actualLines, limit + 1); + actualLines[diffIndex] = ""; + } else { + assert expectedLines.length == limit; + expectedLines = Arrays.copyOf(expectedLines, limit + 1); + expectedLines[diffIndex] = ""; + } + } + // Place a marker next to the first line that differs + expectedLines[diffIndex] = expectedLines[diffIndex] + marker; + actualLines[diffIndex] = actualLines[diffIndex] + marker; + String ediff = String.join("\n", expectedLines); + String adiff = String.join("\n", actualLines); + return "mismatch in preparedSnippetGraphs:\n========= expected (" + expectedGraph + ") =========\n" + ediff + "\n\n========= actual (" + actualGraph + ") =========\n" + adiff; + } else { + return "mismatch in preparedSnippetGraphs"; + } + } + + static String getCanonicalGraphString(StructuredGraph graph, boolean excludeVirtual, boolean checkConstants) { + SchedulePhase schedule = new SchedulePhase(SchedulePhase.SchedulingStrategy.EARLIEST); + schedule.apply(graph); + StructuredGraph.ScheduleResult scheduleResult = graph.getLastSchedule(); + + NodeMap canonicalId = graph.createNodeMap(); + int nextId = 0; + + List constantsLines = new ArrayList<>(); + + StringBuilder result = new StringBuilder(); + for (Block block : scheduleResult.getCFG().getBlocks()) { + result.append("Block ").append(block).append(' '); + if (block == scheduleResult.getCFG().getStartBlock()) { + result.append("* "); + } + result.append("-> "); + for (Block succ : block.getSuccessors()) { + result.append(succ).append(' '); + } + result.append('\n'); + for (Node node : scheduleResult.getBlockToNodesMap().get(block)) { + if (node instanceof ValueNode && node.isAlive()) { + if (!excludeVirtual || !(node instanceof VirtualObjectNode || node instanceof ProxyNode || node instanceof FullInfopointNode || node instanceof ParameterNode)) { + if (node instanceof ConstantNode) { + if (checkConstants) { + String name = node.toString(Verbosity.Name); + if (excludeVirtual) { + constantsLines.add(name); + } else { + constantsLines.add(name + " (" + filteredUsageCount(node) + ")"); + } + } + } else { + int id; + if (canonicalId.get(node) != null) { + id = canonicalId.get(node); + } else { + id = nextId++; + canonicalId.set(node, id); + } + String name = node.getClass().getSimpleName(); + result.append(" ").append(id).append('|').append(name); + if (node instanceof AccessFieldNode) { + result.append('#'); + result.append(((AccessFieldNode) node).field()); + } + if (!excludeVirtual) { + result.append(" ("); + result.append(filteredUsageCount(node)); + result.append(')'); + } + result.append('\n'); + } + } + } + } + } + + StringBuilder constantsLinesResult = new StringBuilder(); + if (checkConstants) { + constantsLinesResult.append(constantsLines.size()).append(" constants:\n"); + } + Collections.sort(constantsLines); + for (String s : constantsLines) { + constantsLinesResult.append(s); + constantsLinesResult.append('\n'); + } + + return constantsLinesResult.toString() + result.toString(); + } + + private static int filteredUsageCount(Node node) { + return node.usages().filter(n -> !(n instanceof FrameState)).count(); + } + + /** + * This horror show of classes exists solely get {@link HotSpotSnippetBytecodeParser} to be used + * as the parser for these snippets. + */ + static class HotSpotSnippetReplacementsImpl extends HotSpotReplacementsImpl { + HotSpotSnippetReplacementsImpl(HotSpotReplacementsImpl replacements, Providers providers) { + super(replacements, providers); + } + + HotSpotSnippetReplacementsImpl(OptionValues options, Providers providers, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider, TargetDescription target) { + super(options, providers, snippetReflection, bytecodeProvider, target); + } + + @Override + protected GraphMaker createGraphMaker(ResolvedJavaMethod substitute, ResolvedJavaMethod original) { + return new SnippetGraphMaker(this, substitute, original); + } + } + + static class SnippetGraphMaker extends ReplacementsImpl.GraphMaker { + SnippetGraphMaker(ReplacementsImpl replacements, ResolvedJavaMethod substitute, ResolvedJavaMethod substitutedMethod) { + super(replacements, substitute, substitutedMethod); + } + + @Override + protected GraphBuilderPhase.Instance createGraphBuilder(MetaAccessProvider metaAccess, StampProvider stampProvider, ConstantReflectionProvider constantReflection, + ConstantFieldProvider constantFieldProvider, GraphBuilderConfiguration graphBuilderConfig, OptimisticOptimizations optimisticOpts, IntrinsicContext initialIntrinsicContext) { + return new HotSpotSnippetGraphBuilderPhase(metaAccess, stampProvider, constantReflection, constantFieldProvider, graphBuilderConfig, optimisticOpts, + initialIntrinsicContext); + } + } + + static class HotSpotSnippetGraphBuilderPhase extends GraphBuilderPhase.Instance { + HotSpotSnippetGraphBuilderPhase(MetaAccessProvider metaAccess, StampProvider stampProvider, ConstantReflectionProvider constantReflection, ConstantFieldProvider constantFieldProvider, + GraphBuilderConfiguration graphBuilderConfig, OptimisticOptimizations optimisticOpts, IntrinsicContext initialIntrinsicContext) { + super(metaAccess, stampProvider, constantReflection, constantFieldProvider, graphBuilderConfig, optimisticOpts, initialIntrinsicContext); + } + + @Override + protected BytecodeParser createBytecodeParser(StructuredGraph graph, BytecodeParser parent, ResolvedJavaMethod method, int entryBCI, IntrinsicContext intrinsicContext) { + return new HotSpotSnippetBytecodeParser(this, graph, parent, method, entryBCI, intrinsicContext); + } + } + + static class HotSpotSnippetBytecodeParser extends BytecodeParser { + HotSpotSnippetBytecodeParser(GraphBuilderPhase.Instance graphBuilderInstance, StructuredGraph graph, BytecodeParser parent, ResolvedJavaMethod method, int entryBCI, + IntrinsicContext intrinsicContext) { + super(graphBuilderInstance, graph, parent, method, entryBCI, intrinsicContext); + } + + @Override + public boolean canDeferPlugin(GeneratedInvocationPlugin plugin) { + return plugin.getSource().equals(Fold.class) || plugin.getSource().equals(Node.NodeIntrinsic.class); + } + + @Override + protected boolean tryInvocationPlugin(CallTargetNode.InvokeKind invokeKind, ValueNode[] args, ResolvedJavaMethod targetMethod, JavaKind resultType, JavaType returnType) { + if (intrinsicContext != null && intrinsicContext.isCallToOriginal(targetMethod)) { + return false; + } + return super.tryInvocationPlugin(invokeKind, args, targetMethod, resultType, returnType); + } + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1ArrayRangePostWriteBarrier.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1ArrayRangePostWriteBarrier.java similarity index 90% rename from src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1ArrayRangePostWriteBarrier.java rename to src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1ArrayRangePostWriteBarrier.java index 6aebdfb754d..5c1d6d03578 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1ArrayRangePostWriteBarrier.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1ArrayRangePostWriteBarrier.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -22,12 +22,14 @@ */ -package org.graalvm.compiler.hotspot.nodes; + +package org.graalvm.compiler.hotspot.gc.g1; import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64; import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_64; import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.hotspot.gc.shared.ArrayRangeWriteBarrier; import org.graalvm.compiler.nodeinfo.NodeInfo; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.memory.address.AddressNode; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1ArrayRangePreWriteBarrier.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1ArrayRangePreWriteBarrier.java similarity index 90% rename from src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1ArrayRangePreWriteBarrier.java rename to src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1ArrayRangePreWriteBarrier.java index 51b274e929f..ea074bb86a2 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1ArrayRangePreWriteBarrier.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1ArrayRangePreWriteBarrier.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -22,12 +22,14 @@ */ -package org.graalvm.compiler.hotspot.nodes; + +package org.graalvm.compiler.hotspot.gc.g1; import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64; import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_64; import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.hotspot.gc.shared.ArrayRangeWriteBarrier; import org.graalvm.compiler.nodeinfo.NodeInfo; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.memory.address.AddressNode; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1BarrierSet.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1BarrierSet.java new file mode 100644 index 00000000000..4e7729bc445 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1BarrierSet.java @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.gc.g1; + +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.hotspot.gc.shared.BarrierSet; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.extended.ArrayRangeWrite; +import org.graalvm.compiler.nodes.java.AbstractCompareAndSwapNode; +import org.graalvm.compiler.nodes.java.LoweredAtomicReadAndWriteNode; +import org.graalvm.compiler.nodes.memory.FixedAccessNode; +import org.graalvm.compiler.nodes.memory.HeapAccess; +import org.graalvm.compiler.nodes.memory.ReadNode; +import org.graalvm.compiler.nodes.memory.WriteNode; +import org.graalvm.compiler.nodes.memory.address.AddressNode; +import org.graalvm.compiler.nodes.type.StampTool; + +public class G1BarrierSet extends BarrierSet { + + @Override + public void addReadNodeBarriers(ReadNode node, StructuredGraph graph) { + if (node.getBarrierType() != HeapAccess.BarrierType.NONE) { + assert (node.getBarrierType() == HeapAccess.BarrierType.PRECISE); + G1ReferentFieldReadBarrier barrier = graph.add(new G1ReferentFieldReadBarrier(node.getAddress(), node, false)); + graph.addAfterFixed(node, barrier); + } + } + + @Override + public void addWriteNodeBarriers(WriteNode node, StructuredGraph graph) { + HeapAccess.BarrierType barrierType = node.getBarrierType(); + switch (barrierType) { + case NONE: + // nothing to do + break; + case IMPRECISE: + case PRECISE: + boolean precise = barrierType == HeapAccess.BarrierType.PRECISE; + if (!node.getLocationIdentity().isInit()) { + // The pre barrier does nothing if the value being read is null, so it can + // be explicitly skipped when this is an initializing store. + addG1PreWriteBarrier(node, node.getAddress(), null, true, node.getNullCheck(), graph); + } + addG1PostWriteBarrier(node, node.getAddress(), node.value(), precise, graph); + break; + default: + throw new GraalError("unexpected barrier type: " + barrierType); + } + } + + @Override + public void addAtomicReadWriteNodeBarriers(LoweredAtomicReadAndWriteNode node, StructuredGraph graph) { + HeapAccess.BarrierType barrierType = node.getBarrierType(); + switch (barrierType) { + case NONE: + // nothing to do + break; + case IMPRECISE: + case PRECISE: + boolean precise = barrierType == HeapAccess.BarrierType.PRECISE; + addG1PreWriteBarrier(node, node.getAddress(), null, true, node.getNullCheck(), graph); + addG1PostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); + break; + default: + throw new GraalError("unexpected barrier type: " + barrierType); + } + } + + @Override + public void addCASBarriers(AbstractCompareAndSwapNode node, StructuredGraph graph) { + HeapAccess.BarrierType barrierType = node.getBarrierType(); + switch (barrierType) { + case NONE: + // nothing to do + break; + case IMPRECISE: + case PRECISE: + boolean precise = barrierType == HeapAccess.BarrierType.PRECISE; + addG1PreWriteBarrier(node, node.getAddress(), node.getExpectedValue(), false, false, graph); + addG1PostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); + break; + default: + throw new GraalError("unexpected barrier type: " + barrierType); + } + } + + @Override + public void addArrayRangeBarriers(ArrayRangeWrite write, StructuredGraph graph) { + if (!write.isInitialization()) { + // The pre barrier does nothing if the value being read is null, so it can + // be explicitly skipped when this is an initializing store. + G1ArrayRangePreWriteBarrier g1ArrayRangePreWriteBarrier = graph.add(new G1ArrayRangePreWriteBarrier(write.getAddress(), write.getLength(), write.getElementStride())); + graph.addBeforeFixed(write.asNode(), g1ArrayRangePreWriteBarrier); + } + G1ArrayRangePostWriteBarrier g1ArrayRangePostWriteBarrier = graph.add(new G1ArrayRangePostWriteBarrier(write.getAddress(), write.getLength(), write.getElementStride())); + graph.addAfterFixed(write.asNode(), g1ArrayRangePostWriteBarrier); + } + + private static void addG1PreWriteBarrier(FixedAccessNode node, AddressNode address, ValueNode value, boolean doLoad, boolean nullCheck, StructuredGraph graph) { + G1PreWriteBarrier preBarrier = graph.add(new G1PreWriteBarrier(address, value, doLoad, nullCheck)); + preBarrier.setStateBefore(node.stateBefore()); + node.setNullCheck(false); + node.setStateBefore(null); + graph.addBeforeFixed(node, preBarrier); + } + + private static void addG1PostWriteBarrier(FixedAccessNode node, AddressNode address, ValueNode value, boolean precise, StructuredGraph graph) { + final boolean alwaysNull = StampTool.isPointerAlwaysNull(value); + graph.addAfterFixed(node, graph.add(new G1PostWriteBarrier(address, value, precise, alwaysNull))); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1PostWriteBarrier.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1PostWriteBarrier.java similarity index 85% rename from src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1PostWriteBarrier.java rename to src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1PostWriteBarrier.java index dcc3e0f0f3e..c106e51cc16 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1PostWriteBarrier.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1PostWriteBarrier.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -22,12 +22,13 @@ */ -package org.graalvm.compiler.hotspot.nodes; +package org.graalvm.compiler.hotspot.gc.g1; import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64; import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_64; import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.hotspot.gc.shared.ObjectWriteBarrier; import org.graalvm.compiler.nodeinfo.NodeInfo; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.memory.address.AddressNode; @@ -42,7 +43,7 @@ public class G1PostWriteBarrier extends ObjectWriteBarrier { this(TYPE, address, value, precise, alwaysNull); } - protected G1PostWriteBarrier(NodeClass c, AddressNode address, ValueNode value, boolean precise, boolean alwaysNull) { + private G1PostWriteBarrier(NodeClass c, AddressNode address, ValueNode value, boolean precise, boolean alwaysNull) { super(c, address, value, precise); this.alwaysNull = alwaysNull; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1PreWriteBarrier.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1PreWriteBarrier.java similarity index 88% rename from src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1PreWriteBarrier.java rename to src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1PreWriteBarrier.java index 1c2b9f2a56d..36545105d14 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1PreWriteBarrier.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1PreWriteBarrier.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -22,12 +22,14 @@ */ -package org.graalvm.compiler.hotspot.nodes; + +package org.graalvm.compiler.hotspot.gc.g1; import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64; import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_64; import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.hotspot.gc.shared.ObjectWriteBarrier; import org.graalvm.compiler.nodeinfo.InputType; import org.graalvm.compiler.nodeinfo.NodeInfo; import org.graalvm.compiler.nodes.DeoptimizingNode; @@ -40,9 +42,9 @@ public final class G1PreWriteBarrier extends ObjectWriteBarrier implements Deopt public static final NodeClass TYPE = NodeClass.create(G1PreWriteBarrier.class); - @OptionalInput(InputType.State) FrameState stateBefore; - protected final boolean nullCheck; - protected final boolean doLoad; + @OptionalInput(InputType.State) private FrameState stateBefore; + private final boolean nullCheck; + private final boolean doLoad; public G1PreWriteBarrier(AddressNode address, ValueNode expectedObject, boolean doLoad, boolean nullCheck) { super(TYPE, address, expectedObject, true); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1ReferentFieldReadBarrier.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1ReferentFieldReadBarrier.java similarity index 91% rename from src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1ReferentFieldReadBarrier.java rename to src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1ReferentFieldReadBarrier.java index 432f4c2b0c3..25553b7f67a 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/G1ReferentFieldReadBarrier.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/g1/G1ReferentFieldReadBarrier.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -22,12 +22,13 @@ */ -package org.graalvm.compiler.hotspot.nodes; +package org.graalvm.compiler.hotspot.gc.g1; import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64; import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_64; import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.hotspot.gc.shared.ObjectWriteBarrier; import org.graalvm.compiler.nodeinfo.NodeInfo; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.memory.address.AddressNode; @@ -42,7 +43,7 @@ import org.graalvm.compiler.nodes.memory.address.AddressNode; public final class G1ReferentFieldReadBarrier extends ObjectWriteBarrier { public static final NodeClass TYPE = NodeClass.create(G1ReferentFieldReadBarrier.class); - protected final boolean doLoad; + private final boolean doLoad; public G1ReferentFieldReadBarrier(AddressNode address, ValueNode expectedObject, boolean doLoad) { super(TYPE, address, expectedObject, true); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/ArrayRangeWriteBarrier.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/ArrayRangeWriteBarrier.java similarity index 92% rename from src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/ArrayRangeWriteBarrier.java rename to src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/ArrayRangeWriteBarrier.java index 30b97511246..9d74288baa0 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/ArrayRangeWriteBarrier.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/ArrayRangeWriteBarrier.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -22,9 +22,10 @@ */ -package org.graalvm.compiler.hotspot.nodes; +package org.graalvm.compiler.hotspot.gc.shared; import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.hotspot.nodes.WriteBarrier; import org.graalvm.compiler.nodeinfo.InputType; import org.graalvm.compiler.nodeinfo.NodeInfo; import org.graalvm.compiler.nodes.ValueNode; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/BarrierSet.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/BarrierSet.java new file mode 100644 index 00000000000..10c77863c76 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/BarrierSet.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.gc.shared; + +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.extended.ArrayRangeWrite; +import org.graalvm.compiler.nodes.java.AbstractCompareAndSwapNode; +import org.graalvm.compiler.nodes.java.LoweredAtomicReadAndWriteNode; +import org.graalvm.compiler.nodes.memory.ReadNode; +import org.graalvm.compiler.nodes.memory.WriteNode; + +public abstract class BarrierSet { + public abstract void addReadNodeBarriers(ReadNode node, StructuredGraph graph); + + public abstract void addWriteNodeBarriers(WriteNode node, StructuredGraph graph); + + public abstract void addAtomicReadWriteNodeBarriers(LoweredAtomicReadAndWriteNode node, StructuredGraph graph); + + public abstract void addCASBarriers(AbstractCompareAndSwapNode node, StructuredGraph graph); + + public abstract void addArrayRangeBarriers(ArrayRangeWrite write, StructuredGraph graph); +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/CardTableBarrierSet.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/CardTableBarrierSet.java new file mode 100644 index 00000000000..401cab8994f --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/CardTableBarrierSet.java @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.gc.shared; + +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.extended.ArrayRangeWrite; +import org.graalvm.compiler.nodes.java.AbstractCompareAndSwapNode; +import org.graalvm.compiler.nodes.java.LoweredAtomicReadAndWriteNode; +import org.graalvm.compiler.nodes.memory.FixedAccessNode; +import org.graalvm.compiler.nodes.memory.HeapAccess; +import org.graalvm.compiler.nodes.memory.ReadNode; +import org.graalvm.compiler.nodes.memory.WriteNode; +import org.graalvm.compiler.nodes.memory.address.AddressNode; +import org.graalvm.compiler.nodes.type.StampTool; + +public class CardTableBarrierSet extends BarrierSet { + + @Override + public void addReadNodeBarriers(ReadNode node, StructuredGraph graph) { + assert node.getBarrierType() == HeapAccess.BarrierType.NONE : "Non precise read barrier has been attached to read node."; + } + + @Override + public void addWriteNodeBarriers(WriteNode node, StructuredGraph graph) { + HeapAccess.BarrierType barrierType = node.getBarrierType(); + switch (barrierType) { + case NONE: + // nothing to do + break; + case IMPRECISE: + case PRECISE: + boolean precise = barrierType == HeapAccess.BarrierType.PRECISE; + addSerialPostWriteBarrier(node, node.getAddress(), node.value(), precise, graph); + break; + default: + throw new GraalError("unexpected barrier type: " + barrierType); + } + } + + @Override + public void addAtomicReadWriteNodeBarriers(LoweredAtomicReadAndWriteNode node, StructuredGraph graph) { + HeapAccess.BarrierType barrierType = node.getBarrierType(); + switch (barrierType) { + case NONE: + // nothing to do + break; + case IMPRECISE: + case PRECISE: + boolean precise = barrierType == HeapAccess.BarrierType.PRECISE; + addSerialPostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); + break; + default: + throw new GraalError("unexpected barrier type: " + barrierType); + } + } + + @Override + public void addCASBarriers(AbstractCompareAndSwapNode node, StructuredGraph graph) { + HeapAccess.BarrierType barrierType = node.getBarrierType(); + switch (barrierType) { + case NONE: + // nothing to do + break; + case IMPRECISE: + case PRECISE: + boolean precise = barrierType == HeapAccess.BarrierType.PRECISE; + addSerialPostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); + break; + default: + throw new GraalError("unexpected barrier type: " + barrierType); + } + } + + @Override + public void addArrayRangeBarriers(ArrayRangeWrite write, StructuredGraph graph) { + SerialArrayRangeWriteBarrier serialArrayRangeWriteBarrier = graph.add(new SerialArrayRangeWriteBarrier(write.getAddress(), write.getLength(), write.getElementStride())); + graph.addAfterFixed(write.asNode(), serialArrayRangeWriteBarrier); + } + + protected void addSerialPostWriteBarrier(FixedAccessNode node, AddressNode address, ValueNode value, boolean precise, StructuredGraph graph) { + final boolean alwaysNull = StampTool.isPointerAlwaysNull(value); + if (alwaysNull) { + // Serial barrier isn't needed for null value + return; + } + graph.addAfterFixed(node, graph.add(new SerialWriteBarrier(address, precise))); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/ObjectWriteBarrier.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/ObjectWriteBarrier.java similarity index 95% rename from src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/ObjectWriteBarrier.java rename to src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/ObjectWriteBarrier.java index 17d387d69f6..4931a3c280b 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/ObjectWriteBarrier.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/ObjectWriteBarrier.java @@ -22,9 +22,10 @@ */ -package org.graalvm.compiler.hotspot.nodes; +package org.graalvm.compiler.hotspot.gc.shared; import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.hotspot.nodes.WriteBarrier; import org.graalvm.compiler.nodeinfo.InputType; import org.graalvm.compiler.nodeinfo.NodeInfo; import org.graalvm.compiler.nodes.ValueNode; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/SerialArrayRangeWriteBarrier.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/SerialArrayRangeWriteBarrier.java similarity index 97% rename from src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/SerialArrayRangeWriteBarrier.java rename to src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/SerialArrayRangeWriteBarrier.java index 455acfb446a..0161014b3f1 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/SerialArrayRangeWriteBarrier.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/SerialArrayRangeWriteBarrier.java @@ -22,7 +22,7 @@ */ -package org.graalvm.compiler.hotspot.nodes; +package org.graalvm.compiler.hotspot.gc.shared; import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_8; import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_8; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/SerialWriteBarrier.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/SerialWriteBarrier.java similarity index 97% rename from src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/SerialWriteBarrier.java rename to src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/SerialWriteBarrier.java index e19d11f4641..ed9d480f93d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/SerialWriteBarrier.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/gc/shared/SerialWriteBarrier.java @@ -22,7 +22,7 @@ */ -package org.graalvm.compiler.hotspot.nodes; +package org.graalvm.compiler.hotspot.gc.shared; import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_8; import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_4; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java index 021129a0c22..1955d772ebf 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java @@ -24,6 +24,7 @@ package org.graalvm.compiler.hotspot.meta; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; import static org.graalvm.compiler.core.common.GraalOptions.AlwaysInlineVTableStubs; import static org.graalvm.compiler.core.common.GraalOptions.InlineVTableStubs; import static org.graalvm.compiler.core.common.GraalOptions.OmitHotExceptionStacktrace; @@ -58,17 +59,17 @@ import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeInputList; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; import org.graalvm.compiler.hotspot.HotSpotGraalRuntimeProvider; +import org.graalvm.compiler.hotspot.gc.g1.G1ArrayRangePostWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1ArrayRangePreWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1PostWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1PreWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1ReferentFieldReadBarrier; +import org.graalvm.compiler.hotspot.gc.shared.SerialArrayRangeWriteBarrier; +import org.graalvm.compiler.hotspot.gc.shared.SerialWriteBarrier; import org.graalvm.compiler.hotspot.nodes.BeginLockScopeNode; -import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePreWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PreWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1ReferentFieldReadBarrier; import org.graalvm.compiler.hotspot.nodes.HotSpotCompressionNode; import org.graalvm.compiler.hotspot.nodes.HotSpotDirectCallTargetNode; import org.graalvm.compiler.hotspot.nodes.HotSpotIndirectCallTargetNode; -import org.graalvm.compiler.hotspot.nodes.SerialArrayRangeWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier; import org.graalvm.compiler.hotspot.nodes.aot.InitializeKlassNode; import org.graalvm.compiler.hotspot.nodes.aot.ResolveConstantNode; import org.graalvm.compiler.hotspot.nodes.aot.ResolveDynamicConstantNode; @@ -162,6 +163,7 @@ import org.graalvm.compiler.replacements.arraycopy.ArrayCopyNode; import org.graalvm.compiler.replacements.arraycopy.ArrayCopySnippets; import org.graalvm.compiler.replacements.arraycopy.ArrayCopyWithSlowPathNode; import org.graalvm.compiler.replacements.nodes.AssertionNode; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import jdk.internal.vm.compiler.word.LocationIdentity; import jdk.vm.ci.code.TargetDescription; @@ -225,7 +227,9 @@ public class DefaultHotSpotLoweringProvider extends DefaultJavaLoweringProvider stringToBytesSnippets = new StringToBytesSnippets.Templates(options, factories, providers, target); hashCodeSnippets = new HashCodeSnippets.Templates(options, factories, providers, target); resolveConstantSnippets = new ResolveConstantSnippets.Templates(options, factories, providers, target); - profileSnippets = new ProfileSnippets.Templates(options, factories, providers, target); + if (!JavaVersionUtil.Java8OrEarlier) { + profileSnippets = new ProfileSnippets.Templates(options, factories, providers, target); + } objectCloneSnippets = new ObjectCloneSnippets.Templates(options, factories, providers, target); foreignCallSnippets = new ForeignCallSnippets.Templates(options, factories, providers, target); } @@ -683,6 +687,9 @@ public class DefaultHotSpotLoweringProvider extends DefaultJavaLoweringProvider } private void throwCachedException(BytecodeExceptionNode node) { + if (IS_IN_NATIVE_IMAGE) { + throw new InternalError("Can't throw exception from SVM object"); + } Throwable exception = Exceptions.cachedExceptions.get(node.getExceptionKind()); assert exception != null; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotClassInitializationPlugin.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotClassInitializationPlugin.java index 09d56891122..a5492d7d47a 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotClassInitializationPlugin.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotClassInitializationPlugin.java @@ -24,9 +24,7 @@ package org.graalvm.compiler.hotspot.meta; -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; +import java.lang.reflect.Method; import java.util.function.Supplier; import org.graalvm.compiler.core.common.type.ObjectStamp; @@ -88,15 +86,16 @@ public final class HotSpotClassInitializationPlugin implements ClassInitializati } private static final Class hscp; - private static final MethodHandle loadReferencedTypeIIZMH; + private static final Method loadReferencedTypeIIZMH; static { - MethodHandle m = null; + Method m = null; Class c = null; try { c = Class.forName("jdk.vm.ci.hotspot.HotSpotConstantPool").asSubclass(ConstantPool.class); - m = MethodHandles.lookup().findVirtual(c, "loadReferencedType", MethodType.methodType(void.class, int.class, int.class, boolean.class)); + m = c.getDeclaredMethod("loadReferencedType", int.class, int.class, boolean.class); } catch (Exception e) { + throw GraalError.shouldNotReachHere(e); } loadReferencedTypeIIZMH = m; hscp = c; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGraalConstantFieldProvider.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGraalConstantFieldProvider.java index b22c17774b9..e464a615527 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGraalConstantFieldProvider.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGraalConstantFieldProvider.java @@ -24,6 +24,8 @@ package org.graalvm.compiler.hotspot.meta; +import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; import static org.graalvm.compiler.core.common.GraalOptions.ImmutableCode; import java.util.ArrayList; @@ -63,7 +65,7 @@ public class HotSpotGraalConstantFieldProvider extends HotSpotConstantFieldProvi private volatile List nonEmbeddableFields; protected boolean isEmbeddableField(ResolvedJavaField field) { - if (nonEmbeddableFields == null) { + if (!IS_IN_NATIVE_IMAGE && (IS_BUILDING_NATIVE_IMAGE || nonEmbeddableFields == null)) { synchronized (this) { if (nonEmbeddableFields == null) { List fields = new ArrayList<>(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java index 8ed62d29802..fe0fab5faf6 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java @@ -25,10 +25,11 @@ package org.graalvm.compiler.hotspot.meta; import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC; +import static org.graalvm.compiler.hotspot.HotSpotBackend.GHASH_PROCESS_BLOCKS; import static org.graalvm.compiler.hotspot.meta.HotSpotAOTProfilingPlugin.Options.TieredAOT; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.JAVA_THREAD_THREAD_OBJECT_LOCATION; import static org.graalvm.compiler.java.BytecodeParserOptions.InlineDuringParsing; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import java.lang.invoke.ConstantCallSite; import java.lang.invoke.MutableCallSite; @@ -46,12 +47,15 @@ import org.graalvm.compiler.core.common.type.TypeReference; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; import org.graalvm.compiler.hotspot.nodes.CurrentJavaThreadNode; import org.graalvm.compiler.hotspot.replacements.AESCryptSubstitutions; +import org.graalvm.compiler.hotspot.replacements.ArraysSupportSubstitutions; import org.graalvm.compiler.hotspot.replacements.BigIntegerSubstitutions; import org.graalvm.compiler.hotspot.replacements.CRC32CSubstitutions; import org.graalvm.compiler.hotspot.replacements.CRC32Substitutions; import org.graalvm.compiler.hotspot.replacements.CallSiteTargetNode; import org.graalvm.compiler.hotspot.replacements.CipherBlockChainingSubstitutions; import org.graalvm.compiler.hotspot.replacements.ClassGetHubNode; +import org.graalvm.compiler.hotspot.replacements.CounterModeSubstitutions; +import org.graalvm.compiler.hotspot.replacements.DigestBaseSubstitutions; import org.graalvm.compiler.hotspot.replacements.HotSpotArraySubstitutions; import org.graalvm.compiler.hotspot.replacements.HotSpotClassSubstitutions; import org.graalvm.compiler.hotspot.replacements.IdentityHashCodeNode; @@ -62,8 +66,10 @@ import org.graalvm.compiler.hotspot.replacements.ReflectionSubstitutions; import org.graalvm.compiler.hotspot.replacements.SHA2Substitutions; import org.graalvm.compiler.hotspot.replacements.SHA5Substitutions; import org.graalvm.compiler.hotspot.replacements.SHASubstitutions; +import org.graalvm.compiler.hotspot.replacements.StringUTF16Substitutions; import org.graalvm.compiler.hotspot.replacements.ThreadSubstitutions; import org.graalvm.compiler.hotspot.word.HotSpotWordTypes; +import org.graalvm.compiler.nodes.ComputeObjectAddressNode; import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.NamedLocationIdentity; import org.graalvm.compiler.nodes.NodeView; @@ -71,6 +77,7 @@ import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.calc.AddNode; import org.graalvm.compiler.nodes.calc.IntegerConvertNode; import org.graalvm.compiler.nodes.calc.LeftShiftNode; +import org.graalvm.compiler.nodes.extended.ForeignCallNode; import org.graalvm.compiler.nodes.graphbuilderconf.ForeignCallPlugin; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.Plugins; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; @@ -163,9 +170,13 @@ public class HotSpotGraphBuilderPlugins { registerCRC32CPlugins(invocationPlugins, config, replacementBytecodeProvider); registerBigIntegerPlugins(invocationPlugins, config, replacementBytecodeProvider); registerSHAPlugins(invocationPlugins, config, replacementBytecodeProvider); + registerGHASHPlugins(invocationPlugins, config, metaAccess, foreignCalls); + registerCounterModePlugins(invocationPlugins, config, replacementBytecodeProvider); registerUnsafePlugins(invocationPlugins, replacementBytecodeProvider); StandardGraphBuilderPlugins.registerInvocationPlugins(metaAccess, snippetReflection, invocationPlugins, replacementBytecodeProvider, true, false); registerArrayPlugins(invocationPlugins, replacementBytecodeProvider); + registerStringPlugins(invocationPlugins, replacementBytecodeProvider); + registerArraysSupportPlugins(invocationPlugins, config, replacementBytecodeProvider); for (NodeIntrinsicPluginFactory factory : GraalServices.load(NodeIntrinsicPluginFactory.class)) { factory.registerPlugins(invocationPlugins, nodeIntrinsificationProvider); @@ -388,6 +399,14 @@ public class HotSpotGraphBuilderPlugins { r.registerMethodSubstitution(HotSpotArraySubstitutions.class, "newInstance", Class.class, int.class); } + private static void registerStringPlugins(InvocationPlugins plugins, BytecodeProvider bytecodeProvider) { + if (!Java8OrEarlier) { + final Registration utf16r = new Registration(plugins, "java.lang.StringUTF16", bytecodeProvider); + utf16r.registerMethodSubstitution(StringUTF16Substitutions.class, "toBytes", char[].class, int.class, int.class); + utf16r.registerMethodSubstitution(StringUTF16Substitutions.class, "getChars", byte[].class, int.class, int.class, char[].class, int.class); + } + } + private static void registerThreadPlugins(InvocationPlugins plugins, MetaAccessProvider metaAccess, WordTypes wordTypes, GraalHotSpotVMConfig config, BytecodeProvider bytecodeProvider) { Registration r = new Registration(plugins, Thread.class, bytecodeProvider); r.register0("currentThread", new InvocationPlugin() { @@ -477,23 +496,73 @@ public class HotSpotGraphBuilderPlugins { } private static void registerSHAPlugins(InvocationPlugins plugins, GraalHotSpotVMConfig config, BytecodeProvider bytecodeProvider) { - if (config.useSHA1Intrinsics()) { + boolean useSha1 = config.useSHA1Intrinsics(); + boolean useSha256 = config.useSHA256Intrinsics(); + boolean useSha512 = config.useSHA512Intrinsics(); + + if (!Java8OrEarlier && (useSha1 || useSha256 || useSha512)) { + Registration r = new Registration(plugins, "sun.security.provider.DigestBase", bytecodeProvider); + r.registerMethodSubstitution(DigestBaseSubstitutions.class, "implCompressMultiBlock0", Receiver.class, byte[].class, int.class, int.class); + } + + if (useSha1) { assert config.sha1ImplCompress != 0L; Registration r = new Registration(plugins, "sun.security.provider.SHA", bytecodeProvider); r.registerMethodSubstitution(SHASubstitutions.class, SHASubstitutions.implCompressName, "implCompress0", Receiver.class, byte[].class, int.class); } - if (config.useSHA256Intrinsics()) { + if (useSha256) { assert config.sha256ImplCompress != 0L; Registration r = new Registration(plugins, "sun.security.provider.SHA2", bytecodeProvider); r.registerMethodSubstitution(SHA2Substitutions.class, SHA2Substitutions.implCompressName, "implCompress0", Receiver.class, byte[].class, int.class); } - if (config.useSHA512Intrinsics()) { + if (useSha512) { assert config.sha512ImplCompress != 0L; Registration r = new Registration(plugins, "sun.security.provider.SHA5", bytecodeProvider); r.registerMethodSubstitution(SHA5Substitutions.class, SHA5Substitutions.implCompressName, "implCompress0", Receiver.class, byte[].class, int.class); } } + private static void registerGHASHPlugins(InvocationPlugins plugins, GraalHotSpotVMConfig config, MetaAccessProvider metaAccess, ForeignCallsProvider foreignCalls) { + if (config.useGHASHIntrinsics()) { + assert config.ghashProcessBlocks != 0L; + Registration r = new Registration(plugins, "com.sun.crypto.provider.GHASH"); + r.register5("processBlocks", + byte[].class, + int.class, + int.class, + long[].class, + long[].class, + new InvocationPlugin() { + @Override + public boolean apply(GraphBuilderContext b, + ResolvedJavaMethod targetMethod, + Receiver receiver, + ValueNode data, + ValueNode inOffset, + ValueNode blocks, + ValueNode state, + ValueNode hashSubkey) { + int longArrayBaseOffset = metaAccess.getArrayBaseOffset(JavaKind.Long); + int byteArrayBaseOffset = metaAccess.getArrayBaseOffset(JavaKind.Byte); + ValueNode dataOffset = AddNode.create(ConstantNode.forInt(byteArrayBaseOffset), inOffset, NodeView.DEFAULT); + ComputeObjectAddressNode dataAddress = b.add(new ComputeObjectAddressNode(data, dataOffset)); + ComputeObjectAddressNode stateAddress = b.add(new ComputeObjectAddressNode(state, ConstantNode.forInt(longArrayBaseOffset))); + ComputeObjectAddressNode hashSubkeyAddress = b.add(new ComputeObjectAddressNode(hashSubkey, ConstantNode.forInt(longArrayBaseOffset))); + b.add(new ForeignCallNode(foreignCalls, GHASH_PROCESS_BLOCKS, stateAddress, hashSubkeyAddress, dataAddress, blocks)); + return true; + } + }); + } + } + + private static void registerCounterModePlugins(InvocationPlugins plugins, GraalHotSpotVMConfig config, BytecodeProvider bytecodeProvider) { + if (config.useAESCTRIntrinsics) { + assert config.counterModeAESCrypt != 0L; + Registration r = new Registration(plugins, "com.sun.crypto.provider.CounterMode", bytecodeProvider); + r.registerMethodSubstitution(CounterModeSubstitutions.class, "implCrypt", Receiver.class, byte[].class, int.class, int.class, byte[].class, int.class); + } + } + private static void registerCRC32Plugins(InvocationPlugins plugins, GraalHotSpotVMConfig config, BytecodeProvider bytecodeProvider) { if (config.useCRC32Intrinsics) { Registration r = new Registration(plugins, CRC32.class, bytecodeProvider); @@ -515,4 +584,11 @@ public class HotSpotGraphBuilderPlugins { r.registerMethodSubstitution(CRC32CSubstitutions.class, "updateDirectByteBuffer", int.class, long.class, int.class, int.class); } } + + private static void registerArraysSupportPlugins(InvocationPlugins plugins, GraalHotSpotVMConfig config, BytecodeProvider bytecodeProvider) { + if (config.useVectorizedMismatchIntrinsic) { + Registration r = new Registration(plugins, "jdk.internal.util.ArraysSupport", bytecodeProvider); + r.registerMethodSubstitution(ArraysSupportSubstitutions.class, "vectorizedMismatch", Object.class, long.class, Object.class, long.class, int.class, int.class); + } + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java index 7ec852fe168..25a9db510b8 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,6 +29,7 @@ import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC; import static org.graalvm.compiler.core.target.Backend.ARITHMETIC_DREM; import static org.graalvm.compiler.core.target.Backend.ARITHMETIC_FREM; import static org.graalvm.compiler.hotspot.HotSpotBackend.BACKEDGE_EVENT; +import static org.graalvm.compiler.hotspot.HotSpotBackend.COUNTERMODE_IMPL_CRYPT; import static org.graalvm.compiler.hotspot.HotSpotBackend.DECRYPT; import static org.graalvm.compiler.hotspot.HotSpotBackend.DECRYPT_BLOCK; import static org.graalvm.compiler.hotspot.HotSpotBackend.DECRYPT_BLOCK_WITH_ORIGINAL_KEY; @@ -36,6 +37,7 @@ import static org.graalvm.compiler.hotspot.HotSpotBackend.DECRYPT_WITH_ORIGINAL_ import static org.graalvm.compiler.hotspot.HotSpotBackend.ENCRYPT; import static org.graalvm.compiler.hotspot.HotSpotBackend.ENCRYPT_BLOCK; import static org.graalvm.compiler.hotspot.HotSpotBackend.EXCEPTION_HANDLER; +import static org.graalvm.compiler.hotspot.HotSpotBackend.GHASH_PROCESS_BLOCKS; import static org.graalvm.compiler.hotspot.HotSpotBackend.IC_MISS_HANDLER; import static org.graalvm.compiler.hotspot.HotSpotBackend.INITIALIZE_KLASS_BY_SYMBOL; import static org.graalvm.compiler.hotspot.HotSpotBackend.INVOCATION_EVENT; @@ -54,10 +56,14 @@ import static org.graalvm.compiler.hotspot.HotSpotBackend.RESOLVE_KLASS_BY_SYMBO import static org.graalvm.compiler.hotspot.HotSpotBackend.RESOLVE_METHOD_BY_SYMBOL_AND_LOAD_COUNTERS; import static org.graalvm.compiler.hotspot.HotSpotBackend.RESOLVE_STRING_BY_SYMBOL; import static org.graalvm.compiler.hotspot.HotSpotBackend.SHA2_IMPL_COMPRESS; +import static org.graalvm.compiler.hotspot.HotSpotBackend.SHA2_IMPL_COMPRESS_MB; import static org.graalvm.compiler.hotspot.HotSpotBackend.SHA5_IMPL_COMPRESS; +import static org.graalvm.compiler.hotspot.HotSpotBackend.SHA5_IMPL_COMPRESS_MB; import static org.graalvm.compiler.hotspot.HotSpotBackend.SHA_IMPL_COMPRESS; +import static org.graalvm.compiler.hotspot.HotSpotBackend.SHA_IMPL_COMPRESS_MB; import static org.graalvm.compiler.hotspot.HotSpotBackend.SQUARE_TO_LEN; import static org.graalvm.compiler.hotspot.HotSpotBackend.UNWIND_EXCEPTION_TO_CALLER; +import static org.graalvm.compiler.hotspot.HotSpotBackend.VECTORIZED_MISMATCHED; import static org.graalvm.compiler.hotspot.HotSpotBackend.VM_ERROR; import static org.graalvm.compiler.hotspot.HotSpotBackend.WRONG_METHOD_HANDLER; import static org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage.Reexecutability.NOT_REEXECUTABLE; @@ -169,23 +175,23 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall return uninitObjectArraycopyDescriptors[aligned ? 1 : 0][disjoint ? 1 : 0]; } if (killAny) { - assert kind == JavaKind.Object; - return objectArraycopyDescriptorsKillAny[aligned ? 1 : 0][disjoint ? 1 : 0]; + return arraycopyDescriptorsKillAny[aligned ? 1 : 0][disjoint ? 1 : 0].get(kind); } return arraycopyDescriptors[aligned ? 1 : 0][disjoint ? 1 : 0].get(kind); } - @SuppressWarnings({"unchecked"}) private static final EnumMap[][] arraycopyDescriptors = (EnumMap[][]) new EnumMap[2][2]; + @SuppressWarnings("unchecked") private static final EnumMap[][] arraycopyDescriptors = (EnumMap[][]) new EnumMap[2][2]; + @SuppressWarnings("unchecked") private static final EnumMap[][] arraycopyDescriptorsKillAny = (EnumMap[][]) new EnumMap[2][2]; private static final ForeignCallDescriptor[][] uninitObjectArraycopyDescriptors = new ForeignCallDescriptor[2][2]; private static final ForeignCallDescriptor[] checkcastArraycopyDescriptors = new ForeignCallDescriptor[2]; - private static ForeignCallDescriptor[][] objectArraycopyDescriptorsKillAny = new ForeignCallDescriptor[2][2]; static { // Populate the EnumMap instances for (int i = 0; i < arraycopyDescriptors.length; i++) { for (int j = 0; j < arraycopyDescriptors[i].length; j++) { arraycopyDescriptors[i][j] = new EnumMap<>(JavaKind.class); + arraycopyDescriptorsKillAny[i][j] = new EnumMap<>(JavaKind.class); } } } @@ -199,13 +205,15 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall if (uninit) { assert kind == JavaKind.Object; uninitObjectArraycopyDescriptors[aligned ? 1 : 0][disjoint ? 1 : 0] = desc; + } else if (killAny) { + arraycopyDescriptorsKillAny[aligned ? 1 : 0][disjoint ? 1 : 0].put(kind, desc); } else { arraycopyDescriptors[aligned ? 1 : 0][disjoint ? 1 : 0].put(kind, desc); } } private ForeignCallDescriptor buildDescriptor(JavaKind kind, boolean aligned, boolean disjoint, boolean uninit, boolean killAny, long routine) { - assert !killAny || kind == JavaKind.Object; + assert !uninit || kind == JavaKind.Object; String name = kind + (aligned ? "Aligned" : "") + (disjoint ? "Disjoint" : "") + (uninit ? "Uninit" : "") + "Arraycopy" + (killAny ? "KillAny" : ""); ForeignCallDescriptor desc = new ForeignCallDescriptor(name, void.class, Word.class, Word.class, Word.class); LocationIdentity killed = killAny ? LocationIdentity.any() : NamedLocationIdentity.getArrayLocation(kind); @@ -253,11 +261,12 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall registerArraycopyDescriptor(descMap, kind, false, true, uninit, false, disjointRoutine); registerArraycopyDescriptor(descMap, kind, true, true, uninit, false, alignedDisjointRoutine); - if (kind == JavaKind.Object && !uninit) { - objectArraycopyDescriptorsKillAny[0][0] = buildDescriptor(kind, false, false, uninit, true, routine); - objectArraycopyDescriptorsKillAny[1][0] = buildDescriptor(kind, true, false, uninit, true, alignedRoutine); - objectArraycopyDescriptorsKillAny[0][1] = buildDescriptor(kind, false, true, uninit, true, disjointRoutine); - objectArraycopyDescriptorsKillAny[1][1] = buildDescriptor(kind, true, true, uninit, true, alignedDisjointRoutine); + if (!uninit) { + EconomicMap killAnyDescMap = EconomicMap.create(); + registerArraycopyDescriptor(killAnyDescMap, kind, false, false, uninit, true, routine); + registerArraycopyDescriptor(killAnyDescMap, kind, true, false, uninit, true, alignedRoutine); + registerArraycopyDescriptor(killAnyDescMap, kind, false, true, uninit, true, disjointRoutine); + registerArraycopyDescriptor(killAnyDescMap, kind, true, true, uninit, true, alignedDisjointRoutine); } } @@ -275,13 +284,9 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall registerForeignCall(JAVA_TIME_MILLIS, c.javaTimeMillisAddress, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE, NO_LOCATIONS); registerForeignCall(JAVA_TIME_NANOS, c.javaTimeNanosAddress, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE, NO_LOCATIONS); - registerForeignCall(SIN.foreignCallDescriptor, c.arithmeticSinAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); - registerForeignCall(COS.foreignCallDescriptor, c.arithmeticCosAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); - registerForeignCall(TAN.foreignCallDescriptor, c.arithmeticTanAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); - registerForeignCall(EXP.foreignCallDescriptor, c.arithmeticExpAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); - registerForeignCall(LOG.foreignCallDescriptor, c.arithmeticLogAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); - registerForeignCall(LOG10.foreignCallDescriptor, c.arithmeticLog10Address, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); - registerForeignCall(POW.foreignCallDescriptor, c.arithmeticPowAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); + + registerMathStubs(c, providers, options); + registerForeignCall(ARITHMETIC_FREM, c.fremAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); registerForeignCall(ARITHMETIC_DREM, c.dremAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); @@ -377,14 +382,21 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall registerForeignCall(MULTIPLY_TO_LEN, c.multiplyToLen, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Int)); } + if (c.useSHA1Intrinsics()) { - registerForeignCall(SHA_IMPL_COMPRESS, c.sha1ImplCompress, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); + registerForeignCall(SHA_IMPL_COMPRESS, c.sha1ImplCompress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); + registerForeignCall(SHA_IMPL_COMPRESS_MB, c.sha1ImplCompressMultiBlock, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); } if (c.useSHA256Intrinsics()) { - registerForeignCall(SHA2_IMPL_COMPRESS, c.sha256ImplCompress, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); + registerForeignCall(SHA2_IMPL_COMPRESS, c.sha256ImplCompress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); + registerForeignCall(SHA2_IMPL_COMPRESS_MB, c.sha256ImplCompressMultiBlock, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); } if (c.useSHA512Intrinsics()) { - registerForeignCall(SHA5_IMPL_COMPRESS, c.sha512ImplCompress, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); + registerForeignCall(SHA5_IMPL_COMPRESS, c.sha512ImplCompress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); + registerForeignCall(SHA5_IMPL_COMPRESS_MB, c.sha512ImplCompressMultiBlock, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); + } + if (c.useGHASHIntrinsics()) { + registerForeignCall(GHASH_PROCESS_BLOCKS, c.ghashProcessBlocks, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.any()); } if (c.useMulAddIntrinsic()) { registerForeignCall(MUL_ADD, c.mulAdd, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Int)); @@ -409,11 +421,11 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall */ try { // These stubs do callee saving - registerForeignCall(ENCRYPT_BLOCK, c.aescryptEncryptBlockStub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, + registerForeignCall(ENCRYPT_BLOCK, c.aescryptEncryptBlockStub, NativeCall, PRESERVES_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Byte)); - registerForeignCall(DECRYPT_BLOCK, c.aescryptDecryptBlockStub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, + registerForeignCall(DECRYPT_BLOCK, c.aescryptDecryptBlockStub, NativeCall, PRESERVES_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Byte)); - registerForeignCall(DECRYPT_BLOCK_WITH_ORIGINAL_KEY, c.aescryptDecryptBlockStub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, + registerForeignCall(DECRYPT_BLOCK_WITH_ORIGINAL_KEY, c.aescryptDecryptBlockStub, NativeCall, PRESERVES_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Byte)); } catch (GraalError e) { if (!(e.getCause() instanceof ClassNotFoundException)) { @@ -422,11 +434,11 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall } try { // These stubs do callee saving - registerForeignCall(ENCRYPT, c.cipherBlockChainingEncryptAESCryptStub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, + registerForeignCall(ENCRYPT, c.cipherBlockChainingEncryptAESCryptStub, NativeCall, PRESERVES_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Byte)); - registerForeignCall(DECRYPT, c.cipherBlockChainingDecryptAESCryptStub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, + registerForeignCall(DECRYPT, c.cipherBlockChainingDecryptAESCryptStub, NativeCall, PRESERVES_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Byte)); - registerForeignCall(DECRYPT_WITH_ORIGINAL_KEY, c.cipherBlockChainingDecryptAESCryptStub, NativeCall, PRESERVES_REGISTERS, LEAF_NOFP, REEXECUTABLE_ONLY_AFTER_EXCEPTION, + registerForeignCall(DECRYPT_WITH_ORIGINAL_KEY, c.cipherBlockChainingDecryptAESCryptStub, NativeCall, PRESERVES_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, NamedLocationIdentity.getArrayLocation(JavaKind.Byte)); } catch (GraalError e) { if (!(e.getCause() instanceof ClassNotFoundException)) { @@ -434,10 +446,34 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall } } } + + if (c.useAESCTRIntrinsics) { + assert (c.counterModeAESCrypt != 0L); + registerForeignCall(COUNTERMODE_IMPL_CRYPT, c.counterModeAESCrypt, NativeCall, PRESERVES_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, + NamedLocationIdentity.any()); + } + + if (c.useVectorizedMismatchIntrinsic) { + assert (c.vectorizedMismatch != 0L); + registerForeignCall(VECTORIZED_MISMATCHED, c.vectorizedMismatch, NativeCall, PRESERVES_REGISTERS, LEAF, REEXECUTABLE_ONLY_AFTER_EXCEPTION, + NamedLocationIdentity.any()); + + } } public HotSpotForeignCallLinkage getForeignCall(ForeignCallDescriptor descriptor) { assert foreignCalls != null : descriptor; return foreignCalls.get(descriptor); } + + @SuppressWarnings("unused") + protected void registerMathStubs(GraalHotSpotVMConfig hotSpotVMConfig, HotSpotProviders providers, OptionValues options) { + registerForeignCall(SIN.foreignCallDescriptor, hotSpotVMConfig.arithmeticSinAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(COS.foreignCallDescriptor, hotSpotVMConfig.arithmeticCosAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(TAN.foreignCallDescriptor, hotSpotVMConfig.arithmeticTanAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(EXP.foreignCallDescriptor, hotSpotVMConfig.arithmeticExpAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(LOG.foreignCallDescriptor, hotSpotVMConfig.arithmeticLogAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(LOG10.foreignCallDescriptor, hotSpotVMConfig.arithmeticLog10Address, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); + registerForeignCall(POW.foreignCallDescriptor, hotSpotVMConfig.arithmeticPowAddress, NativeCall, DESTROYS_REGISTERS, LEAF, REEXECUTABLE, NO_LOCATIONS); + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotNodePlugin.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotNodePlugin.java index ccc2cb348ba..cda6a8bfa2a 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotNodePlugin.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotNodePlugin.java @@ -28,6 +28,8 @@ import static jdk.vm.ci.meta.DeoptimizationAction.None; import static jdk.vm.ci.meta.DeoptimizationReason.TransferToInterpreter; import static org.graalvm.compiler.core.common.GraalOptions.ImmutableCode; +import java.lang.reflect.Field; + import org.graalvm.compiler.core.common.CompilationIdentifier; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.core.common.type.StampPair; @@ -66,8 +68,6 @@ import jdk.vm.ci.meta.JavaTypeProfile; import jdk.vm.ci.meta.ResolvedJavaField; import jdk.vm.ci.meta.ResolvedJavaMethod; import jdk.vm.ci.meta.ResolvedJavaType; - -import java.lang.reflect.Field; import sun.misc.Unsafe; /** @@ -243,10 +243,9 @@ public final class HotSpotNodePlugin implements NodePlugin, TypePlugin { } private static final LocationIdentity JAVA_THREAD_SHOULD_POST_ON_EXCEPTIONS_FLAG_LOCATION = NamedLocationIdentity.mutable("JavaThread::_should_post_on_exceptions_flag"); + static final Unsafe UNSAFE = initUnsafe(); - private static final Unsafe UNSAFE = initUnsafe(); - - private static Unsafe initUnsafe() { + static Unsafe initUnsafe() { try { // Fast path when we are trusted. return Unsafe.getUnsafe(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotUnsafeSubstitutions.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotUnsafeSubstitutions.java index 4ad8babd24a..d0a4fbc5e43 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotUnsafeSubstitutions.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotUnsafeSubstitutions.java @@ -24,7 +24,7 @@ package org.graalvm.compiler.hotspot.meta; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import org.graalvm.compiler.api.replacements.ClassSubstitution; import org.graalvm.compiler.api.replacements.MethodSubstitution; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotWordOperationPlugin.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotWordOperationPlugin.java index 86a28042a46..ec350f7116b 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotWordOperationPlugin.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/meta/HotSpotWordOperationPlugin.java @@ -121,7 +121,7 @@ public class HotSpotWordOperationPlugin extends WordOperationPlugin { ValueNode pointer = args[0]; assert pointer.stamp(NodeView.DEFAULT) instanceof MetaspacePointerStamp; - LogicNode isNull = b.addWithInputs(IsNullNode.create(pointer)); + LogicNode isNull = b.add(IsNullNode.create(pointer)); b.addPush(returnKind, ConditionalNode.create(isNull, b.add(forBoolean(true)), b.add(forBoolean(false)), NodeView.DEFAULT)); break; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/AllocaNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/AllocaNode.java index 1263c647617..b5517f7a1f9 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/AllocaNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/AllocaNode.java @@ -74,7 +74,7 @@ public final class AllocaNode extends FixedWithNextNode implements LIRLowerable @Override public void generate(NodeLIRBuilderTool gen) { - VirtualStackSlot array = gen.getLIRGeneratorTool().getResult().getFrameMapBuilder().allocateStackSlots(slots, objects, null); + VirtualStackSlot array = gen.getLIRGeneratorTool().allocateStackSlots(slots, objects, null); Value result = gen.getLIRGeneratorTool().emitAddress(array); gen.setResult(this, result); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/DimensionsNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/DimensionsNode.java index 14648ede5f0..26ba9227af0 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/DimensionsNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/DimensionsNode.java @@ -64,7 +64,7 @@ public final class DimensionsNode extends FixedWithNextNode implements LIRLowera int size = rank * 4; int wordSize = lirGen.target().wordSize; int slots = roundUp(size, wordSize) / wordSize; - VirtualStackSlot array = lirGen.getResult().getFrameMapBuilder().allocateStackSlots(slots, new BitSet(0), null); + VirtualStackSlot array = lirGen.allocateStackSlots(slots, new BitSet(0), null); Value result = lirGen.emitAddress(array); gen.setResult(this, result); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/type/KlassPointerStamp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/type/KlassPointerStamp.java index fb1b6d3e6eb..629ccce1eda 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/type/KlassPointerStamp.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/nodes/type/KlassPointerStamp.java @@ -71,6 +71,13 @@ public final class KlassPointerStamp extends MetaspacePointerStamp { this.encoding = encoding; } + @Override + public void accept(Visitor v) { + super.accept(v); + v.visitLong(encoding.getBase()); + v.visitInt(encoding.getShift()); + } + @Override protected AbstractPointerStamp copyWith(boolean newNonNull, boolean newAlwaysNull) { return new KlassPointerStamp(newNonNull, newAlwaysNull, encoding); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/OnStackReplacementPhase.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/OnStackReplacementPhase.java index dda65bb50ad..909c80936bf 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/OnStackReplacementPhase.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/OnStackReplacementPhase.java @@ -36,6 +36,7 @@ import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.iterators.NodeIterable; +import org.graalvm.compiler.loop.LoopEx; import org.graalvm.compiler.loop.LoopsData; import org.graalvm.compiler.loop.phases.LoopTransformations; import org.graalvm.compiler.nodeinfo.InputType; @@ -71,6 +72,7 @@ import org.graalvm.compiler.options.OptionType; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.Phase; import org.graalvm.compiler.phases.common.DeadCodeEliminationPhase; +import org.graalvm.compiler.serviceprovider.SpeculationReasonGroup; import jdk.vm.ci.meta.DeoptimizationAction; import jdk.vm.ci.meta.DeoptimizationReason; @@ -87,7 +89,7 @@ public class OnStackReplacementPhase extends Phase { "if there is no mature profile available for the rest of the method.", type = OptionType.Debug) public static final OptionKey DeoptAfterOSR = new OptionKey<>(true); @Option(help = "Support OSR compilations with locks. If DeoptAfterOSR is true we can per definition not have " + - "unbalaced enter/extis mappings. If DeoptAfterOSR is false insert artificial monitor enters after " + + "unbalanced enter/exits mappings. If DeoptAfterOSR is false insert artificial monitor enters after " + "the OSRStart to have balanced enter/exits in the graph.", type = OptionType.Debug) public static final OptionKey SupportOSRWithLocks = new OptionKey<>(true); // @formatter:on @@ -99,6 +101,8 @@ public class OnStackReplacementPhase extends Phase { return Options.SupportOSRWithLocks.getValue(options); } + private static final SpeculationReasonGroup OSR_LOCAL_SPECULATIONS = new SpeculationReasonGroup("OSRLocal", int.class, Stamp.class, int.class); + @Override @SuppressWarnings("try") protected void run(StructuredGraph graph) { @@ -152,7 +156,9 @@ public class OnStackReplacementPhase extends Phase { l = l.getParent(); } - LoopTransformations.peel(loops.loop(l)); + LoopEx loop = loops.loop(l); + loop.loopBegin().markOsrLoop(); + LoopTransformations.peel(loop); osr.replaceAtUsages(InputType.Guard, AbstractBeginNode.prevBegin((FixedNode) osr.predecessor())); for (Node usage : osr.usages().snapshot()) { EntryProxyNode proxy = (EntryProxyNode) usage; @@ -179,7 +185,7 @@ public class OnStackReplacementPhase extends Phase { final int locksSize = osrState.locksSize(); for (int i = 0; i < localsSize + locksSize; i++) { - ValueNode value = null; + ValueNode value; if (i >= localsSize) { value = osrState.lockAt(i - localsSize); } else { @@ -201,7 +207,7 @@ public class OnStackReplacementPhase extends Phase { osrLocal = graph.addOrUnique(new OSRLocalNode(i, unrestrictedStamp)); } // Speculate on the OSRLocal stamps that could be more precise. - OSRLocalSpeculationReason reason = new OSRLocalSpeculationReason(osrState.bci, narrowedStamp, i); + SpeculationReason reason = OSR_LOCAL_SPECULATIONS.createSpeculationReason(osrState.bci, narrowedStamp, i); if (graph.getSpeculationLog().maySpeculate(reason) && osrLocal instanceof OSRLocalNode && value.getStackKind().equals(JavaKind.Object) && !narrowedStamp.isUnrestricted()) { // Add guard. LogicNode check = graph.addOrUniqueWithInputs(InstanceOfNode.createHelper((ObjectStamp) narrowedStamp, osrLocal, null, null)); @@ -305,30 +311,4 @@ public class OnStackReplacementPhase extends Phase { public float codeSizeIncrease() { return 5.0f; } - - private static class OSRLocalSpeculationReason implements SpeculationReason { - private int bci; - private Stamp speculatedStamp; - private int localIndex; - - OSRLocalSpeculationReason(int bci, Stamp speculatedStamp, int localIndex) { - this.bci = bci; - this.speculatedStamp = speculatedStamp; - this.localIndex = localIndex; - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof OSRLocalSpeculationReason) { - OSRLocalSpeculationReason that = (OSRLocalSpeculationReason) obj; - return this.bci == that.bci && this.speculatedStamp.equals(that.speculatedStamp) && this.localIndex == that.localIndex; - } - return false; - } - - @Override - public int hashCode() { - return (bci << 16) ^ speculatedStamp.hashCode() ^ localIndex; - } - } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/WriteBarrierAdditionPhase.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/WriteBarrierAdditionPhase.java index 3de108f4533..5ae608a5a50 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/WriteBarrierAdditionPhase.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/WriteBarrierAdditionPhase.java @@ -25,35 +25,25 @@ package org.graalvm.compiler.hotspot.phases; import org.graalvm.compiler.debug.DebugCloseable; -import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; -import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePreWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PreWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1ReferentFieldReadBarrier; -import org.graalvm.compiler.hotspot.nodes.SerialArrayRangeWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1BarrierSet; +import org.graalvm.compiler.hotspot.gc.shared.BarrierSet; +import org.graalvm.compiler.hotspot.gc.shared.CardTableBarrierSet; import org.graalvm.compiler.nodes.StructuredGraph; -import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.extended.ArrayRangeWrite; import org.graalvm.compiler.nodes.java.AbstractCompareAndSwapNode; import org.graalvm.compiler.nodes.java.LoweredAtomicReadAndWriteNode; -import org.graalvm.compiler.nodes.memory.FixedAccessNode; -import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType; import org.graalvm.compiler.nodes.memory.ReadNode; import org.graalvm.compiler.nodes.memory.WriteNode; -import org.graalvm.compiler.nodes.memory.address.AddressNode; -import org.graalvm.compiler.nodes.type.StampTool; import org.graalvm.compiler.phases.Phase; public class WriteBarrierAdditionPhase extends Phase { - private GraalHotSpotVMConfig config; + private BarrierSet barrierSet; public WriteBarrierAdditionPhase(GraalHotSpotVMConfig config) { - this.config = config; + this.barrierSet = createBarrierSet(config); } @SuppressWarnings("try") @@ -62,141 +52,42 @@ public class WriteBarrierAdditionPhase extends Phase { for (Node n : graph.getNodes()) { try (DebugCloseable scope = n.graph().withNodeSourcePosition(n)) { if (n instanceof ReadNode) { - addReadNodeBarriers((ReadNode) n, graph); + barrierSet.addReadNodeBarriers((ReadNode) n, graph); } else if (n instanceof WriteNode) { - addWriteNodeBarriers((WriteNode) n, graph); + barrierSet.addWriteNodeBarriers((WriteNode) n, graph); } else if (n instanceof LoweredAtomicReadAndWriteNode) { LoweredAtomicReadAndWriteNode loweredAtomicReadAndWriteNode = (LoweredAtomicReadAndWriteNode) n; - addAtomicReadWriteNodeBarriers(loweredAtomicReadAndWriteNode, graph); + barrierSet.addAtomicReadWriteNodeBarriers(loweredAtomicReadAndWriteNode, graph); } else if (n instanceof AbstractCompareAndSwapNode) { - addCASBarriers((AbstractCompareAndSwapNode) n, graph); + barrierSet.addCASBarriers((AbstractCompareAndSwapNode) n, graph); } else if (n instanceof ArrayRangeWrite) { ArrayRangeWrite node = (ArrayRangeWrite) n; if (node.writesObjectArray()) { - addArrayRangeBarriers(node, graph); + barrierSet.addArrayRangeBarriers(node, graph); } } } } } - private void addReadNodeBarriers(ReadNode node, StructuredGraph graph) { - if (node.getBarrierType() == BarrierType.PRECISE) { - assert config.useG1GC; - G1ReferentFieldReadBarrier barrier = graph.add(new G1ReferentFieldReadBarrier(node.getAddress(), node, false)); - graph.addAfterFixed(node, barrier); - } else { - assert node.getBarrierType() == BarrierType.NONE : "Non precise read barrier has been attached to read node."; - } - } - - protected static void addG1PreWriteBarrier(FixedAccessNode node, AddressNode address, ValueNode value, boolean doLoad, boolean nullCheck, StructuredGraph graph) { - G1PreWriteBarrier preBarrier = graph.add(new G1PreWriteBarrier(address, value, doLoad, nullCheck)); - preBarrier.setStateBefore(node.stateBefore()); - node.setNullCheck(false); - node.setStateBefore(null); - graph.addBeforeFixed(node, preBarrier); - } - - protected void addG1PostWriteBarrier(FixedAccessNode node, AddressNode address, ValueNode value, boolean precise, StructuredGraph graph) { - final boolean alwaysNull = StampTool.isPointerAlwaysNull(value); - graph.addAfterFixed(node, graph.add(new G1PostWriteBarrier(address, value, precise, alwaysNull))); - } - - protected void addSerialPostWriteBarrier(FixedAccessNode node, AddressNode address, ValueNode value, boolean precise, StructuredGraph graph) { - final boolean alwaysNull = StampTool.isPointerAlwaysNull(value); - if (alwaysNull) { - // Serial barrier isn't needed for null value - return; - } - graph.addAfterFixed(node, graph.add(new SerialWriteBarrier(address, precise))); - } - - private void addWriteNodeBarriers(WriteNode node, StructuredGraph graph) { - BarrierType barrierType = node.getBarrierType(); - switch (barrierType) { - case NONE: - // nothing to do - break; - case IMPRECISE: - case PRECISE: - boolean precise = barrierType == BarrierType.PRECISE; - if (config.useG1GC) { - if (!node.getLocationIdentity().isInit()) { - // The pre barrier does nothing if the value being read is null, so it can - // be explicitly skipped when this is an initializing store. - addG1PreWriteBarrier(node, node.getAddress(), null, true, node.getNullCheck(), graph); - } - addG1PostWriteBarrier(node, node.getAddress(), node.value(), precise, graph); - } else { - addSerialPostWriteBarrier(node, node.getAddress(), node.value(), precise, graph); - } - break; - default: - throw new GraalError("unexpected barrier type: " + barrierType); - } - } - - private void addAtomicReadWriteNodeBarriers(LoweredAtomicReadAndWriteNode node, StructuredGraph graph) { - BarrierType barrierType = node.getBarrierType(); - switch (barrierType) { - case NONE: - // nothing to do - break; - case IMPRECISE: - case PRECISE: - boolean precise = barrierType == BarrierType.PRECISE; - if (config.useG1GC) { - addG1PreWriteBarrier(node, node.getAddress(), null, true, node.getNullCheck(), graph); - addG1PostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); - } else { - addSerialPostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); - } - break; - default: - throw new GraalError("unexpected barrier type: " + barrierType); - } - } - - private void addCASBarriers(AbstractCompareAndSwapNode node, StructuredGraph graph) { - BarrierType barrierType = node.getBarrierType(); - switch (barrierType) { - case NONE: - // nothing to do - break; - case IMPRECISE: - case PRECISE: - boolean precise = barrierType == BarrierType.PRECISE; - if (config.useG1GC) { - addG1PreWriteBarrier(node, node.getAddress(), node.getExpectedValue(), false, false, graph); - addG1PostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); - } else { - addSerialPostWriteBarrier(node, node.getAddress(), node.getNewValue(), precise, graph); - } - break; - default: - throw new GraalError("unexpected barrier type: " + barrierType); - } - } - - private void addArrayRangeBarriers(ArrayRangeWrite write, StructuredGraph graph) { - if (config.useG1GC) { - if (!write.isInitialization()) { - // The pre barrier does nothing if the value being read is null, so it can - // be explicitly skipped when this is an initializing store. - G1ArrayRangePreWriteBarrier g1ArrayRangePreWriteBarrier = graph.add(new G1ArrayRangePreWriteBarrier(write.getAddress(), write.getLength(), write.getElementStride())); - graph.addBeforeFixed(write.asNode(), g1ArrayRangePreWriteBarrier); - } - G1ArrayRangePostWriteBarrier g1ArrayRangePostWriteBarrier = graph.add(new G1ArrayRangePostWriteBarrier(write.getAddress(), write.getLength(), write.getElementStride())); - graph.addAfterFixed(write.asNode(), g1ArrayRangePostWriteBarrier); - } else { - SerialArrayRangeWriteBarrier serialArrayRangeWriteBarrier = graph.add(new SerialArrayRangeWriteBarrier(write.getAddress(), write.getLength(), write.getElementStride())); - graph.addAfterFixed(write.asNode(), serialArrayRangeWriteBarrier); - } - } - @Override public boolean checkContract() { return false; } + + private BarrierSet createBarrierSet(GraalHotSpotVMConfig config) { + if (config.useG1GC) { + return createG1BarrierSet(); + } else { + return createCardTableBarrierSet(); + } + } + + protected BarrierSet createCardTableBarrierSet() { + return new CardTableBarrierSet(); + } + + protected BarrierSet createG1BarrierSet() { + return new G1BarrierSet(); + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/WriteBarrierVerificationPhase.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/WriteBarrierVerificationPhase.java index feefeb67d69..34f14fc7c41 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/WriteBarrierVerificationPhase.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/phases/WriteBarrierVerificationPhase.java @@ -31,10 +31,10 @@ import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeFlood; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; -import org.graalvm.compiler.hotspot.nodes.ArrayRangeWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.ObjectWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1PostWriteBarrier; +import org.graalvm.compiler.hotspot.gc.shared.ArrayRangeWriteBarrier; +import org.graalvm.compiler.hotspot.gc.shared.ObjectWriteBarrier; +import org.graalvm.compiler.hotspot.gc.shared.SerialWriteBarrier; import org.graalvm.compiler.nodeinfo.Verbosity; import org.graalvm.compiler.nodes.DeoptimizingNode; import org.graalvm.compiler.nodes.FixedWithNextNode; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/ArraysSupportSubstitutions.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/ArraysSupportSubstitutions.java new file mode 100644 index 00000000000..5db1034a337 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/ArraysSupportSubstitutions.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, Red Hat Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + + +package org.graalvm.compiler.hotspot.replacements; + +import org.graalvm.compiler.api.replacements.ClassSubstitution; +import org.graalvm.compiler.api.replacements.MethodSubstitution; +import org.graalvm.compiler.hotspot.HotSpotBackend; +import org.graalvm.compiler.nodes.ComputeObjectAddressNode; +import org.graalvm.compiler.word.Word; +import jdk.internal.vm.compiler.word.WordFactory; + +@ClassSubstitution(className = "jdk.internal.util.ArraysSupport", optional = true) +public class ArraysSupportSubstitutions { + + @SuppressWarnings("unused") + @MethodSubstitution(isStatic = true) + static int vectorizedMismatch(Object a, long aOffset, Object b, long bOffset, int length, int log2ArrayIndexScale) { + Word aAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(a, aOffset)); + Word bAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(b, bOffset)); + + return HotSpotBackend.vectorizedMismatch(aAddr, bAddr, length, log2ArrayIndexScale); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/CounterModeSubstitutions.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/CounterModeSubstitutions.java new file mode 100644 index 00000000000..6ac5b6b44d2 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/CounterModeSubstitutions.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.replacements; + +import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_INTRINSIC_CONTEXT; +import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_METAACCESS; +import static org.graalvm.compiler.hotspot.replacements.CipherBlockChainingSubstitutions.aesCryptType; +import static org.graalvm.compiler.hotspot.replacements.CipherBlockChainingSubstitutions.embeddedCipherOffset; +import static org.graalvm.compiler.nodes.PiNode.piCastNonNull; + +import org.graalvm.compiler.api.replacements.ClassSubstitution; +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.compiler.api.replacements.Fold.InjectedParameter; +import org.graalvm.compiler.api.replacements.MethodSubstitution; +import org.graalvm.compiler.hotspot.HotSpotBackend; +import org.graalvm.compiler.nodes.ComputeObjectAddressNode; +import org.graalvm.compiler.nodes.extended.RawLoadNode; +import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext; +import org.graalvm.compiler.replacements.ReplacementsUtil; +import org.graalvm.compiler.word.Word; +import jdk.internal.vm.compiler.word.LocationIdentity; +import jdk.internal.vm.compiler.word.WordFactory; + +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.ResolvedJavaType; + +@ClassSubstitution(className = "com.sun.crypto.provider.CounterMode", optional = true) +public class CounterModeSubstitutions { + + @MethodSubstitution(isStatic = false) + static int implCrypt(Object receiver, byte[] in, int inOff, int len, byte[] out, int outOff) { + Object realReceiver = piCastNonNull(receiver, HotSpotReplacementsUtil.methodHolderClass(INJECTED_INTRINSIC_CONTEXT)); + Object embeddedCipher = RawLoadNode.load(realReceiver, embeddedCipherOffset(INJECTED_INTRINSIC_CONTEXT), JavaKind.Object, LocationIdentity.any()); + Object aesCipher = piCastNonNull(embeddedCipher, aesCryptType(INJECTED_INTRINSIC_CONTEXT)); + + Word srcAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(in, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Byte) + inOff)); + Word dstAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(out, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Byte) + outOff)); + Word usedPtr = WordFactory.unsigned(ComputeObjectAddressNode.get(realReceiver, usedOffset(INJECTED_INTRINSIC_CONTEXT))); + + int cntOffset = counterOffset(INJECTED_INTRINSIC_CONTEXT); + int encCntOffset = encCounterOffset(INJECTED_INTRINSIC_CONTEXT); + Object kObject = RawLoadNode.load(aesCipher, AESCryptSubstitutions.kOffset(INJECTED_INTRINSIC_CONTEXT), JavaKind.Object, LocationIdentity.any()); + Object cntObj = RawLoadNode.load(realReceiver, cntOffset, JavaKind.Object, LocationIdentity.any()); + Object encCntObj = RawLoadNode.load(realReceiver, encCntOffset, JavaKind.Object, LocationIdentity.any()); + + Word kPtr = Word.objectToTrackedPointer(kObject).add(ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int)); + Word cntPtr = Word.objectToTrackedPointer(cntObj).add(ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Byte)); + Word encCntPtr = Word.objectToTrackedPointer(encCntObj).add(ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Byte)); + + return HotSpotBackend.counterModeAESCrypt(srcAddr, dstAddr, kPtr, cntPtr, len, encCntPtr, usedPtr); + } + + static ResolvedJavaType counterModeType(IntrinsicContext context) { + return HotSpotReplacementsUtil.getType(context, "Lcom/sun/crypto/provider/CounterMode;"); + } + + @Fold + static int counterOffset(@InjectedParameter IntrinsicContext context) { + return HotSpotReplacementsUtil.getFieldOffset(counterModeType(context), "counter"); + } + + @Fold + static int encCounterOffset(@InjectedParameter IntrinsicContext context) { + return HotSpotReplacementsUtil.getFieldOffset(counterModeType(context), "encryptedCounter"); + } + + @Fold + static int usedOffset(@InjectedParameter IntrinsicContext context) { + return HotSpotReplacementsUtil.getFieldOffset(counterModeType(context), "used"); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/DigestBaseSubstitutions.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/DigestBaseSubstitutions.java new file mode 100644 index 00000000000..09f2fe929ca --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/DigestBaseSubstitutions.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.replacements; + +import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_METAACCESS; +import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_INTRINSIC_CONTEXT; +import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG; +import static org.graalvm.compiler.nodes.java.InstanceOfNode.doInstanceof; + +import org.graalvm.compiler.api.replacements.ClassSubstitution; +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.compiler.api.replacements.MethodSubstitution; +import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; +import org.graalvm.compiler.hotspot.HotSpotBackend; +import org.graalvm.compiler.nodes.ComputeObjectAddressNode; +import org.graalvm.compiler.nodes.PiNode; +import org.graalvm.compiler.nodes.extended.RawLoadNode; +import org.graalvm.compiler.replacements.ReplacementsUtil; +import org.graalvm.compiler.word.Word; +import jdk.internal.vm.compiler.word.LocationIdentity; +import jdk.internal.vm.compiler.word.WordFactory; + +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.ResolvedJavaType; + +@ClassSubstitution(className = "sun.security.provider.DigestBase", optional = true) +public class DigestBaseSubstitutions { + + @MethodSubstitution(isStatic = false) + static int implCompressMultiBlock0(Object receiver, byte[] buf, int ofs, int limit) { + Object realReceiver = PiNode.piCastNonNull(receiver, HotSpotReplacementsUtil.methodHolderClass(INJECTED_INTRINSIC_CONTEXT)); + ResolvedJavaType sha1type = HotSpotReplacementsUtil.getType(INJECTED_INTRINSIC_CONTEXT, "Lsun/security/provider/SHA;"); + ResolvedJavaType sha256type = HotSpotReplacementsUtil.getType(INJECTED_INTRINSIC_CONTEXT, "Lsun/security/provider/SHA2;"); + ResolvedJavaType sha512type = HotSpotReplacementsUtil.getType(INJECTED_INTRINSIC_CONTEXT, "Lsun/security/provider/SHA5;"); + + Word bufAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(buf, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Byte) + ofs)); + if (useSHA1Intrinsics(INJECTED_VMCONFIG) && doInstanceof(sha1type, realReceiver)) { + Object sha1obj = PiNode.piCastNonNull(realReceiver, sha1type); + Object state = RawLoadNode.load(sha1obj, HotSpotReplacementsUtil.getFieldOffset(sha1type, "state"), JavaKind.Object, LocationIdentity.any()); + Word stateAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(state, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int))); + return HotSpotBackend.shaImplCompressMBStub(bufAddr, stateAddr, ofs, limit); + } else if (useSHA256Intrinsics(INJECTED_VMCONFIG) && doInstanceof(sha256type, realReceiver)) { + Object sha256obj = PiNode.piCastNonNull(realReceiver, sha256type); + Object state = RawLoadNode.load(sha256obj, HotSpotReplacementsUtil.getFieldOffset(sha256type, "state"), JavaKind.Object, LocationIdentity.any()); + Word stateAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(state, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int))); + return HotSpotBackend.sha2ImplCompressMBStub(bufAddr, stateAddr, ofs, limit); + } else if (useSHA512Intrinsics(INJECTED_VMCONFIG) && doInstanceof(sha512type, realReceiver)) { + Object sha512obj = PiNode.piCastNonNull(realReceiver, sha512type); + Object state = RawLoadNode.load(sha512obj, HotSpotReplacementsUtil.getFieldOffset(sha512type, "state"), JavaKind.Object, LocationIdentity.any()); + Word stateAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(state, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int))); + return HotSpotBackend.sha5ImplCompressMBStub(bufAddr, stateAddr, ofs, limit); + } else { + return implCompressMultiBlock0(realReceiver, buf, ofs, limit); + } + } + + @Fold + public static boolean useSHA1Intrinsics(@Fold.InjectedParameter GraalHotSpotVMConfig config) { + return config.useSHA1Intrinsics(); + } + + @Fold + public static boolean useSHA256Intrinsics(@Fold.InjectedParameter GraalHotSpotVMConfig config) { + return config.useSHA256Intrinsics(); + } + + @Fold + public static boolean useSHA512Intrinsics(@Fold.InjectedParameter GraalHotSpotVMConfig config) { + return config.useSHA512Intrinsics(); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotClassSubstitutions.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotClassSubstitutions.java index bcbe951d10e..e67cd190583 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotClassSubstitutions.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotClassSubstitutions.java @@ -25,6 +25,7 @@ package org.graalvm.compiler.hotspot.replacements; import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG; +import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_METAACCESS; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.ARRAY_KLASS_COMPONENT_MIRROR; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_ACCESS_FLAGS_LOCATION; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_MODIFIER_FLAGS_LOCATION; @@ -38,11 +39,16 @@ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil. import java.lang.reflect.Modifier; import org.graalvm.compiler.api.replacements.ClassSubstitution; +import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.api.replacements.MethodSubstitution; import org.graalvm.compiler.hotspot.word.KlassPointer; +import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.PiNode; import org.graalvm.compiler.nodes.SnippetAnchorNode; +import jdk.vm.ci.meta.MetaAccessProvider; +import jdk.vm.ci.meta.ResolvedJavaType; + // JaCoCo Exclude /** @@ -92,6 +98,11 @@ public class HotSpotClassSubstitutions { return klass.isNull(); } + @Fold + public static ResolvedJavaType getObjectType(@Fold.InjectedParameter MetaAccessProvider metaAccesss) { + return metaAccesss.lookupJavaType(Object.class); + } + @MethodSubstitution(isStatic = false) public static Class getSuperclass(final Class thisObj) { KlassPointer klass = ClassGetHubNode.readClass(thisObj); @@ -100,7 +111,7 @@ public class HotSpotClassSubstitutions { int accessFlags = klassNonNull.readInt(klassAccessFlagsOffset(INJECTED_VMCONFIG), KLASS_ACCESS_FLAGS_LOCATION); if ((accessFlags & Modifier.INTERFACE) == 0) { if (klassIsArray(klassNonNull)) { - return Object.class; + return ConstantNode.forClass(getObjectType(INJECTED_METAACCESS)); } else { KlassPointer superKlass = klassNonNull.readKlassPointer(klassSuperKlassOffset(INJECTED_VMCONFIG), KLASS_SUPER_KLASS_LOCATION); if (superKlass.isNull()) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotReplacementsUtil.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotReplacementsUtil.java index b42d4ebd705..23b6a2246fe 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotReplacementsUtil.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/HotSpotReplacementsUtil.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -138,10 +138,12 @@ public class HotSpotReplacementsUtil { } } - public static ResolvedJavaType methodHolderClass(@Fold.InjectedParameter IntrinsicContext context) { + @Fold + public static ResolvedJavaType methodHolderClass(@InjectedParameter IntrinsicContext context) { return context.getOriginalMethod().getDeclaringClass(); } + @Fold static ResolvedJavaType getType(@Fold.InjectedParameter IntrinsicContext context, String typeName) { try { UnresolvedJavaType unresolved = UnresolvedJavaType.create(typeName); @@ -151,6 +153,7 @@ public class HotSpotReplacementsUtil { } } + @Fold static int getFieldOffset(ResolvedJavaType type, String fieldName) { for (ResolvedJavaField field : type.getInstanceFields(true)) { if (field.getName().equals(fieldName)) { @@ -584,7 +587,7 @@ public class HotSpotReplacementsUtil { * Calls {@link #arrayAllocationSize(int, int, int, int)} using an injected VM configuration * object. */ - public static int arrayAllocationSize(int length, int headerSize, int log2ElementSize) { + public static long arrayAllocationSize(int length, int headerSize, int log2ElementSize) { return arrayAllocationSize(length, headerSize, log2ElementSize, objectAlignment(INJECTED_VMCONFIG)); } @@ -600,9 +603,9 @@ public class HotSpotReplacementsUtil { * requirement} * @return the size of the memory chunk */ - public static int arrayAllocationSize(int length, int headerSize, int log2ElementSize, int alignment) { - int size = (length << log2ElementSize) + headerSize + (alignment - 1); - int mask = ~(alignment - 1); + public static long arrayAllocationSize(int length, int headerSize, int log2ElementSize, int alignment) { + long size = ((long) length << log2ElementSize) + headerSize + (alignment - 1); + long mask = ~(alignment - 1); return size & mask; } @@ -730,7 +733,7 @@ public class HotSpotReplacementsUtil { } public static KlassPointer loadKlassFromObject(Object object, int offset, LocationIdentity identity) { - ReplacementsUtil.staticAssert(offset != hubOffset(INJECTED_VMCONFIG), "Use loadHubIntrinsic instead of loadWordFromObject"); + ReplacementsUtil.staticAssert(offset != hubOffset(INJECTED_VMCONFIG), "Use loadHubIntrinsic instead of loadKlassFromObject"); return loadKlassFromObjectIntrinsic(object, offset, identity, getWordKind()); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/NewObjectSnippets.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/NewObjectSnippets.java index 9da3cd13ecd..4c1fcbaa5bd 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/NewObjectSnippets.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/NewObjectSnippets.java @@ -209,19 +209,19 @@ public class NewObjectSnippets implements Snippets { } @Snippet - public static Object allocateInstance(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents, + public static Object allocateInstance(@ConstantParameter long size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter Counters counters) { return piCastToSnippetReplaceeStamp(allocateInstanceHelper(size, hub, prototypeMarkWord, fillContents, threadRegister, constantSize, typeContext, counters)); } - public static Object allocateInstanceHelper(int size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents, + public static Object allocateInstanceHelper(long size, KlassPointer hub, Word prototypeMarkWord, boolean fillContents, Register threadRegister, boolean constantSize, String typeContext, Counters counters) { Object result; Word thread = registerAsWord(threadRegister); Word top = readTlabTop(thread); Word end = readTlabEnd(thread); - Word newTop = top.add(size); + Word newTop = top.add(WordFactory.unsigned(size)); if (useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) { writeTlabTop(thread, newTop); emitPrefetchAllocate(newTop, false); @@ -252,7 +252,7 @@ public class NewObjectSnippets implements Snippets { private static native Object newInstanceOrNull(@ConstantNodeParameter ForeignCallDescriptor descriptor, KlassPointer hub); @Snippet - public static Object allocateInstancePIC(@ConstantParameter int size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents, + public static Object allocateInstancePIC(@ConstantParameter long size, KlassPointer hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister, @ConstantParameter boolean constantSize, @ConstantParameter String typeContext, @ConstantParameter Counters counters) { // Klass must be initialized by the time the first instance is allocated, therefore we can @@ -316,7 +316,7 @@ public class NewObjectSnippets implements Snippets { @ConstantParameter Counters counters) { // Primitive array types are eagerly pre-resolved. We can use a floating load. KlassPointer picHub = LoadConstantIndirectlyNode.loadKlass(hub); - return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, counters); + return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, counters); } @Snippet @@ -325,7 +325,7 @@ public class NewObjectSnippets implements Snippets { @ConstantParameter Counters counters) { // Array type would be resolved by dominating resolution. KlassPointer picHub = LoadConstantIndirectlyFixedNode.loadKlass(hub); - return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, false, counters); + return allocateArrayImpl(picHub, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, maybeUnroll, typeContext, counters); } @Snippet @@ -348,7 +348,6 @@ public class NewObjectSnippets implements Snippets { threadRegister, maybeUnroll, typeContext, - false, counters); return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length); @@ -364,14 +363,14 @@ public class NewObjectSnippets implements Snippets { } private static Object allocateArrayImpl(KlassPointer hub, int length, Word prototypeMarkWord, int headerSize, int log2ElementSize, boolean fillContents, Register threadRegister, - boolean maybeUnroll, String typeContext, boolean skipNegativeCheck, Counters counters) { + boolean maybeUnroll, String typeContext, Counters counters) { Object result; - int allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize); + long allocationSize = arrayAllocationSize(length, headerSize, log2ElementSize); Word thread = registerAsWord(threadRegister); Word top = readTlabTop(thread); Word end = readTlabEnd(thread); - Word newTop = top.add(allocationSize); - if (probability(FREQUENT_PROBABILITY, skipNegativeCheck || belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) && + Word newTop = top.add(WordFactory.unsigned(allocationSize)); + if (probability(FREQUENT_PROBABILITY, belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) && useTLAB(INJECTED_VMCONFIG) && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) { writeTlabTop(thread, newTop); emitPrefetchAllocate(newTop, true); @@ -486,7 +485,7 @@ public class NewObjectSnippets implements Snippets { int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift(INJECTED_VMCONFIG)) & layoutHelperHeaderSizeMask(INJECTED_VMCONFIG); int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift(INJECTED_VMCONFIG)) & layoutHelperLog2ElementSizeMask(INJECTED_VMCONFIG); - Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", true, counters); + Object result = allocateArrayImpl(nonNullKlass, length, prototypeMarkWord, headerSize, log2ElementSize, fillContents, threadRegister, false, "dynamic type", counters); return piArrayCastToSnippetReplaceeStamp(verifyOop(result), length); } @@ -540,11 +539,11 @@ public class NewObjectSnippets implements Snippets { * @param startOffset offset to begin zeroing. May not be word aligned. * @param manualUnroll maximally unroll zeroing */ - private static void zeroMemory(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) { + private static void zeroMemory(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) { fillMemory(0, size, memory, constantSize, startOffset, manualUnroll, counters); } - private static void fillMemory(long value, int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) { + private static void fillMemory(long value, long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) { ReplacementsUtil.runtimeAssert((size & 0x7) == 0, "unaligned object size"); int offset = startOffset; if ((offset & 0x7) != 0) { @@ -598,14 +597,14 @@ public class NewObjectSnippets implements Snippets { * @param startOffset offset to begin zeroing. May not be word aligned. * @param manualUnroll maximally unroll zeroing */ - private static void fillWithGarbage(int size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) { + private static void fillWithGarbage(long size, Word memory, boolean constantSize, int startOffset, boolean manualUnroll, Counters counters) { fillMemory(0xfefefefefefefefeL, size, memory, constantSize, startOffset, manualUnroll, counters); } /** * Formats some allocated memory with an object header and zeroes out the rest. */ - private static Object formatObject(KlassPointer hub, int size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) { + private static Object formatObject(KlassPointer hub, long size, Word memory, Word compileTimePrototypeMarkWord, boolean fillContents, boolean constantSize, Counters counters) { Word prototypeMarkWord = useBiasedLocking(INJECTED_VMCONFIG) ? hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION) : compileTimePrototypeMarkWord; initializeObjectHeader(memory, prototypeMarkWord, hub); if (fillContents) { @@ -632,7 +631,7 @@ public class NewObjectSnippets implements Snippets { /** * Formats some allocated memory with an object header and zeroes out the rest. */ - private static Object formatArray(KlassPointer hub, int allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll, + private static Object formatArray(KlassPointer hub, long allocationSize, int length, int headerSize, Word memory, Word prototypeMarkWord, boolean fillContents, boolean maybeUnroll, Counters counters) { memory.writeInt(arrayLengthOffset(INJECTED_VMCONFIG), length, LocationIdentity.init()); /* @@ -699,7 +698,7 @@ public class NewObjectSnippets implements Snippets { HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) newInstanceNode.instanceClass(); assert !type.isArray(); ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), providers.getMetaAccess(), graph); - int size = instanceSize(type); + long size = instanceSize(type); OptionValues localOptions = graph.getOptions(); SnippetInfo snippet = GeneratePIC.getValue(localOptions) ? allocateInstancePIC : allocateInstance; @@ -824,8 +823,8 @@ public class NewObjectSnippets implements Snippets { template(newmultiarrayNode, args).instantiate(providers.getMetaAccess(), newmultiarrayNode, DEFAULT_REPLACER, args); } - private static int instanceSize(HotSpotResolvedObjectType type) { - int size = type.instanceSize(); + private static long instanceSize(HotSpotResolvedObjectType type) { + long size = type.instanceSize(); assert size >= 0; return size; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/SHA2Substitutions.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/SHA2Substitutions.java index 6ee2f2b9d2b..ab2c39ac1f5 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/SHA2Substitutions.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/SHA2Substitutions.java @@ -26,7 +26,7 @@ package org.graalvm.compiler.hotspot.replacements; import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_METAACCESS; import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_INTRINSIC_CONTEXT; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import org.graalvm.compiler.api.replacements.ClassSubstitution; import org.graalvm.compiler.api.replacements.Fold; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/SHA5Substitutions.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/SHA5Substitutions.java index 1ff2c07c920..c3c74f4b04e 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/SHA5Substitutions.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/SHA5Substitutions.java @@ -26,14 +26,17 @@ package org.graalvm.compiler.hotspot.replacements; import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_INTRINSIC_CONTEXT; import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_METAACCESS; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import org.graalvm.compiler.api.replacements.ClassSubstitution; +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.compiler.api.replacements.Fold.InjectedParameter; import org.graalvm.compiler.api.replacements.MethodSubstitution; import org.graalvm.compiler.hotspot.HotSpotBackend; import org.graalvm.compiler.nodes.ComputeObjectAddressNode; import org.graalvm.compiler.nodes.PiNode; import org.graalvm.compiler.nodes.extended.RawLoadNode; +import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext; import org.graalvm.compiler.replacements.ReplacementsUtil; import org.graalvm.compiler.word.Word; import jdk.internal.vm.compiler.word.LocationIdentity; @@ -49,14 +52,15 @@ public class SHA5Substitutions { @MethodSubstitution(isStatic = false) static void implCompress0(Object receiver, byte[] buf, int ofs) { Object realReceiver = PiNode.piCastNonNull(receiver, HotSpotReplacementsUtil.methodHolderClass(INJECTED_INTRINSIC_CONTEXT)); - Object state = RawLoadNode.load(realReceiver, stateOffset(), JavaKind.Object, LocationIdentity.any()); + Object state = RawLoadNode.load(realReceiver, stateOffset(INJECTED_INTRINSIC_CONTEXT), JavaKind.Object, LocationIdentity.any()); Word bufAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(buf, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Byte) + ofs)); Word stateAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(state, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int))); HotSpotBackend.sha5ImplCompressStub(bufAddr, stateAddr); } - static long stateOffset() { - return HotSpotReplacementsUtil.getFieldOffset(HotSpotReplacementsUtil.methodHolderClass(INJECTED_INTRINSIC_CONTEXT), "state"); + @Fold + static long stateOffset(@InjectedParameter IntrinsicContext context) { + return HotSpotReplacementsUtil.getFieldOffset(HotSpotReplacementsUtil.methodHolderClass(context), "state"); } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/SHASubstitutions.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/SHASubstitutions.java index 22d413e8233..6623185132f 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/SHASubstitutions.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/SHASubstitutions.java @@ -26,14 +26,17 @@ package org.graalvm.compiler.hotspot.replacements; import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_INTRINSIC_CONTEXT; import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_METAACCESS; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import org.graalvm.compiler.api.replacements.ClassSubstitution; +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.compiler.api.replacements.Fold.InjectedParameter; import org.graalvm.compiler.api.replacements.MethodSubstitution; import org.graalvm.compiler.hotspot.HotSpotBackend; import org.graalvm.compiler.nodes.ComputeObjectAddressNode; import org.graalvm.compiler.nodes.PiNode; import org.graalvm.compiler.nodes.extended.RawLoadNode; +import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext; import org.graalvm.compiler.replacements.ReplacementsUtil; import org.graalvm.compiler.word.Word; import jdk.internal.vm.compiler.word.LocationIdentity; @@ -49,14 +52,15 @@ public class SHASubstitutions { @MethodSubstitution(isStatic = false) static void implCompress0(Object receiver, byte[] buf, int ofs) { Object realReceiver = PiNode.piCastNonNull(receiver, HotSpotReplacementsUtil.methodHolderClass(INJECTED_INTRINSIC_CONTEXT)); - Object state = RawLoadNode.load(realReceiver, stateOffset(), JavaKind.Object, LocationIdentity.any()); + Object state = RawLoadNode.load(realReceiver, stateOffset(INJECTED_INTRINSIC_CONTEXT), JavaKind.Object, LocationIdentity.any()); Word bufAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(buf, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Byte) + ofs)); Word stateAddr = WordFactory.unsigned(ComputeObjectAddressNode.get(state, ReplacementsUtil.getArrayBaseOffset(INJECTED_METAACCESS, JavaKind.Int))); HotSpotBackend.shaImplCompressStub(bufAddr, stateAddr); } - static long stateOffset() { - return HotSpotReplacementsUtil.getFieldOffset(HotSpotReplacementsUtil.methodHolderClass(INJECTED_INTRINSIC_CONTEXT), "state"); + @Fold + static long stateOffset(@InjectedParameter IntrinsicContext context) { + return HotSpotReplacementsUtil.getFieldOffset(HotSpotReplacementsUtil.methodHolderClass(context), "state"); } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/StringUTF16Substitutions.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/StringUTF16Substitutions.java new file mode 100644 index 00000000000..03b74746c2a --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/StringUTF16Substitutions.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.hotspot.replacements; + +import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_VMCONFIG; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; + +import jdk.vm.ci.meta.DeoptimizationAction; +import jdk.vm.ci.meta.DeoptimizationReason; +import jdk.vm.ci.meta.JavaKind; +import org.graalvm.compiler.api.replacements.ClassSubstitution; +import org.graalvm.compiler.api.replacements.MethodSubstitution; +import org.graalvm.compiler.nodes.DeoptimizeNode; +import org.graalvm.compiler.nodes.java.NewArrayNode; +import org.graalvm.compiler.replacements.arraycopy.ArrayCopyCallNode; + +// JaCoCo Exclude + +/** + * Substitutions for {@code StringUTF16} methods for JDK9 and later. + */ +@ClassSubstitution(className = "java.lang.StringUTF16", optional = true) +public class StringUTF16Substitutions { + + private static final int MAX_LENGTH = Integer.MAX_VALUE >> 1; + + @MethodSubstitution + public static byte[] toBytes(char[] value, int srcBegin, int length) { + if (probability(SLOW_PATH_PROBABILITY, srcBegin < 0) || + probability(SLOW_PATH_PROBABILITY, length < 0) || + probability(SLOW_PATH_PROBABILITY, length > MAX_LENGTH) || + probability(SLOW_PATH_PROBABILITY, srcBegin > value.length - length)) { + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.BoundsCheckException); + } + byte[] val = (byte[]) NewArrayNode.newUninitializedArray(Byte.TYPE, length << 1); + // the intrinsic does not perform bounds/type checks, so it can be used here. + // Using KillsAny variant since we are reading and writing 2 different types. + ArrayCopyCallNode.disjointArraycopyKillsAny(value, srcBegin, val, 0, length, JavaKind.Char, HotSpotReplacementsUtil.getHeapWordSize(INJECTED_VMCONFIG)); + return val; + } + + @MethodSubstitution + public static void getChars(byte[] value, int srcBegin, int srcEnd, char[] dst, int dstBegin) { + int length = srcEnd - srcBegin; + if (probability(SLOW_PATH_PROBABILITY, srcBegin < 0) || + probability(SLOW_PATH_PROBABILITY, length < 0) || + probability(SLOW_PATH_PROBABILITY, srcBegin > (value.length >> 1) - length) || + probability(SLOW_PATH_PROBABILITY, dstBegin > dst.length - length)) { + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.BoundsCheckException); + } + // The intrinsic does not perform bounds/type checks, so it can be used here. + // Using KillsAny variant since we are reading and writing 2 different types. + ArrayCopyCallNode.disjointArraycopyKillsAny(value, srcBegin, dst, dstBegin, length, JavaKind.Char, HotSpotReplacementsUtil.getHeapWordSize(INJECTED_VMCONFIG)); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/WriteBarrierSnippets.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/WriteBarrierSnippets.java index 415c86befbe..02923f67378 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/WriteBarrierSnippets.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/WriteBarrierSnippets.java @@ -54,17 +54,17 @@ import org.graalvm.compiler.debug.DebugHandlersFactory; import org.graalvm.compiler.graph.Node.ConstantNodeParameter; import org.graalvm.compiler.graph.Node.NodeIntrinsic; import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig; +import org.graalvm.compiler.hotspot.gc.g1.G1ArrayRangePostWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1ArrayRangePreWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1PostWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1PreWriteBarrier; +import org.graalvm.compiler.hotspot.gc.g1.G1ReferentFieldReadBarrier; +import org.graalvm.compiler.hotspot.gc.shared.SerialArrayRangeWriteBarrier; +import org.graalvm.compiler.hotspot.gc.shared.SerialWriteBarrier; import org.graalvm.compiler.hotspot.meta.HotSpotProviders; import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider; -import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePreWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PostWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1PreWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.G1ReferentFieldReadBarrier; import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode; import org.graalvm.compiler.hotspot.nodes.HotSpotCompressionNode; -import org.graalvm.compiler.hotspot.nodes.SerialArrayRangeWriteBarrier; -import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier; import org.graalvm.compiler.hotspot.nodes.VMErrorNode; import org.graalvm.compiler.nodes.NamedLocationIdentity; import org.graalvm.compiler.nodes.NodeView; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/OutOfBoundsExceptionStub.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/OutOfBoundsExceptionStub.java index 076609eb493..1cf5020444f 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/OutOfBoundsExceptionStub.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/OutOfBoundsExceptionStub.java @@ -34,7 +34,7 @@ import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage; import org.graalvm.compiler.hotspot.meta.HotSpotProviders; import org.graalvm.compiler.hotspot.nodes.AllocaNode; import org.graalvm.compiler.options.OptionValues; -import org.graalvm.compiler.serviceprovider.GraalServices; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import org.graalvm.compiler.word.Word; import jdk.vm.ci.code.Register; @@ -48,7 +48,7 @@ public class OutOfBoundsExceptionStub extends CreateExceptionStub { } // JDK-8201593: Print array length in ArrayIndexOutOfBoundsException. - private static final boolean PRINT_LENGTH_IN_EXCEPTION = GraalServices.JAVA_SPECIFICATION_VERSION >= 11; + private static final boolean PRINT_LENGTH_IN_EXCEPTION = JavaVersionUtil.JAVA_SPECIFICATION_VERSION >= 11; private static final int MAX_INT_STRING_SIZE = Integer.toString(Integer.MIN_VALUE).length(); private static final String STR_INDEX = "Index "; private static final String STR_OUTOFBOUNDSFORLENGTH = " out of bounds for length "; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/Stub.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/Stub.java index 63c2f010044..15fbf21c00c 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/Stub.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/stubs/Stub.java @@ -25,7 +25,6 @@ package org.graalvm.compiler.hotspot.stubs; import static java.util.Collections.singletonList; -import static org.graalvm.compiler.core.GraalCompiler.emitBackEnd; import static org.graalvm.compiler.core.GraalCompiler.emitFrontEnd; import static org.graalvm.compiler.core.common.GraalOptions.GeneratePIC; import static org.graalvm.compiler.debug.DebugContext.DEFAULT_LOG_STREAM; @@ -236,7 +235,7 @@ public abstract class Stub { Suites suites = createSuites(); emitFrontEnd(providers, backend, graph, providers.getSuites().getDefaultGraphBuilderSuite(), OptimisticOptimizations.ALL, DefaultProfilingInfo.get(TriState.UNKNOWN), suites); LIRSuites lirSuites = createLIRSuites(); - emitBackEnd(graph, Stub.this, getInstalledCodeOwner(), backend, compResult, CompilationResultBuilderFactory.Default, getRegisterConfig(), lirSuites); + backend.emitBackEnd(graph, Stub.this, getInstalledCodeOwner(), compResult, CompilationResultBuilderFactory.Default, getRegisterConfig(), lirSuites); assert checkStubInvariants(compResult); } catch (Throwable e) { throw debug.handle(e); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BytecodeParser.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BytecodeParser.java index 85a300062f2..a0b12fafe57 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BytecodeParser.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.java/src/org/graalvm/compiler/java/BytecodeParser.java @@ -38,6 +38,7 @@ import static jdk.vm.ci.meta.DeoptimizationReason.TypeCheckedInliningViolated; import static jdk.vm.ci.meta.DeoptimizationReason.UnreachedCode; import static jdk.vm.ci.meta.DeoptimizationReason.Unresolved; import static jdk.vm.ci.runtime.JVMCICompiler.INVOCATION_ENTRY_BCI; +import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; import static org.graalvm.compiler.bytecode.Bytecodes.AALOAD; import static org.graalvm.compiler.bytecode.Bytecodes.AASTORE; import static org.graalvm.compiler.bytecode.Bytecodes.ACONST_NULL; @@ -270,6 +271,7 @@ import java.util.function.Supplier; import jdk.internal.vm.compiler.collections.EconomicMap; import jdk.internal.vm.compiler.collections.Equivalence; +import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.api.replacements.Snippet; import org.graalvm.compiler.bytecode.Bytecode; import org.graalvm.compiler.bytecode.BytecodeDisassembler; @@ -424,7 +426,7 @@ import org.graalvm.compiler.nodes.util.GraphUtil; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.OptimisticOptimizations; import org.graalvm.compiler.phases.util.ValueMergeUtil; -import org.graalvm.compiler.serviceprovider.GraalServices; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import jdk.internal.vm.compiler.word.LocationIdentity; import jdk.vm.ci.code.BailoutException; @@ -1290,8 +1292,8 @@ public class BytecodeParser implements GraphBuilderContext { return graph.addOrUniqueWithInputs(x); } - protected ValueNode genIfNode(LogicNode condition, FixedNode falseSuccessor, FixedNode trueSuccessor, double d) { - return new IfNode(condition, falseSuccessor, trueSuccessor, d); + protected ValueNode genIfNode(LogicNode condition, FixedNode trueSuccessor, FixedNode falseSuccessor, double d) { + return new IfNode(condition, trueSuccessor, falseSuccessor, d); } protected void genThrow() { @@ -1646,11 +1648,11 @@ public class BytecodeParser implements GraphBuilderContext { private boolean forceInliningEverything; @Override - public void handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean inlineEverything) { + public Invoke handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean inlineEverything) { boolean previous = forceInliningEverything; forceInliningEverything = previous || inlineEverything; try { - appendInvoke(invokeKind, targetMethod, args); + return appendInvoke(invokeKind, targetMethod, args); } finally { forceInliningEverything = previous; } @@ -1720,7 +1722,6 @@ public class BytecodeParser implements GraphBuilderContext { } } if (invokeKind.isDirect()) { - inlineInfo = tryInline(args, targetMethod); if (inlineInfo == SUCCESSFULLY_INLINED) { return null; @@ -2320,7 +2321,7 @@ public class BytecodeParser implements GraphBuilderContext { } return false; } - if (canInlinePartialIntrinsicExit() && InlinePartialIntrinsicExitDuringParsing.getValue(options)) { + if (canInlinePartialIntrinsicExit() && InlinePartialIntrinsicExitDuringParsing.getValue(options) && !IS_BUILDING_NATIVE_IMAGE) { // Otherwise inline the original method. Any frame state created // during the inlining will exclude frame(s) in the // intrinsic method (see FrameStateBuilder.create(int bci)). @@ -2540,6 +2541,7 @@ public class BytecodeParser implements GraphBuilderContext { protected void genReturn(ValueNode returnVal, JavaKind returnKind) { if (parsingIntrinsic() && returnVal != null) { + if (returnVal instanceof StateSplit) { StateSplit stateSplit = (StateSplit) returnVal; FrameState stateAfter = stateSplit.stateAfter(); @@ -2548,7 +2550,20 @@ public class BytecodeParser implements GraphBuilderContext { if (stateAfter.bci == BytecodeFrame.AFTER_BCI) { assert stateAfter.usages().count() == 1; assert stateAfter.usages().first() == stateSplit; - stateAfter.replaceAtUsages(graph.add(new FrameState(BytecodeFrame.AFTER_BCI, returnVal))); + FrameState state; + if (returnVal.getStackKind() == JavaKind.Illegal) { + // This should only occur when Fold and NodeIntrinsic plugins are + // deferred. Their return value might not be a Java type and in that + // case this can't be the final AFTER_BCI so just create a FrameState + // without a return value on the top of stack. + assert stateSplit instanceof Invoke; + ResolvedJavaMethod targetMethod = ((Invoke) stateSplit).getTargetMethod(); + assert targetMethod != null && (targetMethod.getAnnotation(Fold.class) != null || targetMethod.getAnnotation(Node.NodeIntrinsic.class) != null); + state = new FrameState(BytecodeFrame.AFTER_BCI); + } else { + state = new FrameState(BytecodeFrame.AFTER_BCI, returnVal); + } + stateAfter.replaceAtUsages(graph.add(state)); GraphUtil.killWithUnusedFloatingInputs(stateAfter); } else { /* @@ -3395,26 +3410,44 @@ public class BytecodeParser implements GraphBuilderContext { condition = genUnique(condition); } - NodeSourcePosition currentPosition = graph.currentNodeSourcePosition(); + BciBlock deoptBlock = null; + BciBlock noDeoptBlock = null; if (isNeverExecutedCode(probability)) { - NodeSourcePosition survivingSuccessorPosition = graph.trackNodeSourcePosition() - ? new NodeSourcePosition(currentPosition.getCaller(), currentPosition.getMethod(), falseBlock.startBci) - : null; - append(new FixedGuardNode(condition, UnreachedCode, InvalidateReprofile, true, survivingSuccessorPosition)); - if (profilingPlugin != null && profilingPlugin.shouldProfile(this, method)) { - profilingPlugin.profileGoto(this, method, bci(), falseBlock.startBci, stateBefore); - } - appendGoto(falseBlock); - return; + deoptBlock = trueBlock; + noDeoptBlock = falseBlock; } else if (isNeverExecutedCode(1 - probability)) { - NodeSourcePosition survivingSuccessorPosition = graph.trackNodeSourcePosition() - ? new NodeSourcePosition(currentPosition.getCaller(), currentPosition.getMethod(), trueBlock.startBci) - : null; - append(new FixedGuardNode(condition, UnreachedCode, InvalidateReprofile, false, survivingSuccessorPosition)); - if (profilingPlugin != null && profilingPlugin.shouldProfile(this, method)) { - profilingPlugin.profileGoto(this, method, bci(), trueBlock.startBci, stateBefore); + deoptBlock = falseBlock; + noDeoptBlock = trueBlock; + } + + if (deoptBlock != null) { + NodeSourcePosition currentPosition = graph.currentNodeSourcePosition(); + NodeSourcePosition survivingSuccessorPosition = null; + if (graph.trackNodeSourcePosition()) { + survivingSuccessorPosition = new NodeSourcePosition(currentPosition.getCaller(), currentPosition.getMethod(), noDeoptBlock.startBci); + } + boolean negated = deoptBlock == trueBlock; + if (!isPotentialCountedLoopExit(condition, deoptBlock)) { + if (profilingPlugin != null && profilingPlugin.shouldProfile(this, method)) { + profilingPlugin.profileGoto(this, method, bci(), noDeoptBlock.startBci, stateBefore); + } + append(new FixedGuardNode(condition, UnreachedCode, InvalidateReprofile, negated, survivingSuccessorPosition)); + appendGoto(noDeoptBlock); + } else { + this.controlFlowSplit = true; + FixedNode noDeoptSuccessor = createTarget(noDeoptBlock, frameState, false, true); + DeoptimizeNode deopt = graph.add(new DeoptimizeNode(InvalidateReprofile, UnreachedCode)); + /* + * We do not want to `checkLoopExit` here: otherwise the deopt will go to the + * deoptBlock's BCI, skipping the branch in the interpreter, and the profile + * will never see that the branch is taken. This can lead to deopt loops or OSR + * failure. + */ + FixedNode deoptSuccessor = BeginNode.begin(deopt); + ValueNode ifNode = genIfNode(condition, negated ? deoptSuccessor : noDeoptSuccessor, negated ? noDeoptSuccessor : deoptSuccessor, negated ? 1 - probability : probability); + postProcessIfNode(ifNode); + append(ifNode); } - appendGoto(trueBlock); return; } @@ -3442,6 +3475,16 @@ public class BytecodeParser implements GraphBuilderContext { } } + public boolean isPotentialCountedLoopExit(LogicNode condition, BciBlock target) { + if (currentBlock != null) { + long exits = currentBlock.loops & ~target.loops; + if (exits != 0) { + return condition instanceof CompareNode; + } + } + return false; + } + /** * Hook for subclasses to decide whether the IfNode probability should be complemented during * conversion to Graal IR. @@ -3953,7 +3996,7 @@ public class BytecodeParser implements GraphBuilderContext { private String unresolvedMethodAssertionMessage(JavaMethod result) { String message = result.format("%H.%n(%P)%R"); - if (GraalServices.Java8OrEarlier) { + if (JavaVersionUtil.Java8OrEarlier) { JavaType declaringClass = result.getDeclaringClass(); String className = declaringClass.getName(); switch (className) { @@ -4214,13 +4257,10 @@ public class BytecodeParser implements GraphBuilderContext { return; } - ResolvedJavaType[] skippedExceptionTypes = this.graphBuilderConfig.getSkippedExceptionTypes(); - if (skippedExceptionTypes != null) { - for (ResolvedJavaType exceptionType : skippedExceptionTypes) { - if (exceptionType.isAssignableFrom(resolvedType)) { - append(new DeoptimizeNode(DeoptimizationAction.InvalidateRecompile, RuntimeConstraint)); - return; - } + for (ResolvedJavaType exceptionType : this.graphBuilderConfig.getSkippedExceptionTypes()) { + if (exceptionType.isAssignableFrom(resolvedType)) { + append(new DeoptimizeNode(DeoptimizationAction.InvalidateRecompile, RuntimeConstraint)); + return; } } @@ -4513,9 +4553,13 @@ public class BytecodeParser implements GraphBuilderContext { * Javac does not allow use of "$assertionsDisabled" for a field name but Eclipse does, in * which case a suffix is added to the generated field. */ - if ((parsingIntrinsic() || graphBuilderConfig.omitAssertions()) && resolvedField.isSynthetic() && resolvedField.getName().startsWith("$assertionsDisabled")) { - frameState.push(field.getJavaKind(), ConstantNode.forBoolean(true, graph)); - return; + if (resolvedField.isSynthetic() && resolvedField.getName().startsWith("$assertionsDisabled")) { + if (parsingIntrinsic()) { + throw new GraalError("Cannot use an assertion within the context of an intrinsic."); + } else if (graphBuilderConfig.omitAssertions()) { + frameState.push(field.getJavaKind(), ConstantNode.forBoolean(true, graph)); + return; + } } ResolvedJavaType holder = resolvedField.getDeclaringClass(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/optimize/IfNodeCanonicalizationsTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/optimize/IfNodeCanonicalizationsTest.java new file mode 100644 index 00000000000..d6b3bc9749e --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/optimize/IfNodeCanonicalizationsTest.java @@ -0,0 +1,366 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.jtt.optimize; + +import static org.junit.runners.Parameterized.Parameter; +import static org.junit.runners.Parameterized.Parameters; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.graalvm.compiler.jtt.JTTTest; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; +import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import jdk.vm.ci.meta.ResolvedJavaMethod; + +@RunWith(Parameterized.class) +public class IfNodeCanonicalizationsTest extends JTTTest { + + @Override + protected InlineInvokePlugin.InlineInfo bytecodeParserShouldInlineInvoke(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode[] args) { + if (method.getDeclaringClass().getUnqualifiedName().equals(IfNodeCanonicalizationsTest.class.getSimpleName()) && method.getName().startsWith("compare")) { + return InlineInvokePlugin.InlineInfo.createStandardInlineInfo(method); + } + return super.bytecodeParserShouldInlineInvoke(b, method, args); + } + + @Parameter(value = 0) public String testName; + @Parameter(value = 1) public int x; + @Parameter(value = 2) public int y; + + public static int compare0(int a, int b) { + return (a < b) ? 1 : ((a < b) ? 2 : 3); + } + + public static int compare1(int a, int b) { + return (a < b) ? ((a < b) ? 1 : 2) : 3; + } + + public static int compare2(int a, int b) { + return (a < b) ? 1 : ((a > b) ? 2 : 3); + } + + public static int compare3(int a, int b) { + return (a < b) ? ((a > b) ? 1 : 2) : 3; + } + + public static int compare4(int a, int b) { + return (a < b) ? 1 : ((a <= b) ? 2 : 3); + } + + public static int compare5(int a, int b) { + return (a < b) ? ((a <= b) ? 1 : 2) : 3; + } + + public static int compare6(int a, int b) { + return (a < b) ? 1 : ((a >= b) ? 2 : 3); + } + + public static int compare7(int a, int b) { + return (a < b) ? ((a >= b) ? 1 : 2) : 3; + } + + public static int compare8(int a, int b) { + return (a < b) ? 1 : ((a == b) ? 2 : 3); + } + + public static int compare9(int a, int b) { + return (a < b) ? ((a == b) ? 1 : 2) : 3; + } + + public static int compare10(int a, int b) { + return (a < b) ? 1 : ((a != b) ? 2 : 3); + } + + public static int compare11(int a, int b) { + return (a < b) ? ((a != b) ? 1 : 2) : 3; + } + + public static int compare12(int a, int b) { + return (a > b) ? 1 : ((a < b) ? 2 : 3); + } + + public static int compare13(int a, int b) { + return (a > b) ? ((a < b) ? 1 : 2) : 3; + } + + public static int compare14(int a, int b) { + return (a > b) ? 1 : ((a > b) ? 2 : 3); + } + + public static int compare15(int a, int b) { + return (a > b) ? ((a > b) ? 1 : 2) : 3; + } + + public static int compare16(int a, int b) { + return (a > b) ? 1 : ((a <= b) ? 2 : 3); + } + + public static int compare17(int a, int b) { + return (a > b) ? ((a <= b) ? 1 : 2) : 3; + } + + public static int compare18(int a, int b) { + return (a > b) ? 1 : ((a >= b) ? 2 : 3); + } + + public static int compare19(int a, int b) { + return (a > b) ? ((a >= b) ? 1 : 2) : 3; + } + + public static int compare20(int a, int b) { + return (a > b) ? 1 : ((a == b) ? 2 : 3); + } + + public static int compare21(int a, int b) { + return (a > b) ? ((a == b) ? 1 : 2) : 3; + } + + public static int compare22(int a, int b) { + return (a > b) ? 1 : ((a != b) ? 2 : 3); + } + + public static int compare23(int a, int b) { + return (a > b) ? ((a != b) ? 1 : 2) : 3; + } + + public static int compare24(int a, int b) { + return (a <= b) ? 1 : ((a < b) ? 2 : 3); + } + + public static int compare25(int a, int b) { + return (a <= b) ? ((a < b) ? 1 : 2) : 3; + } + + public static int compare26(int a, int b) { + return (a <= b) ? 1 : ((a > b) ? 2 : 3); + } + + public static int compare27(int a, int b) { + return (a <= b) ? ((a > b) ? 1 : 2) : 3; + } + + public static int compare28(int a, int b) { + return (a <= b) ? 1 : ((a <= b) ? 2 : 3); + } + + public static int compare29(int a, int b) { + return (a <= b) ? ((a <= b) ? 1 : 2) : 3; + } + + public static int compare30(int a, int b) { + return (a <= b) ? 1 : ((a >= b) ? 2 : 3); + } + + public static int compare31(int a, int b) { + return (a <= b) ? ((a >= b) ? 1 : 2) : 3; + } + + public static int compare32(int a, int b) { + return (a <= b) ? 1 : ((a == b) ? 2 : 3); + } + + public static int compare33(int a, int b) { + return (a <= b) ? ((a == b) ? 1 : 2) : 3; + } + + public static int compare34(int a, int b) { + return (a <= b) ? 1 : ((a != b) ? 2 : 3); + } + + public static int compare35(int a, int b) { + return (a <= b) ? ((a != b) ? 1 : 2) : 3; + } + + public static int compare36(int a, int b) { + return (a >= b) ? 1 : ((a < b) ? 2 : 3); + } + + public static int compare37(int a, int b) { + return (a >= b) ? ((a < b) ? 1 : 2) : 3; + } + + public static int compare38(int a, int b) { + return (a >= b) ? 1 : ((a > b) ? 2 : 3); + } + + public static int compare39(int a, int b) { + return (a >= b) ? ((a > b) ? 1 : 2) : 3; + } + + public static int compare40(int a, int b) { + return (a >= b) ? 1 : ((a <= b) ? 2 : 3); + } + + public static int compare41(int a, int b) { + return (a >= b) ? ((a <= b) ? 1 : 2) : 3; + } + + public static int compare42(int a, int b) { + return (a >= b) ? 1 : ((a >= b) ? 2 : 3); + } + + public static int compare43(int a, int b) { + return (a >= b) ? ((a >= b) ? 1 : 2) : 3; + } + + public static int compare44(int a, int b) { + return (a >= b) ? 1 : ((a == b) ? 2 : 3); + } + + public static int compare45(int a, int b) { + return (a >= b) ? ((a == b) ? 1 : 2) : 3; + } + + public static int compare46(int a, int b) { + return (a >= b) ? 1 : ((a != b) ? 2 : 3); + } + + public static int compare47(int a, int b) { + return (a >= b) ? ((a != b) ? 1 : 2) : 3; + } + + public static int compare48(int a, int b) { + return (a == b) ? 1 : ((a < b) ? 2 : 3); + } + + public static int compare49(int a, int b) { + return (a == b) ? ((a < b) ? 1 : 2) : 3; + } + + public static int compare50(int a, int b) { + return (a == b) ? 1 : ((a > b) ? 2 : 3); + } + + public static int compare51(int a, int b) { + return (a == b) ? ((a > b) ? 1 : 2) : 3; + } + + public static int compare52(int a, int b) { + return (a == b) ? 1 : ((a <= b) ? 2 : 3); + } + + public static int compare53(int a, int b) { + return (a == b) ? ((a <= b) ? 1 : 2) : 3; + } + + public static int compare54(int a, int b) { + return (a == b) ? 1 : ((a >= b) ? 2 : 3); + } + + public static int compare55(int a, int b) { + return (a == b) ? ((a >= b) ? 1 : 2) : 3; + } + + public static int compare56(int a, int b) { + return (a == b) ? 1 : ((a == b) ? 2 : 3); + } + + public static int compare57(int a, int b) { + return (a == b) ? ((a == b) ? 1 : 2) : 3; + } + + public static int compare58(int a, int b) { + return (a == b) ? 1 : ((a != b) ? 2 : 3); + } + + public static int compare59(int a, int b) { + return (a == b) ? ((a != b) ? 1 : 2) : 3; + } + + public static int compare60(int a, int b) { + return (a != b) ? 1 : ((a < b) ? 2 : 3); + } + + public static int compare61(int a, int b) { + return (a != b) ? ((a < b) ? 1 : 2) : 3; + } + + public static int compare62(int a, int b) { + return (a != b) ? 1 : ((a > b) ? 2 : 3); + } + + public static int compare63(int a, int b) { + return (a != b) ? ((a > b) ? 1 : 2) : 3; + } + + public static int compare64(int a, int b) { + return (a != b) ? 1 : ((a <= b) ? 2 : 3); + } + + public static int compare65(int a, int b) { + return (a != b) ? ((a <= b) ? 1 : 2) : 3; + } + + public static int compare66(int a, int b) { + return (a != b) ? 1 : ((a >= b) ? 2 : 3); + } + + public static int compare67(int a, int b) { + return (a != b) ? ((a >= b) ? 1 : 2) : 3; + } + + public static int compare68(int a, int b) { + return (a != b) ? 1 : ((a == b) ? 2 : 3); + } + + public static int compare69(int a, int b) { + return (a != b) ? ((a == b) ? 1 : 2) : 3; + } + + public static int compare70(int a, int b) { + return (a != b) ? 1 : ((a != b) ? 2 : 3); + } + + public static int compare71(int a, int b) { + return (a != b) ? ((a != b) ? 1 : 2) : 3; + } + + @Test + public void runNamedTest() { + runTest(testName, x, y); + } + + @Parameters(name = "{0}(a = {1}, b = {2})") + public static Collection data() { + List tests = new ArrayList<>(); + for (Method m : IfNodeCanonicalizationsTest.class.getDeclaredMethods()) { + if (m.getName().startsWith("compare") && Modifier.isStatic(m.getModifiers())) { + tests.add(new Object[]{m.getName(), 0, 0}); + tests.add(new Object[]{m.getName(), 0, 1}); + tests.add(new Object[]{m.getName(), 1, 0}); + } + } + return tests; + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/optimize/SwitchHashTableTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/optimize/SwitchHashTableTest.java new file mode 100644 index 00000000000..d1fdf62284a --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.jtt/src/org/graalvm/compiler/jtt/optimize/SwitchHashTableTest.java @@ -0,0 +1,635 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.jtt.optimize; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.stream.Collectors; + +import org.graalvm.compiler.jtt.JTTTest; +import org.graalvm.compiler.lir.hashing.HashFunction; +import org.graalvm.compiler.lir.hashing.Hasher; +import org.junit.Test; + +import jdk.vm.ci.meta.JavaConstant; + +/* + * Tests optimization of hash table switches. + * Code generated by `SwitchHashTableTest.TestGenerator.main` + */ +public class SwitchHashTableTest extends JTTTest { + @Test + public void checkHashFunctionInstances() { + List coveredByTestCases = Arrays.asList("val >> min", "val", "val >> (val & min)", "(val >> min) ^ val", "val - min", "rotateRight(val, prime)", "rotateRight(val, prime) ^ val", + "rotateRight(val, prime) + val", "(val >> min) * val", "(val * prime) >> min"); + Set functions = HashFunction.instances().stream().map(Object::toString).collect(Collectors.toSet()); + functions.removeAll(coveredByTestCases); + assertTrue("The following hash functions are not covered by the `Switch03` test: " + functions + + ". Re-run the `Switch03.TestGenerator.main` and update the test class.", functions.isEmpty()); + } + + // Hasher[function=rotateRight(val, prime), effort=4, cardinality=16] + public static int test1(int arg) { + switch (arg) { + case 3080012: + return 3080012; + case 3080017: + return 3080017; + case 3080029: + return 3080029; + case 3080037: + return 3080037; + case 3080040: + return 3080040; + case 3080054: + return 3080054; + case 3080060: + return 3080060; + case 3080065: + return 3080065; + case 3080073: + return 3080073; + case 3080082: + return 3080082; + case 3080095: + return 3080095; + case 3080103: + return 3080103; + case 3080116: + return 3080116; + case 3080127: + return 3080127; + case 3080130: + return 3080130; + default: + return -1; + } + } + + @Test + public void run1() throws Throwable { + runTest("test1", 0); // zero + runTest("test1", 3080011); // bellow + runTest("test1", 3080012); // first + runTest("test1", 3080065); // middle + runTest("test1", 3080130); // last + runTest("test1", 3080131); // above + runTest("test1", 3080013); // miss + } + + // Hasher[function=rotateRight(val, prime) ^ val, effort=5, cardinality=28] + public static int test2(int arg) { + switch (arg) { + case 718707335: + return 718707335; + case 718707336: + return 718707336; + case 718707347: + return 718707347; + case 718707359: + return 718707359; + case 718707366: + return 718707366; + case 718707375: + return 718707375; + case 718707378: + return 718707378; + case 718707386: + return 718707386; + case 718707396: + return 718707396; + case 718707401: + return 718707401; + case 718707408: + return 718707408; + case 718707409: + return 718707409; + case 718707420: + return 718707420; + case 718707431: + return 718707431; + case 718707436: + return 718707436; + default: + return -1; + } + } + + @Test + public void run2() throws Throwable { + runTest("test2", 0); // zero + runTest("test2", 718707334); // bellow + runTest("test2", 718707335); // first + runTest("test2", 718707386); // middle + runTest("test2", 718707436); // last + runTest("test2", 718707437); // above + runTest("test2", 718707337); // miss + } + + // Hasher[function=(val * prime) >> min, effort=4, cardinality=16] + public static int test3(int arg) { + switch (arg) { + case 880488712: + return 880488712; + case 880488723: + return 880488723; + case 880488737: + return 880488737; + case 880488744: + return 880488744; + case 880488752: + return 880488752; + case 880488757: + return 880488757; + case 880488767: + return 880488767; + case 880488777: + return 880488777; + case 880488781: + return 880488781; + case 880488794: + return 880488794; + case 880488795: + return 880488795; + case 880488807: + return 880488807; + case 880488814: + return 880488814; + case 880488821: + return 880488821; + case 880488831: + return 880488831; + default: + return -1; + } + } + + @Test + public void run3() throws Throwable { + runTest("test3", 0); // zero + runTest("test3", 880488711); // bellow + runTest("test3", 880488712); // first + runTest("test3", 880488777); // middle + runTest("test3", 880488831); // last + runTest("test3", 880488832); // above + runTest("test3", 880488713); // miss + } + + // Hasher[function=rotateRight(val, prime) + val, effort=5, cardinality=28] + public static int test4(int arg) { + switch (arg) { + case 189404658: + return 189404658; + case 189404671: + return 189404671; + case 189404678: + return 189404678; + case 189404680: + return 189404680; + case 189404687: + return 189404687; + case 189404698: + return 189404698; + case 189404699: + return 189404699; + case 189404711: + return 189404711; + case 189404724: + return 189404724; + case 189404725: + return 189404725; + case 189404732: + return 189404732; + case 189404739: + return 189404739; + case 189404748: + return 189404748; + case 189404754: + return 189404754; + case 189404765: + return 189404765; + default: + return -1; + } + } + + @Test + public void run4() throws Throwable { + runTest("test4", 0); // zero + runTest("test4", 189404657); // bellow + runTest("test4", 189404658); // first + runTest("test4", 189404711); // middle + runTest("test4", 189404765); // last + runTest("test4", 189404766); // above + runTest("test4", 189404659); // miss + } + + // Hasher[function=val - min, effort=2, cardinality=24] + public static int test5(int arg) { + switch (arg) { + case 527674226: + return 527674226; + case 527674235: + return 527674235; + case 527674236: + return 527674236; + case 527674247: + return 527674247; + case 527674251: + return 527674251; + case 527674253: + return 527674253; + case 527674257: + return 527674257; + case 527674263: + return 527674263; + case 527674265: + return 527674265; + case 527674272: + return 527674272; + case 527674286: + return 527674286; + case 527674293: + return 527674293; + case 527674294: + return 527674294; + case 527674306: + return 527674306; + case 527674308: + return 527674308; + default: + return -1; + } + } + + @Test + public void run5() throws Throwable { + runTest("test5", 0); // zero + runTest("test5", 527674225); // bellow + runTest("test5", 527674226); // first + runTest("test5", 527674263); // middle + runTest("test5", 527674308); // last + runTest("test5", 527674309); // above + runTest("test5", 527674227); // miss + } + + // Hasher[function=val, effort=1, cardinality=24] + public static int test6(int arg) { + switch (arg) { + case 676979121: + return 676979121; + case 676979128: + return 676979128; + case 676979135: + return 676979135; + case 676979146: + return 676979146; + case 676979148: + return 676979148; + case 676979156: + return 676979156; + case 676979158: + return 676979158; + case 676979169: + return 676979169; + case 676979175: + return 676979175; + case 676979179: + return 676979179; + case 676979182: + return 676979182; + case 676979194: + return 676979194; + case 676979200: + return 676979200; + case 676979205: + return 676979205; + case 676979219: + return 676979219; + default: + return -1; + } + } + + @Test + public void run6() throws Throwable { + runTest("test6", 0); // zero + runTest("test6", 676979120); // bellow + runTest("test6", 676979121); // first + runTest("test6", 676979169); // middle + runTest("test6", 676979219); // last + runTest("test6", 676979220); // above + runTest("test6", 676979122); // miss + } + + // Hasher[function=(val >> min) ^ val, effort=3, cardinality=16] + public static int test7(int arg) { + switch (arg) { + case 634218696: + return 634218696; + case 634218710: + return 634218710; + case 634218715: + return 634218715; + case 634218720: + return 634218720; + case 634218724: + return 634218724; + case 634218732: + return 634218732; + case 634218737: + return 634218737; + case 634218749: + return 634218749; + case 634218751: + return 634218751; + case 634218758: + return 634218758; + case 634218767: + return 634218767; + case 634218772: + return 634218772; + case 634218786: + return 634218786; + case 634218792: + return 634218792; + case 634218795: + return 634218795; + default: + return -1; + } + } + + @Test + public void run7() throws Throwable { + runTest("test7", 0); // zero + runTest("test7", 634218695); // bellow + runTest("test7", 634218696); // first + runTest("test7", 634218749); // middle + runTest("test7", 634218795); // last + runTest("test7", 634218796); // above + runTest("test7", 634218697); // miss + } + + // Hasher[function=val >> min, effort=2, cardinality=16] + public static int test8(int arg) { + switch (arg) { + case 473982403: + return 473982403; + case 473982413: + return 473982413; + case 473982416: + return 473982416; + case 473982425: + return 473982425; + case 473982439: + return 473982439; + case 473982445: + return 473982445; + case 473982459: + return 473982459; + case 473982468: + return 473982468; + case 473982479: + return 473982479; + case 473982482: + return 473982482; + case 473982494: + return 473982494; + case 473982501: + return 473982501; + case 473982505: + return 473982505; + case 473982519: + return 473982519; + case 473982523: + return 473982523; + default: + return -1; + } + } + + @Test + public void run8() throws Throwable { + runTest("test8", 0); // zero + runTest("test8", 473982402); // bellow + runTest("test8", 473982403); // first + runTest("test8", 473982468); // middle + runTest("test8", 473982523); // last + runTest("test8", 473982524); // above + runTest("test8", 473982404); // miss + } + + // Hasher[function=val >> (val & min), effort=3, cardinality=16] + public static int test9(int arg) { + switch (arg) { + case 15745090: + return 15745090; + case 15745093: + return 15745093; + case 15745102: + return 15745102; + case 15745108: + return 15745108; + case 15745122: + return 15745122; + case 15745131: + return 15745131; + case 15745132: + return 15745132; + case 15745146: + return 15745146; + case 15745151: + return 15745151; + case 15745163: + return 15745163; + case 15745169: + return 15745169; + case 15745182: + return 15745182; + case 15745191: + return 15745191; + case 15745198: + return 15745198; + case 15745207: + return 15745207; + default: + return -1; + } + } + + @Test + public void run9() throws Throwable { + runTest("test9", 0); // zero + runTest("test9", 15745089); // bellow + runTest("test9", 15745090); // first + runTest("test9", 15745146); // middle + runTest("test9", 15745207); // last + runTest("test9", 15745208); // above + runTest("test9", 15745091); // miss + } + + // Hasher[function=(val >> min) * val, effort=4, cardinality=28] + public static int test10(int arg) { + switch (arg) { + case 989358996: + return 989358996; + case 989359010: + return 989359010; + case 989359022: + return 989359022; + case 989359030: + return 989359030; + case 989359038: + return 989359038; + case 989359047: + return 989359047; + case 989359053: + return 989359053; + case 989359059: + return 989359059; + case 989359061: + return 989359061; + case 989359072: + return 989359072; + case 989359073: + return 989359073; + case 989359087: + return 989359087; + case 989359097: + return 989359097; + case 989359099: + return 989359099; + case 989359108: + return 989359108; + default: + return -1; + } + } + + @Test + public void run10() throws Throwable { + runTest("test10", 0); // zero + runTest("test10", 989358995); // bellow + runTest("test10", 989358996); // first + runTest("test10", 989359059); // middle + runTest("test10", 989359108); // last + runTest("test10", 989359109); // above + runTest("test10", 989358997); // miss + } + + public static class TestGenerator { + + private static int nextId = 0; + private static final int size = 15; + private static double minDensity = 0.5; + + // test code generator + public static void main(String[] args) { + + Random r = new Random(0); + Set seen = new HashSet<>(); + Set all = HashFunction.instances().stream().map(Object::toString).collect(Collectors.toSet()); + + println("@Test"); + println("public void checkHashFunctionInstances() {"); + println(" List coveredByTestCases = Arrays.asList(" + String.join(", ", all.stream().map(s -> "\"" + s + "\"").collect(Collectors.toSet())) + ");"); + println(" Set functions = HashFunction.instances().stream().map(Object::toString).collect(Collectors.toSet());"); + println(" functions.removeAll(coveredByTestCases);"); + println(" assertTrue(\"The following hash functions are not covered by the `Switch03` test: \" + functions +"); + println(" \". Re-run the `Switch03.TestGenerator.main` and update the test class.\", functions.isEmpty());"); + println("}"); + + while (seen.size() < all.size()) { + int v = r.nextInt(Integer.MAX_VALUE / 2); + List keys = new ArrayList<>(); + while (keys.size() < 15) { + keys.add(v); + v += r.nextInt(15); + } + keys.sort(Integer::compare); + double density = ((double) keys.size() + 1) / (keys.get(keys.size() - 1) - keys.get(0)); + if (density < minDensity) { + Hasher.forKeys(toConstants(keys), minDensity).ifPresent(h -> { + String f = h.function().toString(); + if (!seen.contains(f)) { + gen(keys, h); + seen.add(f); + } + }); + } + } + } + + private static void gen(List keys, Hasher hasher) { + int id = ++nextId; + + println("// " + hasher + ""); + println("public static int test" + id + "(int arg) {"); + println(" switch (arg) {"); + + for (Integer key : keys) { + println(" case " + key + ": return " + key + ";"); + } + + println(" default: return -1;"); + println(" }"); + println("}"); + + int miss = keys.get(0) + 1; + while (keys.contains(miss)) { + miss++; + } + + println("@Test"); + println("public void run" + id + "() throws Throwable {"); + println(" runTest(\"test" + id + "\", 0); // zero "); + println(" runTest(\"test" + id + "\", " + (keys.get(0) - 1) + "); // bellow "); + println(" runTest(\"test" + id + "\", " + keys.get(0) + "); // first "); + println(" runTest(\"test" + id + "\", " + keys.get(size / 2) + "); // middle "); + println(" runTest(\"test" + id + "\", " + keys.get(size - 1) + "); // last "); + println(" runTest(\"test" + id + "\", " + (keys.get(size - 1) + 1) + "); // above "); + println(" runTest(\"test" + id + "\", " + miss + "); // miss "); + println("}"); + } + + private static void println(String s) { + System.out.println(s); + } + + private static JavaConstant[] toConstants(List keys) { + JavaConstant[] ckeys = new JavaConstant[keys.size()]; + + for (int i = 0; i < keys.size(); i++) { + ckeys[i] = JavaConstant.forInt(keys.get(i)); + } + return ckeys; + } + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64Call.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64Call.java index 70d6741d57e..043adec714e 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64Call.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64Call.java @@ -40,6 +40,7 @@ import org.graalvm.compiler.core.common.spi.ForeignCallLinkage; import org.graalvm.compiler.lir.LIRFrameState; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.Opcode; +import org.graalvm.compiler.lir.StandardOp.LabelHoldingOp; import org.graalvm.compiler.lir.asm.CompilationResultBuilder; import jdk.vm.ci.code.Register; @@ -126,7 +127,7 @@ public class AArch64Call { } } - public abstract static class ForeignCallOp extends CallOp { + public abstract static class ForeignCallOp extends CallOp implements LabelHoldingOp { protected final ForeignCallLinkage callTarget; protected final Label label; @@ -147,6 +148,11 @@ public class AArch64Call { } protected abstract void emitCall(CompilationResultBuilder crb, AArch64MacroAssembler masm); + + @Override + public Label getLabel() { + return label; + } } @Opcode("NEAR_FOREIGN_CALL") diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ControlFlow.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ControlFlow.java index 680622dac57..31ff3aa3f1e 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ControlFlow.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64ControlFlow.java @@ -31,6 +31,7 @@ import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; import java.util.function.Function; +import jdk.vm.ci.meta.AllocatableValue; import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.core.common.NumUtil; import org.graalvm.compiler.asm.aarch64.AArch64Assembler; @@ -59,53 +60,21 @@ import jdk.vm.ci.meta.Value; public class AArch64ControlFlow { - /** - * Compares integer register to 0 and branches if condition is true. Condition may only be equal - * or non-equal. - */ - // TODO (das) where do we need this? - // public static class CompareAndBranchOp extends AArch64LIRInstruction implements - // StandardOp.BranchOp { - // private final ConditionFlag condition; - // private final LabelRef destination; - // @Use({REG}) private Value x; - // - // public CompareAndBranchOp(Condition condition, LabelRef destination, Value x) { - // assert condition == Condition.EQ || condition == Condition.NE; - // assert ARMv8.isGpKind(x.getKind()); - // this.condition = condition == Condition.EQ ? ConditionFlag.EQ : ConditionFlag.NE; - // this.destination = destination; - // this.x = x; - // } - // - // @Override - // public void emitCode(CompilationResultBuilder crb, ARMv8MacroAssembler masm) { - // int size = ARMv8.bitsize(x.getKind()); - // if (condition == ConditionFlag.EQ) { - // masm.cbz(size, asRegister(x), destination.label()); - // } else { - // masm.cbnz(size, asRegister(x), destination.label()); - // } - // } - // } - - public static class BranchOp extends AArch64BlockEndOp implements StandardOp.BranchOp { - public static final LIRInstructionClass TYPE = LIRInstructionClass.create(BranchOp.class); - - private final AArch64Assembler.ConditionFlag condition; + public abstract static class AbstractBranchOp extends AArch64BlockEndOp implements StandardOp.BranchOp { private final LabelRef trueDestination; private final LabelRef falseDestination; private final double trueDestinationProbability; - public BranchOp(AArch64Assembler.ConditionFlag condition, LabelRef trueDestination, LabelRef falseDestination, double trueDestinationProbability) { - super(TYPE); - this.condition = condition; + private AbstractBranchOp(LIRInstructionClass c, LabelRef trueDestination, LabelRef falseDestination, double trueDestinationProbability) { + super(c); this.trueDestination = trueDestination; this.falseDestination = falseDestination; this.trueDestinationProbability = trueDestinationProbability; } + protected abstract void emitBranch(CompilationResultBuilder crb, AArch64MacroAssembler masm, LabelRef target, boolean negate); + @Override public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { /* @@ -115,18 +84,104 @@ public class AArch64ControlFlow { * executing two instructions instead of one. */ if (crb.isSuccessorEdge(trueDestination)) { - masm.branchConditionally(condition.negate(), falseDestination.label()); + emitBranch(crb, masm, falseDestination, true); } else if (crb.isSuccessorEdge(falseDestination)) { - masm.branchConditionally(condition, trueDestination.label()); + emitBranch(crb, masm, trueDestination, false); } else if (trueDestinationProbability < 0.5) { - masm.branchConditionally(condition.negate(), falseDestination.label()); + emitBranch(crb, masm, falseDestination, true); masm.jmp(trueDestination.label()); } else { - masm.branchConditionally(condition, trueDestination.label()); + emitBranch(crb, masm, trueDestination, false); masm.jmp(falseDestination.label()); } } + } + public static class BranchOp extends AbstractBranchOp implements StandardOp.BranchOp { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(BranchOp.class); + + private final AArch64Assembler.ConditionFlag condition; + + public BranchOp(AArch64Assembler.ConditionFlag condition, LabelRef trueDestination, LabelRef falseDestination, double trueDestinationProbability) { + super(TYPE, trueDestination, falseDestination, trueDestinationProbability); + this.condition = condition; + } + + @Override + protected void emitBranch(CompilationResultBuilder crb, AArch64MacroAssembler masm, LabelRef target, boolean negate) { + AArch64Assembler.ConditionFlag finalCond = negate ? condition.negate() : condition; + masm.branchConditionally(finalCond, target.label()); + } + } + + public static class CompareBranchZeroOp extends AbstractBranchOp implements StandardOp.BranchOp { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(CompareBranchZeroOp.class); + + @Use(REG) private AllocatableValue value; + + public CompareBranchZeroOp(AllocatableValue value, LabelRef trueDestination, LabelRef falseDestination, double trueDestinationProbability) { + super(TYPE, trueDestination, falseDestination, trueDestinationProbability); + this.value = value; + } + + @Override + protected void emitBranch(CompilationResultBuilder crb, AArch64MacroAssembler masm, LabelRef target, boolean negate) { + AArch64Kind kind = (AArch64Kind) this.value.getPlatformKind(); + assert kind.isInteger(); + int size = kind.getSizeInBytes() * Byte.SIZE; + + if (negate) { + masm.cbnz(size, asRegister(this.value), target.label()); + } else { + masm.cbz(size, asRegister(this.value), target.label()); + } + } + } + + public static class BitTestAndBranchOp extends AbstractBranchOp implements StandardOp.BranchOp { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(BitTestAndBranchOp.class); + + @Use protected AllocatableValue value; + private final int index; + + public BitTestAndBranchOp(LabelRef trueDestination, LabelRef falseDestination, AllocatableValue value, double trueDestinationProbability, int index) { + super(TYPE, trueDestination, falseDestination, trueDestinationProbability); + this.value = value; + this.index = index; + } + + @Override + protected void emitBranch(CompilationResultBuilder crb, AArch64MacroAssembler masm, LabelRef target, boolean negate) { + ConditionFlag cond = negate ? ConditionFlag.NE : ConditionFlag.EQ; + Label label = target.label(); + boolean isFarBranch; + + if (label.isBound()) { + isFarBranch = NumUtil.isSignedNbit(18, masm.position() - label.position()); + } else { + // Max range of tbz is +-2^13 instructions. We estimate that each LIR instruction + // emits 2 AArch64 instructions on average. Thus we test for maximum 2^12 LIR + // instruction offset. + int maxLIRDistance = (1 << 12); + isFarBranch = !crb.labelWithinRange(this, label, maxLIRDistance); + } + + if (isFarBranch) { + cond = cond.negate(); + label = new Label(); + } + + if (cond == ConditionFlag.EQ) { + masm.tbz(asRegister(value), index, label); + } else { + masm.tbnz(asRegister(value), index, label); + } + + if (isFarBranch) { + masm.jmp(target.label()); + masm.bind(label); + } + } } @Opcode("CMOVE") diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64Move.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64Move.java index efcba926f06..9474b028a3c 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64Move.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.aarch64/src/org/graalvm/compiler/lir/aarch64/AArch64Move.java @@ -24,13 +24,6 @@ package org.graalvm.compiler.lir.aarch64; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.HINT; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.UNINITIALIZED; -import static org.graalvm.compiler.lir.LIRValueUtil.asJavaConstant; -import static org.graalvm.compiler.lir.LIRValueUtil.isJavaConstant; import static jdk.vm.ci.aarch64.AArch64.sp; import static jdk.vm.ci.aarch64.AArch64.zr; import static jdk.vm.ci.code.ValueUtil.asAllocatableValue; @@ -38,6 +31,13 @@ import static jdk.vm.ci.code.ValueUtil.asRegister; import static jdk.vm.ci.code.ValueUtil.asStackSlot; import static jdk.vm.ci.code.ValueUtil.isRegister; import static jdk.vm.ci.code.ValueUtil.isStackSlot; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.HINT; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK; +import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.UNINITIALIZED; +import static org.graalvm.compiler.lir.LIRValueUtil.asJavaConstant; +import static org.graalvm.compiler.lir.LIRValueUtil.isJavaConstant; import org.graalvm.compiler.asm.aarch64.AArch64Address; import org.graalvm.compiler.asm.aarch64.AArch64MacroAssembler; @@ -56,6 +56,7 @@ import org.graalvm.compiler.lir.VirtualStackSlot; import org.graalvm.compiler.lir.asm.CompilationResultBuilder; import jdk.vm.ci.aarch64.AArch64Kind; +import jdk.vm.ci.code.MemoryBarriers; import jdk.vm.ci.code.Register; import jdk.vm.ci.code.StackSlot; import jdk.vm.ci.meta.AllocatableValue; @@ -212,14 +213,20 @@ public class AArch64Move { // a compiler bug which warns that crb is unused, and also // warns that @SuppressWarnings("unused") is unnecessary. public void emitCode(@SuppressWarnings("all") CompilationResultBuilder crb, AArch64MacroAssembler masm) { - // As I understand it load acquire/store release have the same semantics as on IA64 - // and allow us to handle LoadStore, LoadLoad and StoreStore without an explicit - // barrier. - // But Graal support to figure out if a load/store is volatile is non-existant so for - // now just use memory barriers everywhere. - // if ((barrier & MemoryBarriers.STORE_LOAD) != 0) { - masm.dmb(AArch64MacroAssembler.BarrierKind.ANY_ANY); - // } + assert barriers >= MemoryBarriers.LOAD_LOAD && barriers <= (MemoryBarriers.STORE_STORE | MemoryBarriers.STORE_LOAD | MemoryBarriers.LOAD_STORE | MemoryBarriers.LOAD_LOAD); + switch (barriers) { + case MemoryBarriers.STORE_STORE: + masm.dmb(AArch64MacroAssembler.BarrierKind.STORE_STORE); + break; + case MemoryBarriers.LOAD_LOAD: + case MemoryBarriers.LOAD_STORE: + case MemoryBarriers.LOAD_LOAD | MemoryBarriers.LOAD_STORE: + masm.dmb(AArch64MacroAssembler.BarrierKind.LOAD_LOAD); + break; + default: + masm.dmb(AArch64MacroAssembler.BarrierKind.ANY_ANY); + break; + } } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayEqualsOp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayEqualsOp.java index c2f16af9164..fc935fef4e4 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayEqualsOp.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ArrayEqualsOp.java @@ -41,7 +41,7 @@ import org.graalvm.compiler.asm.amd64.AMD64BaseAssembler.OperandSize; import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; import org.graalvm.compiler.asm.amd64.AVXKind; import org.graalvm.compiler.core.common.LIRKind; -import org.graalvm.compiler.core.common.NumUtil; +import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.Opcode; import org.graalvm.compiler.lir.asm.CompilationResultBuilder; @@ -51,18 +51,29 @@ import static jdk.vm.ci.code.ValueUtil.asRegister; import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.ILLEGAL; import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; +import java.util.Objects; + /** * Emits code which compares two arrays of the same length. If the CPU supports any vector * instructions specialized code is emitted to leverage these instructions. + * + * This op can also compare arrays of different integer types (e.g. {@code byte[]} and + * {@code char[]}) with on-the-fly sign- or zero-extension. If one of the given arrays is a + * {@code char[]} array, the smaller elements are zero-extended, otherwise they are sign-extended. */ @Opcode("ARRAY_EQUALS") public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64ArrayEqualsOp.class); - private final JavaKind kind; - private final int arrayBaseOffset; - private final int arrayIndexScale; - private final int constantByteLength; + private final JavaKind kind1; + private final JavaKind kind2; + private final int arrayBaseOffset1; + private final int arrayBaseOffset2; + private final Scale arrayIndexScale1; + private final Scale arrayIndexScale2; + private final AVXKind.AVXSize vectorSize; + private final int constantLength; + private final boolean signExtend; @Def({REG}) private Value resultValue; @Alive({REG}) private Value array1Value; @@ -81,20 +92,21 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { @Temp({REG, ILLEGAL}) private Value vectorTemp3; @Temp({REG, ILLEGAL}) private Value vectorTemp4; - public AMD64ArrayEqualsOp(LIRGeneratorTool tool, JavaKind kind, Value result, Value array1, Value array2, Value length, + public AMD64ArrayEqualsOp(LIRGeneratorTool tool, JavaKind kind1, JavaKind kind2, Value result, Value array1, Value array2, Value length, int constantLength, boolean directPointers, int maxVectorSize) { super(TYPE); - this.kind = kind; + this.kind1 = kind1; + this.kind2 = kind2; + this.signExtend = kind1 != JavaKind.Char && kind2 != JavaKind.Char; - this.arrayBaseOffset = directPointers ? 0 : tool.getProviders().getMetaAccess().getArrayBaseOffset(kind); - this.arrayIndexScale = tool.getProviders().getMetaAccess().getArrayIndexScale(kind); + assert kind1.isNumericInteger() && kind2.isNumericInteger() || kind1 == kind2; - if (constantLength >= 0 && arrayIndexScale > 1) { - // scale length - this.constantByteLength = constantLength << NumUtil.log2Ceil(arrayIndexScale); - } else { - this.constantByteLength = constantLength; - } + this.arrayBaseOffset1 = directPointers ? 0 : tool.getProviders().getMetaAccess().getArrayBaseOffset(kind1); + this.arrayBaseOffset2 = directPointers ? 0 : tool.getProviders().getMetaAccess().getArrayBaseOffset(kind2); + this.arrayIndexScale1 = Objects.requireNonNull(Scale.fromInt(tool.getProviders().getMetaAccess().getArrayIndexScale(kind1))); + this.arrayIndexScale2 = Objects.requireNonNull(Scale.fromInt(tool.getProviders().getMetaAccess().getArrayIndexScale(kind2))); + this.vectorSize = ((AMD64) tool.target().arch).getFeatures().contains(CPUFeature.AVX2) && (maxVectorSize < 0 || maxVectorSize >= 32) ? AVXKind.AVXSize.YMM : AVXKind.AVXSize.XMM; + this.constantLength = constantLength; this.resultValue = result; this.array1Value = array1; @@ -107,10 +119,10 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { this.temp3 = tool.newVariable(LIRKind.value(tool.target().arch.getWordKind())); this.temp4 = tool.newVariable(LIRKind.value(tool.target().arch.getWordKind())); - this.temp5 = kind.isNumericFloat() ? tool.newVariable(LIRKind.value(tool.target().arch.getWordKind())) : Value.ILLEGAL; - if (kind == JavaKind.Float) { + this.temp5 = kind1.isNumericFloat() || kind1 != kind2 ? tool.newVariable(LIRKind.value(tool.target().arch.getWordKind())) : Value.ILLEGAL; + if (kind1 == JavaKind.Float) { this.tempXMM = tool.newVariable(LIRKind.value(AMD64Kind.SINGLE)); - } else if (kind == JavaKind.Double) { + } else if (kind1 == JavaKind.Double) { this.tempXMM = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); } else { this.tempXMM = Value.ILLEGAL; @@ -119,7 +131,7 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { // We only need the vector temporaries if we generate SSE code. if (supportsSSE41(tool.target())) { if (canGenerateConstantLengthCompare(tool.target())) { - LIRKind lirKind = LIRKind.value(supportsAVX2(tool.target()) && (maxVectorSize < 0 || maxVectorSize >= 32) ? AMD64Kind.V256_BYTE : AMD64Kind.V128_BYTE); + LIRKind lirKind = LIRKind.value(vectorSize == AVXKind.AVXSize.YMM ? AMD64Kind.V256_BYTE : AMD64Kind.V128_BYTE); this.vectorTemp1 = tool.newVariable(lirKind); this.vectorTemp2 = tool.newVariable(lirKind); this.vectorTemp3 = tool.newVariable(lirKind); @@ -139,7 +151,7 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { } private boolean canGenerateConstantLengthCompare(TargetDescription target) { - return constantByteLength >= 0 && kind.isNumericInteger() && supportsSSE41(target); + return constantLength >= 0 && kind1.isNumericInteger() && (kind1 == kind2 || getElementsPerVector(AVXKind.AVXSize.XMM) <= constantLength) && supportsSSE41(target); } @Override @@ -153,26 +165,19 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { Label done = new Label(); // Load array base addresses. - masm.leaq(array1, new AMD64Address(asRegister(array1Value), arrayBaseOffset)); - masm.leaq(array2, new AMD64Address(asRegister(array2Value), arrayBaseOffset)); + masm.leaq(array1, new AMD64Address(asRegister(array1Value), arrayBaseOffset1)); + masm.leaq(array2, new AMD64Address(asRegister(array2Value), arrayBaseOffset2)); if (canGenerateConstantLengthCompare(crb.target)) { - emitConstantLengthArrayCompareBytes(masm, array1, array2, asRegister(temp3), asRegister(temp4), - new Register[]{asRegister(vectorTemp1), asRegister(vectorTemp2), asRegister(vectorTemp3), asRegister(vectorTemp4)}, - falseLabel, constantByteLength, AVXKind.getRegisterSize(vectorTemp1).getBytes()); + emitConstantLengthArrayCompareBytes(crb, masm, array1, array2, asRegister(temp3), asRegister(temp4), + new Register[]{asRegister(vectorTemp1), asRegister(vectorTemp2), asRegister(vectorTemp3), asRegister(vectorTemp4)}, falseLabel); } else { Register length = asRegister(temp3); - - // Get array length in bytes. + // Get array length. masm.movl(length, asRegister(lengthValue)); - - if (arrayIndexScale > 1) { - masm.shll(length, NumUtil.log2Ceil(arrayIndexScale)); // scale length - } - - masm.movl(result, length); // copy - - emitArrayCompare(crb, masm, kind, result, array1, array2, length, temp4, temp5, tempXMM, vectorTemp1, vectorTemp2, trueLabel, falseLabel); + // copy + masm.movl(result, length); + emitArrayCompare(crb, masm, result, array1, array2, length, trueLabel, falseLabel); } // Return true @@ -188,19 +193,18 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { masm.bind(done); } - private static void emitArrayCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm, JavaKind kind, + private void emitArrayCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm, Register result, Register array1, Register array2, Register length, - Value temp4, Value temp5, Value tempXMM, Value vectorTemp1, Value vectorTemp2, Label trueLabel, Label falseLabel) { - if (supportsAVX2(crb.target)) { - emitAVXCompare(crb, masm, kind, result, array1, array2, length, temp4, temp5, tempXMM, vectorTemp1, vectorTemp2, trueLabel, falseLabel); - } else if (supportsSSE41(crb.target)) { - // this code is used for AVX as well because our backend correctly ensures that - // VEX-prefixed instructions are emitted if AVX is supported - emitSSE41Compare(crb, masm, kind, result, array1, array2, length, temp4, temp5, tempXMM, vectorTemp1, vectorTemp2, trueLabel, falseLabel); + if (supportsSSE41(crb.target)) { + emitVectorCompare(crb, masm, result, array1, array2, length, trueLabel, falseLabel); + } + if (kind1 == kind2) { + emit8ByteCompare(crb, masm, result, array1, array2, length, trueLabel, falseLabel); + emitTailCompares(masm, result, array1, array2, length, trueLabel, falseLabel); + } else { + emitDifferentKindsElementWiseCompare(crb, masm, result, array1, array2, length, trueLabel, falseLabel); } - emit8ByteCompare(crb, masm, kind, result, array1, array2, length, temp4, tempXMM, trueLabel, falseLabel); - emitTailCompares(masm, kind, result, array1, array2, length, temp4, tempXMM, trueLabel, falseLabel); } /** @@ -215,49 +219,44 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { } /** - * Vector size used in {@link #emitSSE41Compare}. + * Emits code that uses SSE4.1/AVX1 128-bit (16-byte) or AVX2 256-bit (32-byte) vector compares. */ - private static final int SSE4_1_VECTOR_SIZE = 16; - - /** - * Emits code that uses SSE4.1 128-bit (16-byte) vector compares. - */ - private static void emitSSE41Compare(CompilationResultBuilder crb, AMD64MacroAssembler masm, JavaKind kind, + private void emitVectorCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm, Register result, Register array1, Register array2, Register length, - Value temp4, Value temp5, Value tempXMM, Value vectorTemp1, Value vectorTemp2, Label trueLabel, Label falseLabel) { assert supportsSSE41(crb.target); Register vector1 = asRegister(vectorTemp1); Register vector2 = asRegister(vectorTemp2); + int elementsPerVector = getElementsPerVector(vectorSize); + Label loop = new Label(); Label compareTail = new Label(); - boolean requiresNaNCheck = kind.isNumericFloat(); + boolean requiresNaNCheck = kind1.isNumericFloat(); Label loopCheck = new Label(); Label nanCheck = new Label(); // Compare 16-byte vectors - masm.andl(result, SSE4_1_VECTOR_SIZE - 1); // tail count (in bytes) - masm.andl(length, ~(SSE4_1_VECTOR_SIZE - 1)); // vector count (in bytes) + masm.andl(result, elementsPerVector - 1); // tail count + masm.andl(length, ~(elementsPerVector - 1)); // vector count masm.jcc(ConditionFlag.Zero, compareTail); - masm.leaq(array1, new AMD64Address(array1, length, Scale.Times1, 0)); - masm.leaq(array2, new AMD64Address(array2, length, Scale.Times1, 0)); + masm.leaq(array1, new AMD64Address(array1, length, arrayIndexScale1, 0)); + masm.leaq(array2, new AMD64Address(array2, length, arrayIndexScale2, 0)); masm.negq(length); // Align the main loop masm.align(crb.target.wordSize * 2); masm.bind(loop); - masm.movdqu(vector1, new AMD64Address(array1, length, Scale.Times1, 0)); - masm.movdqu(vector2, new AMD64Address(array2, length, Scale.Times1, 0)); - masm.pxor(vector1, vector2); - masm.ptest(vector1, vector1); + emitVectorLoad1(masm, vector1, array1, length, 0, vectorSize); + emitVectorLoad2(masm, vector2, array2, length, 0, vectorSize); + emitVectorCmp(masm, vector1, vector2, vectorSize); masm.jcc(ConditionFlag.NotZero, requiresNaNCheck ? nanCheck : falseLabel); masm.bind(loopCheck); - masm.addq(length, SSE4_1_VECTOR_SIZE); + masm.addq(length, elementsPerVector); masm.jcc(ConditionFlag.NotZero, loop); masm.testl(result, result); @@ -267,7 +266,7 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { Label unalignedCheck = new Label(); masm.jmpb(unalignedCheck); masm.bind(nanCheck); - emitFloatCompareWithinRange(crb, masm, kind, array1, array2, length, temp4, temp5, tempXMM, 0, falseLabel, SSE4_1_VECTOR_SIZE); + emitFloatCompareWithinRange(crb, masm, array1, array2, length, 0, falseLabel, elementsPerVector); masm.jmpb(loopCheck); masm.bind(unalignedCheck); } @@ -276,13 +275,12 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { * Compare the remaining bytes with an unaligned memory load aligned to the end of the * array. */ - masm.movdqu(vector1, new AMD64Address(array1, result, Scale.Times1, -SSE4_1_VECTOR_SIZE)); - masm.movdqu(vector2, new AMD64Address(array2, result, Scale.Times1, -SSE4_1_VECTOR_SIZE)); - masm.pxor(vector1, vector2); - masm.ptest(vector1, vector1); + emitVectorLoad1(masm, vector1, array1, result, scaleDisplacement1(-vectorSize.getBytes()), vectorSize); + emitVectorLoad2(masm, vector2, array2, result, scaleDisplacement2(-vectorSize.getBytes()), vectorSize); + emitVectorCmp(masm, vector1, vector2, vectorSize); if (requiresNaNCheck) { masm.jcc(ConditionFlag.Zero, trueLabel); - emitFloatCompareWithinRange(crb, masm, kind, array1, array2, result, temp4, temp5, tempXMM, -SSE4_1_VECTOR_SIZE, falseLabel, SSE4_1_VECTOR_SIZE); + emitFloatCompareWithinRange(crb, masm, array1, array2, result, -vectorSize.getBytes(), falseLabel, elementsPerVector); } else { masm.jcc(ConditionFlag.NotZero, falseLabel); } @@ -292,90 +290,159 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { masm.movl(length, result); } - /** - * Returns if the underlying AMD64 architecture supports AVX instructions. - * - * @param target target description of the underlying architecture - * @return true if the underlying architecture supports AVX - */ - private static boolean supportsAVX2(TargetDescription target) { - AMD64 arch = (AMD64) target.arch; - return arch.getFeatures().contains(CPUFeature.AVX2); + private int getElementsPerVector(AVXKind.AVXSize vSize) { + return vSize.getBytes() >> Math.max(arrayIndexScale1.log2, arrayIndexScale2.log2); } - /** - * Vector size used in {@link #emitAVXCompare}. - */ - private static final int AVX_VECTOR_SIZE = 32; + private void emitVectorLoad1(AMD64MacroAssembler asm, Register dst, Register src, int displacement, AVXKind.AVXSize size) { + emitVectorLoad1(asm, dst, src, Register.None, displacement, size); + } - private static void emitAVXCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm, JavaKind kind, Register result, - Register array1, Register array2, Register length, - Value temp4, Value temp5, Value tempXMM, Value vectorTemp1, Value vectorTemp2, - Label trueLabel, Label falseLabel) { - assert supportsAVX2(crb.target); + private void emitVectorLoad2(AMD64MacroAssembler asm, Register dst, Register src, int displacement, AVXKind.AVXSize size) { + emitVectorLoad2(asm, dst, src, Register.None, displacement, size); + } - Register vector1 = asRegister(vectorTemp1); - Register vector2 = asRegister(vectorTemp2); + private void emitVectorLoad1(AMD64MacroAssembler asm, Register dst, Register src, Register index, int displacement, AVXKind.AVXSize size) { + emitVectorLoad(asm, dst, src, index, displacement, arrayIndexScale1, arrayIndexScale2, size); + } - Label loop = new Label(); - Label compareTail = new Label(); + private void emitVectorLoad2(AMD64MacroAssembler asm, Register dst, Register src, Register index, int displacement, AVXKind.AVXSize size) { + emitVectorLoad(asm, dst, src, index, displacement, arrayIndexScale2, arrayIndexScale1, size); + } - boolean requiresNaNCheck = kind.isNumericFloat(); - Label loopCheck = new Label(); - Label nanCheck = new Label(); - - // Compare 32-byte vectors - masm.andl(result, AVX_VECTOR_SIZE - 1); // tail count (in bytes) - masm.andl(length, ~(AVX_VECTOR_SIZE - 1)); // vector count (in bytes) - masm.jcc(ConditionFlag.Zero, compareTail); - - masm.leaq(array1, new AMD64Address(array1, length, Scale.Times1, 0)); - masm.leaq(array2, new AMD64Address(array2, length, Scale.Times1, 0)); - masm.negq(length); - - // Align the main loop - masm.align(crb.target.wordSize * 2); - masm.bind(loop); - masm.vmovdqu(vector1, new AMD64Address(array1, length, Scale.Times1, 0)); - masm.vmovdqu(vector2, new AMD64Address(array2, length, Scale.Times1, 0)); - masm.vpxor(vector1, vector1, vector2); - masm.vptest(vector1, vector1); - masm.jcc(ConditionFlag.NotZero, requiresNaNCheck ? nanCheck : falseLabel); - - masm.bind(loopCheck); - masm.addq(length, AVX_VECTOR_SIZE); - masm.jcc(ConditionFlag.NotZero, loop); - - masm.testl(result, result); - masm.jcc(ConditionFlag.Zero, trueLabel); - - if (requiresNaNCheck) { - Label unalignedCheck = new Label(); - masm.jmpb(unalignedCheck); - masm.bind(nanCheck); - emitFloatCompareWithinRange(crb, masm, kind, array1, array2, length, temp4, temp5, tempXMM, 0, falseLabel, AVX_VECTOR_SIZE); - masm.jmpb(loopCheck); - masm.bind(unalignedCheck); - } - - /* - * Compare the remaining bytes with an unaligned memory load aligned to the end of the - * array. - */ - masm.vmovdqu(vector1, new AMD64Address(array1, result, Scale.Times1, -AVX_VECTOR_SIZE)); - masm.vmovdqu(vector2, new AMD64Address(array2, result, Scale.Times1, -AVX_VECTOR_SIZE)); - masm.vpxor(vector1, vector1, vector2); - masm.vptest(vector1, vector1); - if (requiresNaNCheck) { - masm.jcc(ConditionFlag.Zero, trueLabel); - emitFloatCompareWithinRange(crb, masm, kind, array1, array2, result, temp4, temp5, tempXMM, -AVX_VECTOR_SIZE, falseLabel, AVX_VECTOR_SIZE); + private void emitVectorLoad(AMD64MacroAssembler asm, Register dst, Register src, Register index, int displacement, Scale ownScale, Scale otherScale, AVXKind.AVXSize size) { + AMD64Address address = new AMD64Address(src, index, ownScale, displacement); + if (ownScale.value < otherScale.value) { + if (size == AVXKind.AVXSize.YMM) { + getAVX2LoadAndExtendOp(ownScale, otherScale, signExtend).emit(asm, size, dst, address); + } else { + loadAndExtendSSE(asm, dst, address, ownScale, otherScale, signExtend); + } } else { - masm.jcc(ConditionFlag.NotZero, falseLabel); + if (size == AVXKind.AVXSize.YMM) { + asm.vmovdqu(dst, address); + } else { + asm.movdqu(dst, address); + } } - masm.jmp(trueLabel); + } - masm.bind(compareTail); - masm.movl(length, result); + private int scaleDisplacement1(int displacement) { + return scaleDisplacement(displacement, arrayIndexScale1, arrayIndexScale2); + } + + private int scaleDisplacement2(int displacement) { + return scaleDisplacement(displacement, arrayIndexScale2, arrayIndexScale1); + } + + private static int scaleDisplacement(int displacement, Scale ownScale, Scale otherScale) { + if (ownScale.value < otherScale.value) { + return displacement >> (otherScale.log2 - ownScale.log2); + } + return displacement; + } + + private static AMD64Assembler.VexRMOp getAVX2LoadAndExtendOp(Scale ownScale, Scale otherScale, boolean signExtend) { + switch (ownScale) { + case Times1: + switch (otherScale) { + case Times2: + return signExtend ? AMD64Assembler.VexRMOp.VPMOVSXBW : AMD64Assembler.VexRMOp.VPMOVZXBW; + case Times4: + return signExtend ? AMD64Assembler.VexRMOp.VPMOVSXBD : AMD64Assembler.VexRMOp.VPMOVZXBD; + case Times8: + return signExtend ? AMD64Assembler.VexRMOp.VPMOVSXBQ : AMD64Assembler.VexRMOp.VPMOVZXBQ; + } + throw GraalError.shouldNotReachHere(); + case Times2: + switch (otherScale) { + case Times4: + return signExtend ? AMD64Assembler.VexRMOp.VPMOVSXWD : AMD64Assembler.VexRMOp.VPMOVZXWD; + case Times8: + return signExtend ? AMD64Assembler.VexRMOp.VPMOVSXWQ : AMD64Assembler.VexRMOp.VPMOVZXWQ; + } + throw GraalError.shouldNotReachHere(); + case Times4: + return signExtend ? AMD64Assembler.VexRMOp.VPMOVSXDQ : AMD64Assembler.VexRMOp.VPMOVZXDQ; + } + throw GraalError.shouldNotReachHere(); + } + + private static void loadAndExtendSSE(AMD64MacroAssembler asm, Register dst, AMD64Address src, Scale ownScale, Scale otherScale, boolean signExtend) { + switch (ownScale) { + case Times1: + switch (otherScale) { + case Times2: + if (signExtend) { + asm.pmovsxbw(dst, src); + } else { + asm.pmovzxbw(dst, src); + } + return; + case Times4: + if (signExtend) { + asm.pmovsxbd(dst, src); + } else { + asm.pmovzxbd(dst, src); + } + return; + case Times8: + if (signExtend) { + asm.pmovsxbq(dst, src); + } else { + asm.pmovzxbq(dst, src); + } + return; + } + throw GraalError.shouldNotReachHere(); + case Times2: + switch (otherScale) { + case Times4: + if (signExtend) { + asm.pmovsxwd(dst, src); + } else { + asm.pmovzxwd(dst, src); + } + return; + case Times8: + if (signExtend) { + asm.pmovsxwq(dst, src); + } else { + asm.pmovzxwq(dst, src); + } + return; + } + throw GraalError.shouldNotReachHere(); + case Times4: + if (signExtend) { + asm.pmovsxdq(dst, src); + } else { + asm.pmovzxdq(dst, src); + } + return; + } + throw GraalError.shouldNotReachHere(); + } + + private static void emitVectorCmp(AMD64MacroAssembler masm, Register vector1, Register vector2, AVXKind.AVXSize size) { + emitVectorXor(masm, vector1, vector2, size); + emitVectorTest(masm, vector1, size); + } + + private static void emitVectorXor(AMD64MacroAssembler masm, Register vector1, Register vector2, AVXKind.AVXSize size) { + if (size == AVXKind.AVXSize.YMM) { + masm.vpxor(vector1, vector1, vector2); + } else { + masm.pxor(vector1, vector2); + } + } + + private static void emitVectorTest(AMD64MacroAssembler masm, Register vector1, AVXKind.AVXSize size) { + if (size == AVXKind.AVXSize.YMM) { + masm.vptest(vector1, vector1); + } else { + masm.ptest(vector1, vector1); + } } /** @@ -386,34 +453,37 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { /** * Emits code that uses 8-byte vector compares. */ - private static void emit8ByteCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm, JavaKind kind, Register result, Register array1, Register array2, Register length, Value temp4, - Value tempXMM, Label trueLabel, Label falseLabel) { + private void emit8ByteCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm, + Register result, Register array1, Register array2, Register length, Label trueLabel, Label falseLabel) { + assert kind1 == kind2; Label loop = new Label(); Label compareTail = new Label(); - boolean requiresNaNCheck = kind.isNumericFloat(); + int elementsPerVector = 8 >> arrayIndexScale1.log2; + + boolean requiresNaNCheck = kind1.isNumericFloat(); Label loopCheck = new Label(); Label nanCheck = new Label(); Register temp = asRegister(temp4); - masm.andl(result, VECTOR_SIZE - 1); // tail count (in bytes) - masm.andl(length, ~(VECTOR_SIZE - 1)); // vector count (in bytes) + masm.andl(result, elementsPerVector - 1); // tail count + masm.andl(length, ~(elementsPerVector - 1)); // vector count masm.jcc(ConditionFlag.Zero, compareTail); - masm.leaq(array1, new AMD64Address(array1, length, Scale.Times1, 0)); - masm.leaq(array2, new AMD64Address(array2, length, Scale.Times1, 0)); + masm.leaq(array1, new AMD64Address(array1, length, arrayIndexScale1, 0)); + masm.leaq(array2, new AMD64Address(array2, length, arrayIndexScale2, 0)); masm.negq(length); // Align the main loop masm.align(crb.target.wordSize * 2); masm.bind(loop); - masm.movq(temp, new AMD64Address(array1, length, Scale.Times1, 0)); - masm.cmpq(temp, new AMD64Address(array2, length, Scale.Times1, 0)); + masm.movq(temp, new AMD64Address(array1, length, arrayIndexScale1, 0)); + masm.cmpq(temp, new AMD64Address(array2, length, arrayIndexScale2, 0)); masm.jcc(ConditionFlag.NotEqual, requiresNaNCheck ? nanCheck : falseLabel); masm.bind(loopCheck); - masm.addq(length, VECTOR_SIZE); + masm.addq(length, elementsPerVector); masm.jccb(ConditionFlag.NotZero, loop); masm.testl(result, result); @@ -425,8 +495,8 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { masm.jmpb(unalignedCheck); masm.bind(nanCheck); // At most two iterations, unroll in the emitted code. - for (int offset = 0; offset < VECTOR_SIZE; offset += kind.getByteCount()) { - emitFloatCompare(masm, kind, array1, array2, length, temp4, tempXMM, offset, falseLabel, kind.getByteCount() == VECTOR_SIZE); + for (int offset = 0; offset < VECTOR_SIZE; offset += kind1.getByteCount()) { + emitFloatCompare(masm, array1, array2, length, offset, falseLabel, kind1.getByteCount() == VECTOR_SIZE); } masm.jmpb(loopCheck); masm.bind(unalignedCheck); @@ -436,13 +506,13 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { * Compare the remaining bytes with an unaligned memory load aligned to the end of the * array. */ - masm.movq(temp, new AMD64Address(array1, result, Scale.Times1, -VECTOR_SIZE)); - masm.cmpq(temp, new AMD64Address(array2, result, Scale.Times1, -VECTOR_SIZE)); + masm.movq(temp, new AMD64Address(array1, result, arrayIndexScale1, -VECTOR_SIZE)); + masm.cmpq(temp, new AMD64Address(array2, result, arrayIndexScale2, -VECTOR_SIZE)); if (requiresNaNCheck) { masm.jcc(ConditionFlag.Equal, trueLabel); // At most two iterations, unroll in the emitted code. - for (int offset = 0; offset < VECTOR_SIZE; offset += kind.getByteCount()) { - emitFloatCompare(masm, kind, array1, array2, result, temp4, tempXMM, -VECTOR_SIZE + offset, falseLabel, kind.getByteCount() == VECTOR_SIZE); + for (int offset = 0; offset < VECTOR_SIZE; offset += kind1.getByteCount()) { + emitFloatCompare(masm, array1, array2, result, -VECTOR_SIZE + offset, falseLabel, kind1.getByteCount() == VECTOR_SIZE); } } else { masm.jccb(ConditionFlag.NotEqual, falseLabel); @@ -456,34 +526,35 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { /** * Emits code to compare the remaining 1 to 4 bytes. */ - private static void emitTailCompares(AMD64MacroAssembler masm, JavaKind kind, Register result, Register array1, Register array2, Register length, Value temp4, Value tempXMM, - Label trueLabel, Label falseLabel) { + private void emitTailCompares(AMD64MacroAssembler masm, + Register result, Register array1, Register array2, Register length, Label trueLabel, Label falseLabel) { + assert kind1 == kind2; Label compare2Bytes = new Label(); Label compare1Byte = new Label(); Register temp = asRegister(temp4); - if (kind.getByteCount() <= 4) { + if (kind1.getByteCount() <= 4) { // Compare trailing 4 bytes, if any. - masm.testl(result, 4); + masm.testl(result, arrayIndexScale1.log2 == 0 ? 4 : 4 >> arrayIndexScale1.log2); masm.jccb(ConditionFlag.Zero, compare2Bytes); masm.movl(temp, new AMD64Address(array1, 0)); masm.cmpl(temp, new AMD64Address(array2, 0)); - if (kind == JavaKind.Float) { + if (kind1 == JavaKind.Float) { masm.jccb(ConditionFlag.Equal, trueLabel); - emitFloatCompare(masm, kind, array1, array2, Register.None, temp4, tempXMM, 0, falseLabel, true); + emitFloatCompare(masm, array1, array2, Register.None, 0, falseLabel, true); masm.jmpb(trueLabel); } else { masm.jccb(ConditionFlag.NotEqual, falseLabel); } - if (kind.getByteCount() <= 2) { + if (kind1.getByteCount() <= 2) { // Move array pointers forward. masm.leaq(array1, new AMD64Address(array1, 4)); masm.leaq(array2, new AMD64Address(array2, 4)); // Compare trailing 2 bytes, if any. masm.bind(compare2Bytes); - masm.testl(result, 2); + masm.testl(result, arrayIndexScale1.log2 == 0 ? 2 : 2 >> arrayIndexScale1.log2); masm.jccb(ConditionFlag.Zero, compare1Byte); masm.movzwl(temp, new AMD64Address(array1, 0)); masm.movzwl(length, new AMD64Address(array2, 0)); @@ -491,7 +562,7 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { masm.jccb(ConditionFlag.NotEqual, falseLabel); // The one-byte tail compare is only required for boolean and byte arrays. - if (kind.getByteCount() <= 1) { + if (kind1.getByteCount() <= 1) { // Move array pointers forward before we compare the last trailing byte. masm.leaq(array1, new AMD64Address(array1, 2)); masm.leaq(array2, new AMD64Address(array2, 2)); @@ -513,28 +584,82 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { } } + private void emitDifferentKindsElementWiseCompare(CompilationResultBuilder crb, AMD64MacroAssembler masm, + Register result, Register array1, Register array2, Register length, Label trueLabel, Label falseLabel) { + assert kind1 != kind2; + assert kind1.isNumericInteger() && kind2.isNumericInteger(); + Label loop = new Label(); + Label compareTail = new Label(); + + int elementsPerLoopIteration = 4; + + Register tmp1 = asRegister(temp4); + Register tmp2 = asRegister(temp5); + + masm.andl(result, elementsPerLoopIteration - 1); // tail count + masm.andl(length, ~(elementsPerLoopIteration - 1)); // bulk loop count + masm.jcc(ConditionFlag.Zero, compareTail); + + masm.leaq(array1, new AMD64Address(array1, length, arrayIndexScale1, 0)); + masm.leaq(array2, new AMD64Address(array2, length, arrayIndexScale2, 0)); + masm.negq(length); + + // clear comparison registers because of the missing movzlq instruction + masm.xorq(tmp1, tmp1); + masm.xorq(tmp2, tmp2); + + // Align the main loop + masm.align(crb.target.wordSize * 2); + masm.bind(loop); + for (int i = 0; i < elementsPerLoopIteration; i++) { + emitMovBytes(masm, tmp1, new AMD64Address(array1, length, arrayIndexScale1, i << arrayIndexScale1.log2), kind1.getByteCount()); + emitMovBytes(masm, tmp2, new AMD64Address(array2, length, arrayIndexScale2, i << arrayIndexScale2.log2), kind2.getByteCount()); + masm.cmpq(tmp1, tmp2); + masm.jcc(ConditionFlag.NotEqual, falseLabel); + } + masm.addq(length, elementsPerLoopIteration); + masm.jccb(ConditionFlag.NotZero, loop); + + masm.bind(compareTail); + masm.testl(result, result); + masm.jcc(ConditionFlag.Zero, trueLabel); + for (int i = 0; i < elementsPerLoopIteration - 1; i++) { + emitMovBytes(masm, tmp1, new AMD64Address(array1, length, arrayIndexScale1, 0), kind1.getByteCount()); + emitMovBytes(masm, tmp2, new AMD64Address(array2, length, arrayIndexScale2, 0), kind2.getByteCount()); + masm.cmpq(tmp1, tmp2); + masm.jcc(ConditionFlag.NotEqual, falseLabel); + if (i < elementsPerLoopIteration - 2) { + masm.incrementq(length, 1); + masm.decrementq(result, 1); + masm.jcc(ConditionFlag.Zero, trueLabel); + } else { + masm.jmpb(trueLabel); + } + } + } + /** * Emits code to fall through if {@code src} is NaN, otherwise jump to {@code branchOrdered}. */ - private static void emitNaNCheck(AMD64MacroAssembler masm, JavaKind kind, Value tempXMM, AMD64Address src, Label branchIfNonNaN) { - assert kind.isNumericFloat(); + private void emitNaNCheck(AMD64MacroAssembler masm, AMD64Address src, Label branchIfNonNaN) { + assert kind1.isNumericFloat(); Register tempXMMReg = asRegister(tempXMM); - if (kind == JavaKind.Float) { + if (kind1 == JavaKind.Float) { masm.movflt(tempXMMReg, src); } else { masm.movdbl(tempXMMReg, src); } - SSEOp.UCOMIS.emit(masm, kind == JavaKind.Float ? OperandSize.PS : OperandSize.PD, tempXMMReg, tempXMMReg); + SSEOp.UCOMIS.emit(masm, kind1 == JavaKind.Float ? OperandSize.PS : OperandSize.PD, tempXMMReg, tempXMMReg); masm.jcc(ConditionFlag.NoParity, branchIfNonNaN); } /** * Emits code to compare if two floats are bitwise equal or both NaN. */ - private static void emitFloatCompare(AMD64MacroAssembler masm, JavaKind kind, Register base1, Register base2, Register index, Value temp4, Value tempXMM, int offset, Label falseLabel, + private void emitFloatCompare(AMD64MacroAssembler masm, Register base1, Register base2, Register index, int offset, Label falseLabel, boolean skipBitwiseCompare) { - AMD64Address address1 = new AMD64Address(base1, index, Scale.Times1, offset); - AMD64Address address2 = new AMD64Address(base2, index, Scale.Times1, offset); + AMD64Address address1 = new AMD64Address(base1, index, arrayIndexScale1, offset); + AMD64Address address2 = new AMD64Address(base2, index, arrayIndexScale2, offset); Label bitwiseEqual = new Label(); @@ -542,7 +667,7 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { // Bitwise compare Register temp = asRegister(temp4); - if (kind == JavaKind.Float) { + if (kind1 == JavaKind.Float) { masm.movl(temp, address1); masm.cmpl(temp, address2); } else { @@ -552,8 +677,8 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { masm.jccb(ConditionFlag.Equal, bitwiseEqual); } - emitNaNCheck(masm, kind, tempXMM, address1, falseLabel); - emitNaNCheck(masm, kind, tempXMM, address2, falseLabel); + emitNaNCheck(masm, address1, falseLabel); + emitNaNCheck(masm, address2, falseLabel); masm.bind(bitwiseEqual); } @@ -561,9 +686,9 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { /** * Emits code to compare float equality within a range. */ - private static void emitFloatCompareWithinRange(CompilationResultBuilder crb, AMD64MacroAssembler masm, JavaKind kind, Register base1, Register base2, Register index, Value temp4, Value temp5, - Value tempXMM, int offset, Label falseLabel, int range) { - assert kind.isNumericFloat(); + private void emitFloatCompareWithinRange(CompilationResultBuilder crb, AMD64MacroAssembler masm, + Register base1, Register base2, Register index, int offset, Label falseLabel, int range) { + assert kind1.isNumericFloat(); Label loop = new Label(); Register i = asRegister(temp5); @@ -572,9 +697,9 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { // Align the main loop masm.align(crb.target.wordSize * 2); masm.bind(loop); - emitFloatCompare(masm, kind, base1, base2, index, temp4, tempXMM, offset, falseLabel, kind.getByteCount() == range); - masm.addq(index, kind.getByteCount()); - masm.addq(i, kind.getByteCount()); + emitFloatCompare(masm, base1, base2, index, offset, falseLabel, range == 1); + masm.incrementq(index, 1); + masm.incrementq(i, 1); masm.jccb(ConditionFlag.NotZero, loop); // Floats within the range are equal, revert change to the register index masm.subq(index, range); @@ -585,189 +710,104 @@ public final class AMD64ArrayEqualsOp extends AMD64LIRInstruction { * {@code arrayPtr1[0..nBytes]} and {@code arrayPtr2[0..nBytes]}. If they match, execution * continues directly after the emitted code block, otherwise we jump to {@code noMatch}. */ - private static void emitConstantLengthArrayCompareBytes( + private void emitConstantLengthArrayCompareBytes( + CompilationResultBuilder crb, AMD64MacroAssembler asm, Register arrayPtr1, Register arrayPtr2, Register tmp1, Register tmp2, Register[] tmpVectors, - Label noMatch, - int nBytes, - int bytesPerVector) { - assert bytesPerVector >= 16; - if (nBytes == 0) { + Label noMatch) { + if (constantLength == 0) { // do nothing return; } - if (nBytes < 16) { + AVXKind.AVXSize vSize = vectorSize; + if (constantLength < getElementsPerVector(vectorSize)) { + vSize = AVXKind.AVXSize.XMM; + } + int elementsPerVector = getElementsPerVector(vSize); + if (elementsPerVector > constantLength) { + assert kind1 == kind2; + int byteLength = constantLength << arrayIndexScale1.log2; // array is shorter than any vector register, use regular CMP instructions - int movSize = (nBytes < 2) ? 1 : ((nBytes < 4) ? 2 : ((nBytes < 8) ? 4 : 8)); + int movSize = (byteLength < 2) ? 1 : ((byteLength < 4) ? 2 : ((byteLength < 8) ? 4 : 8)); emitMovBytes(asm, tmp1, new AMD64Address(arrayPtr1), movSize); emitMovBytes(asm, tmp2, new AMD64Address(arrayPtr2), movSize); emitCmpBytes(asm, tmp1, tmp2, movSize); asm.jcc(AMD64Assembler.ConditionFlag.NotEqual, noMatch); - if (nBytes > movSize) { - emitMovBytes(asm, tmp1, new AMD64Address(arrayPtr1, nBytes - movSize), movSize); - emitMovBytes(asm, tmp2, new AMD64Address(arrayPtr2, nBytes - movSize), movSize); + if (byteLength > movSize) { + emitMovBytes(asm, tmp1, new AMD64Address(arrayPtr1, byteLength - movSize), movSize); + emitMovBytes(asm, tmp2, new AMD64Address(arrayPtr2, byteLength - movSize), movSize); emitCmpBytes(asm, tmp1, tmp2, movSize); asm.jcc(AMD64Assembler.ConditionFlag.NotEqual, noMatch); } - } else if (nBytes < 32 && bytesPerVector >= 32) { - // we could use YMM registers, but the array is too short, force XMM registers - int bytesPerXMMVector = AVXKind.AVXSize.XMM.getBytes(); - AMD64Assembler.VexMoveOp.VMOVDQU.emit(asm, AVXKind.AVXSize.XMM, tmpVectors[0], new AMD64Address(arrayPtr1)); - AMD64Assembler.VexMoveOp.VMOVDQU.emit(asm, AVXKind.AVXSize.XMM, tmpVectors[1], new AMD64Address(arrayPtr2)); - AMD64Assembler.VexRVMOp.VPXOR.emit(asm, AVXKind.AVXSize.XMM, tmpVectors[0], tmpVectors[0], tmpVectors[1]); - if (nBytes > bytesPerXMMVector) { - AMD64Assembler.VexMoveOp.VMOVDQU.emit(asm, AVXKind.AVXSize.XMM, tmpVectors[2], new AMD64Address(arrayPtr1, nBytes - bytesPerXMMVector)); - AMD64Assembler.VexMoveOp.VMOVDQU.emit(asm, AVXKind.AVXSize.XMM, tmpVectors[3], new AMD64Address(arrayPtr2, nBytes - bytesPerXMMVector)); - AMD64Assembler.VexRVMOp.VPXOR.emit(asm, AVXKind.AVXSize.XMM, tmpVectors[2], tmpVectors[2], tmpVectors[3]); - AMD64Assembler.VexRMOp.VPTEST.emit(asm, AVXKind.AVXSize.XMM, tmpVectors[2], tmpVectors[2]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - } - AMD64Assembler.VexRMOp.VPTEST.emit(asm, AVXKind.AVXSize.XMM, tmpVectors[0], tmpVectors[0]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - } else if (bytesPerVector >= 32) { - // AVX2 supported, use YMM vectors - assert asm.supports(CPUFeature.AVX2); - int loopCount = nBytes / (bytesPerVector * 2); - int rest = nBytes % (bytesPerVector * 2); - if (loopCount > 0) { - if (0 < rest && rest < bytesPerVector) { - loopCount--; - } - if (loopCount > 0) { - if (loopCount > 1) { - asm.movl(tmp1, loopCount); - } - Label loopBegin = new Label(); - asm.bind(loopBegin); - asm.vmovdqu(tmpVectors[0], new AMD64Address(arrayPtr1)); - asm.vmovdqu(tmpVectors[1], new AMD64Address(arrayPtr2)); - asm.vmovdqu(tmpVectors[2], new AMD64Address(arrayPtr1, bytesPerVector)); - asm.vmovdqu(tmpVectors[3], new AMD64Address(arrayPtr2, bytesPerVector)); - asm.vpxor(tmpVectors[0], tmpVectors[0], tmpVectors[1]); - asm.vpxor(tmpVectors[2], tmpVectors[2], tmpVectors[3]); - asm.vptest(tmpVectors[0], tmpVectors[0]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - asm.vptest(tmpVectors[2], tmpVectors[2]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - asm.addq(arrayPtr1, bytesPerVector * 2); - asm.addq(arrayPtr2, bytesPerVector * 2); - if (loopCount > 1) { - asm.decrementl(tmp1); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, loopBegin); - } - } - if (0 < rest && rest < bytesPerVector) { - asm.vmovdqu(tmpVectors[0], new AMD64Address(arrayPtr1)); - asm.vmovdqu(tmpVectors[1], new AMD64Address(arrayPtr2)); - asm.vmovdqu(tmpVectors[2], new AMD64Address(arrayPtr1, bytesPerVector)); - asm.vmovdqu(tmpVectors[3], new AMD64Address(arrayPtr2, bytesPerVector)); - asm.vpxor(tmpVectors[0], tmpVectors[0], tmpVectors[1]); - asm.vpxor(tmpVectors[2], tmpVectors[2], tmpVectors[3]); - asm.vptest(tmpVectors[0], tmpVectors[0]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - asm.vptest(tmpVectors[2], tmpVectors[2]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - asm.vmovdqu(tmpVectors[0], new AMD64Address(arrayPtr1, bytesPerVector + rest)); - asm.vmovdqu(tmpVectors[1], new AMD64Address(arrayPtr2, bytesPerVector + rest)); - asm.vpxor(tmpVectors[0], tmpVectors[0], tmpVectors[1]); - asm.vptest(tmpVectors[0], tmpVectors[0]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - } - } - if (rest >= bytesPerVector) { - asm.vmovdqu(tmpVectors[0], new AMD64Address(arrayPtr1)); - asm.vmovdqu(tmpVectors[1], new AMD64Address(arrayPtr2)); - asm.vpxor(tmpVectors[0], tmpVectors[0], tmpVectors[1]); - if (rest > bytesPerVector) { - asm.vmovdqu(tmpVectors[2], new AMD64Address(arrayPtr1, rest - bytesPerVector)); - asm.vmovdqu(tmpVectors[3], new AMD64Address(arrayPtr2, rest - bytesPerVector)); - asm.vpxor(tmpVectors[2], tmpVectors[2], tmpVectors[3]); - asm.vptest(tmpVectors[2], tmpVectors[2]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - } - asm.vptest(tmpVectors[0], tmpVectors[0]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - } } else { - // on AVX or SSE, use XMM vectors - int loopCount = nBytes / (bytesPerVector * 2); - int rest = nBytes % (bytesPerVector * 2); - if (loopCount > 0) { - if (0 < rest && rest < bytesPerVector) { - loopCount--; - } - if (loopCount > 0) { - if (loopCount > 1) { - asm.movl(tmp1, loopCount); - } - Label loopBegin = new Label(); - asm.bind(loopBegin); - asm.movdqu(tmpVectors[0], new AMD64Address(arrayPtr1)); - asm.movdqu(tmpVectors[1], new AMD64Address(arrayPtr2)); - asm.movdqu(tmpVectors[2], new AMD64Address(arrayPtr1, bytesPerVector)); - asm.movdqu(tmpVectors[3], new AMD64Address(arrayPtr2, bytesPerVector)); - asm.pxor(tmpVectors[0], tmpVectors[1]); - asm.pxor(tmpVectors[2], tmpVectors[3]); - asm.ptest(tmpVectors[0], tmpVectors[0]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - asm.ptest(tmpVectors[2], tmpVectors[2]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - asm.addq(arrayPtr1, bytesPerVector * 2); - asm.addq(arrayPtr2, bytesPerVector * 2); - if (loopCount > 1) { - asm.decrementl(tmp1); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, loopBegin); - } - } - if (0 < rest && rest < bytesPerVector) { - asm.movdqu(tmpVectors[0], new AMD64Address(arrayPtr1)); - asm.movdqu(tmpVectors[1], new AMD64Address(arrayPtr2)); - asm.movdqu(tmpVectors[2], new AMD64Address(arrayPtr1, bytesPerVector)); - asm.movdqu(tmpVectors[3], new AMD64Address(arrayPtr2, bytesPerVector)); - asm.pxor(tmpVectors[0], tmpVectors[1]); - asm.pxor(tmpVectors[2], tmpVectors[3]); - asm.ptest(tmpVectors[0], tmpVectors[0]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - asm.ptest(tmpVectors[2], tmpVectors[2]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - asm.movdqu(tmpVectors[0], new AMD64Address(arrayPtr1, bytesPerVector + rest)); - asm.movdqu(tmpVectors[1], new AMD64Address(arrayPtr2, bytesPerVector + rest)); - asm.pxor(tmpVectors[0], tmpVectors[1]); - asm.ptest(tmpVectors[0], tmpVectors[0]); - asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); - } + int elementsPerVectorLoop = 2 * elementsPerVector; + int tailCount = constantLength & (elementsPerVectorLoop - 1); + int vectorCount = constantLength & ~(elementsPerVectorLoop - 1); + int bytesPerVector = vSize.getBytes(); + if (vectorCount > 0) { + Label loopBegin = new Label(); + asm.leaq(arrayPtr1, new AMD64Address(arrayPtr1, vectorCount << arrayIndexScale1.log2)); + asm.leaq(arrayPtr2, new AMD64Address(arrayPtr2, vectorCount << arrayIndexScale2.log2)); + asm.movq(tmp1, -vectorCount); + asm.align(crb.target.wordSize * 2); + asm.bind(loopBegin); + emitVectorLoad1(asm, tmpVectors[0], arrayPtr1, tmp1, 0, vSize); + emitVectorLoad2(asm, tmpVectors[1], arrayPtr2, tmp1, 0, vSize); + emitVectorLoad1(asm, tmpVectors[2], arrayPtr1, tmp1, scaleDisplacement1(bytesPerVector), vSize); + emitVectorLoad2(asm, tmpVectors[3], arrayPtr2, tmp1, scaleDisplacement2(bytesPerVector), vSize); + emitVectorXor(asm, tmpVectors[0], tmpVectors[1], vSize); + emitVectorXor(asm, tmpVectors[2], tmpVectors[3], vSize); + emitVectorTest(asm, tmpVectors[0], vSize); + asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); + emitVectorTest(asm, tmpVectors[2], vSize); + asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); + asm.addq(tmp1, elementsPerVectorLoop); + asm.jcc(AMD64Assembler.ConditionFlag.NotZero, loopBegin); } - if (rest >= bytesPerVector) { - asm.movdqu(tmpVectors[0], new AMD64Address(arrayPtr1)); - asm.movdqu(tmpVectors[1], new AMD64Address(arrayPtr2)); - asm.pxor(tmpVectors[0], tmpVectors[1]); - if (rest > bytesPerVector) { - asm.movdqu(tmpVectors[2], new AMD64Address(arrayPtr1, rest - bytesPerVector)); - asm.movdqu(tmpVectors[3], new AMD64Address(arrayPtr2, rest - bytesPerVector)); - asm.pxor(tmpVectors[2], tmpVectors[3]); - asm.ptest(tmpVectors[2], tmpVectors[2]); + if (tailCount > 0) { + emitVectorLoad1(asm, tmpVectors[0], arrayPtr1, (tailCount << arrayIndexScale1.log2) - scaleDisplacement1(bytesPerVector), vSize); + emitVectorLoad2(asm, tmpVectors[1], arrayPtr2, (tailCount << arrayIndexScale2.log2) - scaleDisplacement2(bytesPerVector), vSize); + emitVectorXor(asm, tmpVectors[0], tmpVectors[1], vSize); + if (tailCount > elementsPerVector) { + emitVectorLoad1(asm, tmpVectors[2], arrayPtr1, 0, vSize); + emitVectorLoad2(asm, tmpVectors[3], arrayPtr2, 0, vSize); + emitVectorXor(asm, tmpVectors[2], tmpVectors[3], vSize); + emitVectorTest(asm, tmpVectors[2], vSize); asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); } - asm.ptest(tmpVectors[0], tmpVectors[0]); + emitVectorTest(asm, tmpVectors[0], vSize); asm.jcc(AMD64Assembler.ConditionFlag.NotZero, noMatch); } } } - private static void emitMovBytes(AMD64MacroAssembler asm, Register dst, AMD64Address src, int size) { + private void emitMovBytes(AMD64MacroAssembler asm, Register dst, AMD64Address src, int size) { switch (size) { case 1: - asm.movzbl(dst, src); + if (signExtend) { + asm.movsbq(dst, src); + } else { + asm.movzbq(dst, src); + } break; case 2: - asm.movzwl(dst, src); + if (signExtend) { + asm.movswq(dst, src); + } else { + asm.movzwq(dst, src); + } break; case 4: - asm.movl(dst, src); + if (signExtend) { + asm.movslq(dst, src); + } else { + // there is no movzlq + asm.movl(dst, src); + } break; case 8: asm.movq(dst, src); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ControlFlow.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ControlFlow.java index f010ea0ae81..e4b44e443ec 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ControlFlow.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64ControlFlow.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -283,12 +283,10 @@ public class AMD64ControlFlow { masm.jmp(scratchReg); // Inserting padding so that jump table address is 4-byte aligned - if ((masm.position() & 0x3) != 0) { - masm.nop(4 - (masm.position() & 0x3)); - } + masm.align(4); // Patch LEA instruction above now that we know the position of the jump table - // TODO this is ugly and should be done differently + // this is ugly but there is no better way to do this given the assembler API final int jumpTablePos = masm.position(); final int leaDisplacementPosition = afterLea - 4; masm.emitInt(jumpTablePos - afterLea, leaDisplacementPosition); @@ -314,6 +312,99 @@ public class AMD64ControlFlow { } } + public static final class HashTableSwitchOp extends AMD64BlockEndOp { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(HashTableSwitchOp.class); + private final JavaConstant[] keys; + private final LabelRef defaultTarget; + private final LabelRef[] targets; + @Alive protected Value value; + @Alive protected Value hash; + @Temp({REG}) protected Value entryScratch; + @Temp({REG}) protected Value scratch; + + public HashTableSwitchOp(final JavaConstant[] keys, final LabelRef defaultTarget, LabelRef[] targets, Value value, Value hash, Variable scratch, Variable entryScratch) { + super(TYPE); + this.keys = keys; + this.defaultTarget = defaultTarget; + this.targets = targets; + this.value = value; + this.hash = hash; + this.scratch = scratch; + this.entryScratch = entryScratch; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + Register valueReg = asRegister(value, AMD64Kind.DWORD); + Register indexReg = asRegister(hash, AMD64Kind.DWORD); + Register scratchReg = asRegister(scratch, AMD64Kind.QWORD); + Register entryScratchReg = asRegister(entryScratch, AMD64Kind.QWORD); + + // Set scratch to address of jump table + masm.leaq(scratchReg, new AMD64Address(AMD64.rip, 0)); + final int afterLea = masm.position(); + + // When the default target is set, the jump table contains entries with two DWORDS: + // the original key before hashing and the label jump address + if (defaultTarget != null) { + + // Move the table entry (two DWORDs) into a QWORD + masm.movq(entryScratchReg, new AMD64Address(scratchReg, indexReg, Scale.Times8, 0)); + + // Jump to the default target if the first DWORD (original key) doesn't match the + // current key. Accounts for hash collisions with unknown keys + masm.cmpl(entryScratchReg, valueReg); + masm.jcc(ConditionFlag.NotEqual, defaultTarget.label()); + + // Shift to the second DWORD + masm.sarq(entryScratchReg, 32); + } else { + + // The jump table has a single DWORD with the label address if there's no + // default target + masm.movslq(entryScratchReg, new AMD64Address(scratchReg, indexReg, Scale.Times4, 0)); + } + masm.addq(scratchReg, entryScratchReg); + masm.jmp(scratchReg); + + // Inserting padding so that jump the table address is aligned + if (defaultTarget != null) { + masm.align(8); + } else { + masm.align(4); + } + + // Patch LEA instruction above now that we know the position of the jump table + // this is ugly but there is no better way to do this given the assembler API + final int jumpTablePos = masm.position(); + final int leaDisplacementPosition = afterLea - 4; + masm.emitInt(jumpTablePos - afterLea, leaDisplacementPosition); + + // Emit jump table entries + for (int i = 0; i < targets.length; i++) { + + Label label = targets[i].label(); + + if (defaultTarget != null) { + masm.emitInt(keys[i].asInt()); + } + if (label.isBound()) { + int imm32 = label.position() - jumpTablePos; + masm.emitInt(imm32); + } else { + int offsetToJumpTableBase = masm.position() - jumpTablePos; + label.addPatchAt(masm.position()); + masm.emitByte(0); // pseudo-opcode for jump table entry + masm.emitShort(offsetToJumpTableBase); + masm.emitByte(0); // padding to make jump table entry 4 bytes wide + } + } + + JumpTable jt = new JumpTable(jumpTablePos, keys[0].asInt(), keys[keys.length - 1].asInt(), 4); + crb.compilationResult.addAnnotation(jt); + } + } + @Opcode("SETcc") public static final class CondSetOp extends AMD64LIRInstruction { public static final LIRInstructionClass TYPE = LIRInstructionClass.create(CondSetOp.class); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64HotSpotHelper.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64HotSpotHelper.java new file mode 100644 index 00000000000..9ab7e29a3f0 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64HotSpotHelper.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.amd64; + +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.core.common.LIRKind; +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; + +import jdk.vm.ci.amd64.AMD64; +import jdk.vm.ci.amd64.AMD64Kind; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.Value; + +public final class AMD64HotSpotHelper { + + private AMD64HotSpotHelper() { + } + + protected static Value[] registersToValues(Register[] registers) { + Value[] temps = new Value[registers.length]; + for (int i = 0; i < registers.length; i++) { + Register register = registers[i]; + if (AMD64.CPU.equals(register.getRegisterCategory())) { + temps[i] = register.asValue(LIRKind.value(AMD64Kind.QWORD)); + } else if (AMD64.XMM.equals(register.getRegisterCategory())) { + temps[i] = register.asValue(LIRKind.value(AMD64Kind.DOUBLE)); + } else { + throw GraalError.shouldNotReachHere("Unsupported register type in math stubs."); + } + } + return temps; + } + + protected static AMD64Address recordExternalAddress(CompilationResultBuilder crb, ArrayDataPointerConstant ptr) { + return (AMD64Address) crb.recordDataReferenceInCode(ptr); + } + + protected static ArrayDataPointerConstant pointerConstant(int alignment, int[] ints) { + return new ArrayDataPointerConstant(ints, alignment); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathCosOp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathCosOp.java new file mode 100644 index 00000000000..2e27a702ad0 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathCosOp.java @@ -0,0 +1,869 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, Intel Corporation. All rights reserved. + * Intel Math Library (LIBM) Source Code + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.amd64; + +import static jdk.vm.ci.amd64.AMD64.r10; +import static jdk.vm.ci.amd64.AMD64.r11; +import static jdk.vm.ci.amd64.AMD64.r8; +import static jdk.vm.ci.amd64.AMD64.r9; +import static jdk.vm.ci.amd64.AMD64.rax; +import static jdk.vm.ci.amd64.AMD64.rbx; +import static jdk.vm.ci.amd64.AMD64.rcx; +import static jdk.vm.ci.amd64.AMD64.rdi; +import static jdk.vm.ci.amd64.AMD64.rdx; +import static jdk.vm.ci.amd64.AMD64.rsi; +import static jdk.vm.ci.amd64.AMD64.rsp; +import static jdk.vm.ci.amd64.AMD64.xmm0; +import static jdk.vm.ci.amd64.AMD64.xmm1; +import static jdk.vm.ci.amd64.AMD64.xmm2; +import static jdk.vm.ci.amd64.AMD64.xmm3; +import static jdk.vm.ci.amd64.AMD64.xmm4; +import static jdk.vm.ci.amd64.AMD64.xmm5; +import static jdk.vm.ci.amd64.AMD64.xmm6; +import static jdk.vm.ci.amd64.AMD64.xmm7; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.pointerConstant; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.recordExternalAddress; + +import org.graalvm.compiler.asm.Label; +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; +import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.lir.LIRInstructionClass; +import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; + +/** + * 

+ *                     ALGORITHM DESCRIPTION - COS()
+ *                     ---------------------
+ *
+ *     1. RANGE REDUCTION
+ *
+ *     We perform an initial range reduction from X to r with
+ *
+ *          X =~= N * pi/32 + r
+ *
+ *     so that |r| <= pi/64 + epsilon. We restrict inputs to those
+ *     where |N| <= 932560. Beyond this, the range reduction is
+ *     insufficiently accurate. For extremely small inputs,
+ *     denormalization can occur internally, impacting performance.
+ *     This means that the main path is actually only taken for
+ *     2^-252 <= |X| < 90112.
+ *
+ *     To avoid branches, we perform the range reduction to full
+ *     accuracy each time.
+ *
+ *          X - N * (P_1 + P_2 + P_3)
+ *
+ *     where P_1 and P_2 are 32-bit numbers (so multiplication by N
+ *     is exact) and P_3 is a 53-bit number. Together, these
+ *     approximate pi well enough for all cases in the restricted
+ *     range.
+ *
+ *     The main reduction sequence is:
+ *
+ *             y = 32/pi * x
+ *             N = integer(y)
+ *     (computed by adding and subtracting off SHIFTER)
+ *
+ *             m_1 = N * P_1
+ *             m_2 = N * P_2
+ *             r_1 = x - m_1
+ *             r = r_1 - m_2
+ *     (this r can be used for most of the calculation)
+ *
+ *             c_1 = r_1 - r
+ *             m_3 = N * P_3
+ *             c_2 = c_1 - m_2
+ *             c = c_2 - m_3
+ *
+ *     2. MAIN ALGORITHM
+ *
+ *     The algorithm uses a table lookup based on B = M * pi / 32
+ *     where M = N mod 64. The stored values are:
+ *       sigma             closest power of 2 to cos(B)
+ *       C_hl              53-bit cos(B) - sigma
+ *       S_hi + S_lo       2 * 53-bit sin(B)
+ *
+ *     The computation is organized as follows:
+ *
+ *          sin(B + r + c) = [sin(B) + sigma * r] +
+ *                           r * (cos(B) - sigma) +
+ *                           sin(B) * [cos(r + c) - 1] +
+ *                           cos(B) * [sin(r + c) - r]
+ *
+ *     which is approximately:
+ *
+ *          [S_hi + sigma * r] +
+ *          C_hl * r +
+ *          S_lo + S_hi * [(cos(r) - 1) - r * c] +
+ *          (C_hl + sigma) * [(sin(r) - r) + c]
+ *
+ *     and this is what is actually computed. We separate this sum
+ *     into four parts:
+ *
+ *          hi + med + pols + corr
+ *
+ *     where
+ *
+ *          hi       = S_hi + sigma r
+ *          med      = C_hl * r
+ *          pols     = S_hi * (cos(r) - 1) + (C_hl + sigma) * (sin(r) - r)
+ *          corr     = S_lo + c * ((C_hl + sigma) - S_hi * r)
+ *
+ *     3. POLYNOMIAL
+ *
+ *     The polynomial S_hi * (cos(r) - 1) + (C_hl + sigma) *
+ *     (sin(r) - r) can be rearranged freely, since it is quite
+ *     small, so we exploit parallelism to the fullest.
+ *
+ *          psc4       =   SC_4 * r_1
+ *          msc4       =   psc4 * r
+ *          r2         =   r * r
+ *          msc2       =   SC_2 * r2
+ *          r4         =   r2 * r2
+ *          psc3       =   SC_3 + msc4
+ *          psc1       =   SC_1 + msc2
+ *          msc3       =   r4 * psc3
+ *          sincospols =   psc1 + msc3
+ *          pols       =   sincospols *
+ *                         
+ *
+ *     4. CORRECTION TERM
+ *
+ *     This is where the "c" component of the range reduction is
+ *     taken into account; recall that just "r" is used for most of
+ *     the calculation.
+ *
+ *          -c   = m_3 - c_2
+ *          -d   = S_hi * r - (C_hl + sigma)
+ *          corr = -c * -d + S_lo
+ *
+ *     5. COMPENSATED SUMMATIONS
+ *
+ *     The two successive compensated summations add up the high
+ *     and medium parts, leaving just the low parts to add up at
+ *     the end.
+ *
+ *          rs        =  sigma * r
+ *          res_int   =  S_hi + rs
+ *          k_0       =  S_hi - res_int
+ *          k_2       =  k_0 + rs
+ *          med       =  C_hl * r
+ *          res_hi    =  res_int + med
+ *          k_1       =  res_int - res_hi
+ *          k_3       =  k_1 + med
+ *
+ *     6. FINAL SUMMATION
+ *
+ *     We now add up all the small parts:
+ *
+ *          res_lo = pols(hi) + pols(lo) + corr + k_1 + k_3
+ *
+ *     Now the overall result is just:
+ *
+ *          res_hi + res_lo
+ *
+ *     7. SMALL ARGUMENTS
+ *
+ *     Inputs with |X| < 2^-252 are treated specially as
+ *     1 - |x|.
+ *
+ * Special cases:
+ *  cos(NaN) = quiet NaN, and raise invalid exception
+ *  cos(INF) = NaN and raise invalid exception
+ *  cos(0) = 1
+ *
+ */ +public final class AMD64MathCosOp extends AMD64MathIntrinsicUnaryOp { + + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64MathCosOp.class); + + public AMD64MathCosOp() { + super(TYPE, /* GPR */ rax, rcx, rdx, rbx, rsi, rdi, r8, r9, r10, r11, + /* XMM */ xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7); + } + + private ArrayDataPointerConstant onehalf = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x3fe00000, 0x00000000, 0x3fe00000 + // @formatter:on + }); + + private ArrayDataPointerConstant p2 = pointerConstant(16, new int[]{ + // @formatter:off + 0x1a600000, 0x3d90b461, 0x1a600000, 0x3d90b461 + // @formatter:on + }); + + private ArrayDataPointerConstant sc4 = pointerConstant(16, new int[]{ + // @formatter:off + 0xa556c734, 0x3ec71de3, 0x1a01a01a, 0x3efa01a0 + // @formatter:on + }); + + private ArrayDataPointerConstant ctable = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x3ff00000, 0x176d6d31, 0xbf73b92e, + 0xbc29b42c, 0x3fb917a6, 0xe0000000, 0xbc3e2718, 0x00000000, + 0x3ff00000, 0x011469fb, 0xbf93ad06, 0x3c69a60b, 0x3fc8f8b8, + 0xc0000000, 0xbc626d19, 0x00000000, 0x3ff00000, 0x939d225a, + 0xbfa60bea, 0x2ed59f06, 0x3fd29406, 0xa0000000, 0xbc75d28d, + 0x00000000, 0x3ff00000, 0x866b95cf, 0xbfb37ca1, 0xa6aea963, + 0x3fd87de2, 0xe0000000, 0xbc672ced, 0x00000000, 0x3ff00000, + 0x73fa1279, 0xbfbe3a68, 0x3806f63b, 0x3fde2b5d, 0x20000000, + 0x3c5e0d89, 0x00000000, 0x3ff00000, 0x5bc57974, 0xbfc59267, + 0x39ae68c8, 0x3fe1c73b, 0x20000000, 0x3c8b25dd, 0x00000000, + 0x3ff00000, 0x53aba2fd, 0xbfcd0dfe, 0x25091dd6, 0x3fe44cf3, + 0x20000000, 0x3c68076a, 0x00000000, 0x3ff00000, 0x99fcef32, + 0x3fca8279, 0x667f3bcd, 0x3fe6a09e, 0x20000000, 0xbc8bdd34, + 0x00000000, 0x3fe00000, 0x94247758, 0x3fc133cc, 0x6b151741, + 0x3fe8bc80, 0x20000000, 0xbc82c5e1, 0x00000000, 0x3fe00000, + 0x9ae68c87, 0x3fac73b3, 0x290ea1a3, 0x3fea9b66, 0xe0000000, + 0x3c39f630, 0x00000000, 0x3fe00000, 0x7f909c4e, 0xbf9d4a2c, + 0xf180bdb1, 0x3fec38b2, 0x80000000, 0xbc76e0b1, 0x00000000, + 0x3fe00000, 0x65455a75, 0xbfbe0875, 0xcf328d46, 0x3fed906b, + 0x20000000, 0x3c7457e6, 0x00000000, 0x3fe00000, 0x76acf82d, + 0x3fa4a031, 0x56c62dda, 0x3fee9f41, 0xe0000000, 0x3c8760b1, + 0x00000000, 0x3fd00000, 0x0e5967d5, 0xbfac1d1f, 0xcff75cb0, + 0x3fef6297, 0x20000000, 0x3c756217, 0x00000000, 0x3fd00000, + 0x0f592f50, 0xbf9ba165, 0xa3d12526, 0x3fefd88d, 0x40000000, + 0xbc887df6, 0x00000000, 0x3fc00000, 0x00000000, 0x00000000, + 0x00000000, 0x3ff00000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x0f592f50, 0x3f9ba165, 0xa3d12526, 0x3fefd88d, + 0x40000000, 0xbc887df6, 0x00000000, 0xbfc00000, 0x0e5967d5, + 0x3fac1d1f, 0xcff75cb0, 0x3fef6297, 0x20000000, 0x3c756217, + 0x00000000, 0xbfd00000, 0x76acf82d, 0xbfa4a031, 0x56c62dda, + 0x3fee9f41, 0xe0000000, 0x3c8760b1, 0x00000000, 0xbfd00000, + 0x65455a75, 0x3fbe0875, 0xcf328d46, 0x3fed906b, 0x20000000, + 0x3c7457e6, 0x00000000, 0xbfe00000, 0x7f909c4e, 0x3f9d4a2c, + 0xf180bdb1, 0x3fec38b2, 0x80000000, 0xbc76e0b1, 0x00000000, + 0xbfe00000, 0x9ae68c87, 0xbfac73b3, 0x290ea1a3, 0x3fea9b66, + 0xe0000000, 0x3c39f630, 0x00000000, 0xbfe00000, 0x94247758, + 0xbfc133cc, 0x6b151741, 0x3fe8bc80, 0x20000000, 0xbc82c5e1, + 0x00000000, 0xbfe00000, 0x99fcef32, 0xbfca8279, 0x667f3bcd, + 0x3fe6a09e, 0x20000000, 0xbc8bdd34, 0x00000000, 0xbfe00000, + 0x53aba2fd, 0x3fcd0dfe, 0x25091dd6, 0x3fe44cf3, 0x20000000, + 0x3c68076a, 0x00000000, 0xbff00000, 0x5bc57974, 0x3fc59267, + 0x39ae68c8, 0x3fe1c73b, 0x20000000, 0x3c8b25dd, 0x00000000, + 0xbff00000, 0x73fa1279, 0x3fbe3a68, 0x3806f63b, 0x3fde2b5d, + 0x20000000, 0x3c5e0d89, 0x00000000, 0xbff00000, 0x866b95cf, + 0x3fb37ca1, 0xa6aea963, 0x3fd87de2, 0xe0000000, 0xbc672ced, + 0x00000000, 0xbff00000, 0x939d225a, 0x3fa60bea, 0x2ed59f06, + 0x3fd29406, 0xa0000000, 0xbc75d28d, 0x00000000, 0xbff00000, + 0x011469fb, 0x3f93ad06, 0x3c69a60b, 0x3fc8f8b8, 0xc0000000, + 0xbc626d19, 0x00000000, 0xbff00000, 0x176d6d31, 0x3f73b92e, + 0xbc29b42c, 0x3fb917a6, 0xe0000000, 0xbc3e2718, 0x00000000, + 0xbff00000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0xbff00000, 0x176d6d31, + 0x3f73b92e, 0xbc29b42c, 0xbfb917a6, 0xe0000000, 0x3c3e2718, + 0x00000000, 0xbff00000, 0x011469fb, 0x3f93ad06, 0x3c69a60b, + 0xbfc8f8b8, 0xc0000000, 0x3c626d19, 0x00000000, 0xbff00000, + 0x939d225a, 0x3fa60bea, 0x2ed59f06, 0xbfd29406, 0xa0000000, + 0x3c75d28d, 0x00000000, 0xbff00000, 0x866b95cf, 0x3fb37ca1, + 0xa6aea963, 0xbfd87de2, 0xe0000000, 0x3c672ced, 0x00000000, + 0xbff00000, 0x73fa1279, 0x3fbe3a68, 0x3806f63b, 0xbfde2b5d, + 0x20000000, 0xbc5e0d89, 0x00000000, 0xbff00000, 0x5bc57974, + 0x3fc59267, 0x39ae68c8, 0xbfe1c73b, 0x20000000, 0xbc8b25dd, + 0x00000000, 0xbff00000, 0x53aba2fd, 0x3fcd0dfe, 0x25091dd6, + 0xbfe44cf3, 0x20000000, 0xbc68076a, 0x00000000, 0xbff00000, + 0x99fcef32, 0xbfca8279, 0x667f3bcd, 0xbfe6a09e, 0x20000000, + 0x3c8bdd34, 0x00000000, 0xbfe00000, 0x94247758, 0xbfc133cc, + 0x6b151741, 0xbfe8bc80, 0x20000000, 0x3c82c5e1, 0x00000000, + 0xbfe00000, 0x9ae68c87, 0xbfac73b3, 0x290ea1a3, 0xbfea9b66, + 0xe0000000, 0xbc39f630, 0x00000000, 0xbfe00000, 0x7f909c4e, + 0x3f9d4a2c, 0xf180bdb1, 0xbfec38b2, 0x80000000, 0x3c76e0b1, + 0x00000000, 0xbfe00000, 0x65455a75, 0x3fbe0875, 0xcf328d46, + 0xbfed906b, 0x20000000, 0xbc7457e6, 0x00000000, 0xbfe00000, + 0x76acf82d, 0xbfa4a031, 0x56c62dda, 0xbfee9f41, 0xe0000000, + 0xbc8760b1, 0x00000000, 0xbfd00000, 0x0e5967d5, 0x3fac1d1f, + 0xcff75cb0, 0xbfef6297, 0x20000000, 0xbc756217, 0x00000000, + 0xbfd00000, 0x0f592f50, 0x3f9ba165, 0xa3d12526, 0xbfefd88d, + 0x40000000, 0x3c887df6, 0x00000000, 0xbfc00000, 0x00000000, + 0x00000000, 0x00000000, 0xbff00000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x0f592f50, 0xbf9ba165, 0xa3d12526, + 0xbfefd88d, 0x40000000, 0x3c887df6, 0x00000000, 0x3fc00000, + 0x0e5967d5, 0xbfac1d1f, 0xcff75cb0, 0xbfef6297, 0x20000000, + 0xbc756217, 0x00000000, 0x3fd00000, 0x76acf82d, 0x3fa4a031, + 0x56c62dda, 0xbfee9f41, 0xe0000000, 0xbc8760b1, 0x00000000, + 0x3fd00000, 0x65455a75, 0xbfbe0875, 0xcf328d46, 0xbfed906b, + 0x20000000, 0xbc7457e6, 0x00000000, 0x3fe00000, 0x7f909c4e, + 0xbf9d4a2c, 0xf180bdb1, 0xbfec38b2, 0x80000000, 0x3c76e0b1, + 0x00000000, 0x3fe00000, 0x9ae68c87, 0x3fac73b3, 0x290ea1a3, + 0xbfea9b66, 0xe0000000, 0xbc39f630, 0x00000000, 0x3fe00000, + 0x94247758, 0x3fc133cc, 0x6b151741, 0xbfe8bc80, 0x20000000, + 0x3c82c5e1, 0x00000000, 0x3fe00000, 0x99fcef32, 0x3fca8279, + 0x667f3bcd, 0xbfe6a09e, 0x20000000, 0x3c8bdd34, 0x00000000, + 0x3fe00000, 0x53aba2fd, 0xbfcd0dfe, 0x25091dd6, 0xbfe44cf3, + 0x20000000, 0xbc68076a, 0x00000000, 0x3ff00000, 0x5bc57974, + 0xbfc59267, 0x39ae68c8, 0xbfe1c73b, 0x20000000, 0xbc8b25dd, + 0x00000000, 0x3ff00000, 0x73fa1279, 0xbfbe3a68, 0x3806f63b, + 0xbfde2b5d, 0x20000000, 0xbc5e0d89, 0x00000000, 0x3ff00000, + 0x866b95cf, 0xbfb37ca1, 0xa6aea963, 0xbfd87de2, 0xe0000000, + 0x3c672ced, 0x00000000, 0x3ff00000, 0x939d225a, 0xbfa60bea, + 0x2ed59f06, 0xbfd29406, 0xa0000000, 0x3c75d28d, 0x00000000, + 0x3ff00000, 0x011469fb, 0xbf93ad06, 0x3c69a60b, 0xbfc8f8b8, + 0xc0000000, 0x3c626d19, 0x00000000, 0x3ff00000, 0x176d6d31, + 0xbf73b92e, 0xbc29b42c, 0xbfb917a6, 0xe0000000, 0x3c3e2718, + 0x00000000, 0x3ff00000 + // @formatter:on + }); + + private ArrayDataPointerConstant sc2 = pointerConstant(16, new int[]{ + // @formatter:off + 0x11111111, 0x3f811111, 0x55555555, 0x3fa55555 + // @formatter:on + }); + + private ArrayDataPointerConstant sc3 = pointerConstant(16, new int[]{ + // @formatter:off + 0x1a01a01a, 0xbf2a01a0, 0x16c16c17, 0xbf56c16c + // @formatter:on + }); + + private ArrayDataPointerConstant sc1 = pointerConstant(16, new int[]{ + // @formatter:off + 0x55555555, 0xbfc55555, 0x00000000, 0xbfe00000 + // @formatter:on + }); + + private ArrayDataPointerConstant piInvTable = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x00000000, 0xa2f9836e, 0x4e441529, 0xfc2757d1, + 0xf534ddc0, 0xdb629599, 0x3c439041, 0xfe5163ab, 0xdebbc561, + 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c, 0xfe1deb1c, + 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41, + 0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, + 0xde05980f, 0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, + 0x4f463f66, 0x9e5fea2d, 0x7527bac7, 0xebe5f17b, 0x3d0739f7, + 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08, 0x56033046, 0xfc7b6bab, + 0xf0cfbc21 + // @formatter:on + }); + + private ArrayDataPointerConstant pi4 = pointerConstant(8, new int[]{ + // @formatter:off + 0x40000000, 0x3fe921fb, 0x18469899, 0x3e64442d + // @formatter:on + }); + + private ArrayDataPointerConstant pi48 = pointerConstant(8, new int[]{ + // @formatter:off + 0x18469899, 0x3e64442d + // @formatter:on + }); + + private ArrayDataPointerConstant pi32Inv = pointerConstant(8, new int[]{ + // @formatter:off + 0x6dc9c883, 0x40245f30 + // @formatter:on + }); + + private ArrayDataPointerConstant signMask = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x80000000 + // @formatter:on + }); + + private ArrayDataPointerConstant p3 = pointerConstant(8, new int[]{ + // @formatter:off + 0x2e037073, 0x3b63198a + // @formatter:on + }); + + private ArrayDataPointerConstant p1 = pointerConstant(8, new int[]{ + // @formatter:off + 0x54400000, 0x3fb921fb + // @formatter:on + }); + + private ArrayDataPointerConstant negZero = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x80000000 + // @formatter:on + }); + + // The 64 bit code is at most SSE2 compliant + private ArrayDataPointerConstant one = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x3ff00000 + // @formatter:on + }); + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + Label block0 = new Label(); + Label block1 = new Label(); + Label block2 = new Label(); + Label block3 = new Label(); + Label block4 = new Label(); + Label block5 = new Label(); + Label block6 = new Label(); + Label block7 = new Label(); + Label block8 = new Label(); + Label block9 = new Label(); + Label block10 = new Label(); + Label block11 = new Label(); + Label block12 = new Label(); + Label block13 = new Label(); + + masm.push(rbx); + masm.subq(rsp, 16); + masm.movsd(new AMD64Address(rsp, 8), xmm0); + + masm.movl(rax, new AMD64Address(rsp, 12)); + masm.movq(xmm1, recordExternalAddress(crb, pi32Inv)); // 0x6dc9c883, 0x40245f30 + masm.andl(rax, 2147418112); + masm.subl(rax, 808452096); + masm.cmpl(rax, 281346048); + masm.jcc(ConditionFlag.Above, block0); + masm.mulsd(xmm1, xmm0); + masm.movdqu(xmm5, recordExternalAddress(crb, onehalf)); // 0x00000000, 0x3fe00000, + // 0x00000000, 0x3fe00000 + masm.movq(xmm4, recordExternalAddress(crb, signMask)); // 0x00000000, 0x80000000 + masm.pand(xmm4, xmm0); + masm.por(xmm5, xmm4); + masm.addpd(xmm1, xmm5); + masm.cvttsd2sil(rdx, xmm1); + masm.cvtsi2sdl(xmm1, rdx); + masm.movdqu(xmm2, recordExternalAddress(crb, p2)); // 0x1a600000, 0x3d90b461, + // 0x1a600000, 0x3d90b461 + masm.movq(xmm3, recordExternalAddress(crb, p1)); // 0x54400000, 0x3fb921fb + masm.mulsd(xmm3, xmm1); + masm.unpcklpd(xmm1, xmm1); + masm.addq(rdx, 1865232); + masm.movdqu(xmm4, xmm0); + masm.andq(rdx, 63); + masm.movdqu(xmm5, recordExternalAddress(crb, sc4)); // 0xa556c734, 0x3ec71de3, + // 0x1a01a01a, 0x3efa01a0 + masm.leaq(rax, recordExternalAddress(crb, ctable)); + masm.shlq(rdx, 5); + masm.addq(rax, rdx); + masm.mulpd(xmm2, xmm1); + masm.subsd(xmm0, xmm3); + masm.mulsd(xmm1, recordExternalAddress(crb, p3)); // 0x2e037073, 0x3b63198a + masm.subsd(xmm4, xmm3); + masm.movq(xmm7, new AMD64Address(rax, 8)); + masm.unpcklpd(xmm0, xmm0); + masm.movdqu(xmm3, xmm4); + masm.subsd(xmm4, xmm2); + masm.mulpd(xmm5, xmm0); + masm.subpd(xmm0, xmm2); + masm.movdqu(xmm6, recordExternalAddress(crb, sc2)); // 0x11111111, 0x3f811111, + // 0x55555555, 0x3fa55555 + masm.mulsd(xmm7, xmm4); + masm.subsd(xmm3, xmm4); + masm.mulpd(xmm5, xmm0); + masm.mulpd(xmm0, xmm0); + masm.subsd(xmm3, xmm2); + masm.movdqu(xmm2, new AMD64Address(rax, 0)); + masm.subsd(xmm1, xmm3); + masm.movq(xmm3, new AMD64Address(rax, 24)); + masm.addsd(xmm2, xmm3); + masm.subsd(xmm7, xmm2); + masm.mulsd(xmm2, xmm4); + masm.mulpd(xmm6, xmm0); + masm.mulsd(xmm3, xmm4); + masm.mulpd(xmm2, xmm0); + masm.mulpd(xmm0, xmm0); + masm.addpd(xmm5, recordExternalAddress(crb, sc3)); // 0x1a01a01a, 0xbf2a01a0, + // 0x16c16c17, 0xbf56c16c + masm.mulsd(xmm4, new AMD64Address(rax, 0)); + masm.addpd(xmm6, recordExternalAddress(crb, sc1)); // 0x55555555, 0xbfc55555, + // 0x00000000, 0xbfe00000 + masm.mulpd(xmm5, xmm0); + masm.movdqu(xmm0, xmm3); + masm.addsd(xmm3, new AMD64Address(rax, 8)); + masm.mulpd(xmm1, xmm7); + masm.movdqu(xmm7, xmm4); + masm.addsd(xmm4, xmm3); + masm.addpd(xmm6, xmm5); + masm.movq(xmm5, new AMD64Address(rax, 8)); + masm.subsd(xmm5, xmm3); + masm.subsd(xmm3, xmm4); + masm.addsd(xmm1, new AMD64Address(rax, 16)); + masm.mulpd(xmm6, xmm2); + masm.addsd(xmm0, xmm5); + masm.addsd(xmm3, xmm7); + masm.addsd(xmm0, xmm1); + masm.addsd(xmm0, xmm3); + masm.addsd(xmm0, xmm6); + masm.unpckhpd(xmm6, xmm6); + masm.addsd(xmm0, xmm6); + masm.addsd(xmm0, xmm4); + masm.jmp(block13); + + masm.bind(block0); + masm.jcc(ConditionFlag.Greater, block1); + masm.pextrw(rax, xmm0, 3); + masm.andl(rax, 32767); + masm.pinsrw(xmm0, rax, 3); + masm.movq(xmm1, recordExternalAddress(crb, one)); // 0x00000000, 0x3ff00000 + masm.subsd(xmm1, xmm0); + masm.movdqu(xmm0, xmm1); + masm.jmp(block13); + + masm.bind(block1); + masm.pextrw(rax, xmm0, 3); + masm.andl(rax, 32752); + masm.cmpl(rax, 32752); + masm.jcc(ConditionFlag.Equal, block2); + masm.pextrw(rcx, xmm0, 3); + masm.andl(rcx, 32752); + masm.subl(rcx, 16224); + masm.shrl(rcx, 7); + masm.andl(rcx, 65532); + masm.leaq(r11, recordExternalAddress(crb, piInvTable)); + masm.addq(rcx, r11); + masm.movdq(rax, xmm0); + masm.movl(r10, new AMD64Address(rcx, 20)); + masm.movl(r8, new AMD64Address(rcx, 24)); + masm.movl(rdx, rax); + masm.shrq(rax, 21); + masm.orl(rax, Integer.MIN_VALUE); + masm.shrl(rax, 11); + masm.movl(r9, r10); + masm.imulq(r10, rdx); + masm.imulq(r9, rax); + masm.imulq(r8, rax); + masm.movl(rsi, new AMD64Address(rcx, 16)); + masm.movl(rdi, new AMD64Address(rcx, 12)); + masm.movl(r11, r10); + masm.shrq(r10, 32); + masm.addq(r9, r10); + masm.addq(r11, r8); + masm.movl(r8, r11); + masm.shrq(r11, 32); + masm.addq(r9, r11); + masm.movl(r10, rsi); + masm.imulq(rsi, rdx); + masm.imulq(r10, rax); + masm.movl(r11, rdi); + masm.imulq(rdi, rdx); + masm.movl(rbx, rsi); + masm.shrq(rsi, 32); + masm.addq(r9, rbx); + masm.movl(rbx, r9); + masm.shrq(r9, 32); + masm.addq(r10, rsi); + masm.addq(r10, r9); + masm.shlq(rbx, 32); + masm.orq(r8, rbx); + masm.imulq(r11, rax); + masm.movl(r9, new AMD64Address(rcx, 8)); + masm.movl(rsi, new AMD64Address(rcx, 4)); + masm.movl(rbx, rdi); + masm.shrq(rdi, 32); + masm.addq(r10, rbx); + masm.movl(rbx, r10); + masm.shrq(r10, 32); + masm.addq(r11, rdi); + masm.addq(r11, r10); + masm.movq(rdi, r9); + masm.imulq(r9, rdx); + masm.imulq(rdi, rax); + masm.movl(r10, r9); + masm.shrq(r9, 32); + masm.addq(r11, r10); + masm.movl(r10, r11); + masm.shrq(r11, 32); + masm.addq(rdi, r9); + masm.addq(rdi, r11); + masm.movq(r9, rsi); + masm.imulq(rsi, rdx); + masm.imulq(r9, rax); + masm.shlq(r10, 32); + masm.orq(r10, rbx); + masm.movl(rax, new AMD64Address(rcx, 0)); + masm.movl(r11, rsi); + masm.shrq(rsi, 32); + masm.addq(rdi, r11); + masm.movl(r11, rdi); + masm.shrq(rdi, 32); + masm.addq(r9, rsi); + masm.addq(r9, rdi); + masm.imulq(rdx, rax); + masm.pextrw(rbx, xmm0, 3); + masm.leaq(rdi, recordExternalAddress(crb, piInvTable)); + masm.subq(rcx, rdi); + masm.addl(rcx, rcx); + masm.addl(rcx, rcx); + masm.addl(rcx, rcx); + masm.addl(rcx, 19); + masm.movl(rsi, 32768); + masm.andl(rsi, rbx); + masm.shrl(rbx, 4); + masm.andl(rbx, 2047); + masm.subl(rbx, 1023); + masm.subl(rcx, rbx); + masm.addq(r9, rdx); + masm.movl(rdx, rcx); + masm.addl(rdx, 32); + masm.cmpl(rcx, 1); + masm.jcc(ConditionFlag.Less, block3); + masm.negl(rcx); + masm.addl(rcx, 29); + masm.shll(r9); + masm.movl(rdi, r9); + masm.andl(r9, 536870911); + masm.testl(r9, 268435456); + masm.jcc(ConditionFlag.NotEqual, block4); + masm.shrl(r9); + masm.movl(rbx, 0); + masm.shlq(r9, 32); + masm.orq(r9, r11); + + masm.bind(block5); + + masm.bind(block6); + masm.cmpq(r9, 0); + masm.jcc(ConditionFlag.Equal, block7); + + masm.bind(block8); + masm.bsrq(r11, r9); + masm.movl(rcx, 29); + masm.subl(rcx, r11); + masm.jcc(ConditionFlag.LessEqual, block9); + masm.shlq(r9); + masm.movq(rax, r10); + masm.shlq(r10); + masm.addl(rdx, rcx); + masm.negl(rcx); + masm.addl(rcx, 64); + masm.shrq(rax); + masm.shrq(r8); + masm.orq(r9, rax); + masm.orq(r10, r8); + + masm.bind(block10); + masm.cvtsi2sdq(xmm0, r9); + masm.shrq(r10, 1); + masm.cvtsi2sdq(xmm3, r10); + masm.xorpd(xmm4, xmm4); + masm.shll(rdx, 4); + masm.negl(rdx); + masm.addl(rdx, 16368); + masm.orl(rdx, rsi); + masm.xorl(rdx, rbx); + masm.pinsrw(xmm4, rdx, 3); + masm.movq(xmm2, recordExternalAddress(crb, pi4)); // 0x40000000, 0x3fe921fb, + // 0x18469899, 0x3e64442d + masm.movq(xmm6, recordExternalAddress(crb, pi48)); // 0x3fe921fb, 0x18469899, + // 0x3e64442d + masm.xorpd(xmm5, xmm5); + masm.subl(rdx, 1008); + masm.pinsrw(xmm5, rdx, 3); + masm.mulsd(xmm0, xmm4); + masm.shll(rsi, 16); + masm.sarl(rsi, 31); + masm.mulsd(xmm3, xmm5); + masm.movdqu(xmm1, xmm0); + masm.mulsd(xmm0, xmm2); + masm.shrl(rdi, 29); + masm.addsd(xmm1, xmm3); + masm.mulsd(xmm3, xmm2); + masm.addl(rdi, rsi); + masm.xorl(rdi, rsi); + masm.mulsd(xmm6, xmm1); + masm.movl(rax, rdi); + masm.addsd(xmm6, xmm3); + masm.movdqu(xmm2, xmm0); + masm.addsd(xmm0, xmm6); + masm.subsd(xmm2, xmm0); + masm.addsd(xmm6, xmm2); + + masm.bind(block11); + masm.movq(xmm1, recordExternalAddress(crb, pi32Inv)); // 0x6dc9c883, 0x40245f30 + masm.mulsd(xmm1, xmm0); + masm.movq(xmm5, recordExternalAddress(crb, onehalf)); // 0x00000000, 0x3fe00000, + // 0x00000000, 0x3fe00000 + masm.movq(xmm4, recordExternalAddress(crb, signMask)); // 0x00000000, 0x80000000 + masm.pand(xmm4, xmm0); + masm.por(xmm5, xmm4); + masm.addpd(xmm1, xmm5); + masm.cvttsd2siq(rdx, xmm1); + masm.cvtsi2sdq(xmm1, rdx); + masm.movq(xmm3, recordExternalAddress(crb, p1)); // 0x54400000, 0x3fb921fb + masm.movdqu(xmm2, recordExternalAddress(crb, p2)); // 0x1a600000, 0x3d90b461, + // 0x1a600000, 0x3d90b461 + masm.mulsd(xmm3, xmm1); + masm.unpcklpd(xmm1, xmm1); + masm.shll(rax, 3); + masm.addl(rdx, 1865232); + masm.movdqu(xmm4, xmm0); + masm.addl(rdx, rax); + masm.andl(rdx, 63); + masm.movdqu(xmm5, recordExternalAddress(crb, sc4)); // 0xa556c734, 0x3ec71de3, + // 0x1a01a01a, 0x3efa01a0 + masm.leaq(rax, recordExternalAddress(crb, ctable)); + masm.shll(rdx, 5); + masm.addq(rax, rdx); + masm.mulpd(xmm2, xmm1); + masm.subsd(xmm0, xmm3); + masm.mulsd(xmm1, recordExternalAddress(crb, p3)); // 0x2e037073, 0x3b63198a + masm.subsd(xmm4, xmm3); + masm.movq(xmm7, new AMD64Address(rax, 8)); + masm.unpcklpd(xmm0, xmm0); + masm.movdqu(xmm3, xmm4); + masm.subsd(xmm4, xmm2); + masm.mulpd(xmm5, xmm0); + masm.subpd(xmm0, xmm2); + masm.mulsd(xmm7, xmm4); + masm.subsd(xmm3, xmm4); + masm.mulpd(xmm5, xmm0); + masm.mulpd(xmm0, xmm0); + masm.subsd(xmm3, xmm2); + masm.movdqu(xmm2, new AMD64Address(rax, 0)); + masm.subsd(xmm1, xmm3); + masm.movq(xmm3, new AMD64Address(rax, 24)); + masm.addsd(xmm2, xmm3); + masm.subsd(xmm7, xmm2); + masm.subsd(xmm1, xmm6); + masm.movdqu(xmm6, recordExternalAddress(crb, sc2)); // 0x11111111, 0x3f811111, + // 0x55555555, 0x3fa55555 + masm.mulsd(xmm2, xmm4); + masm.mulpd(xmm6, xmm0); + masm.mulsd(xmm3, xmm4); + masm.mulpd(xmm2, xmm0); + masm.mulpd(xmm0, xmm0); + masm.addpd(xmm5, recordExternalAddress(crb, sc3)); // 0x1a01a01a, 0xbf2a01a0, + // 0x16c16c17, 0xbf56c16c + masm.mulsd(xmm4, new AMD64Address(rax, 0)); + masm.addpd(xmm6, recordExternalAddress(crb, sc1)); // 0x55555555, 0xbfc55555, + // 0x00000000, 0xbfe00000 + masm.mulpd(xmm5, xmm0); + masm.movdqu(xmm0, xmm3); + masm.addsd(xmm3, new AMD64Address(rax, 8)); + masm.mulpd(xmm1, xmm7); + masm.movdqu(xmm7, xmm4); + masm.addsd(xmm4, xmm3); + masm.addpd(xmm6, xmm5); + masm.movq(xmm5, new AMD64Address(rax, 8)); + masm.subsd(xmm5, xmm3); + masm.subsd(xmm3, xmm4); + masm.addsd(xmm1, new AMD64Address(rax, 16)); + masm.mulpd(xmm6, xmm2); + masm.addsd(xmm5, xmm0); + masm.addsd(xmm3, xmm7); + masm.addsd(xmm1, xmm5); + masm.addsd(xmm1, xmm3); + masm.addsd(xmm1, xmm6); + masm.unpckhpd(xmm6, xmm6); + masm.movdqu(xmm0, xmm4); + masm.addsd(xmm1, xmm6); + masm.addsd(xmm0, xmm1); + masm.jmp(block13); + + masm.bind(block7); + masm.addl(rdx, 64); + masm.movq(r9, r10); + masm.movq(r10, r8); + masm.movl(r8, 0); + masm.cmpq(r9, 0); + masm.jcc(ConditionFlag.NotEqual, block8); + masm.addl(rdx, 64); + masm.movq(r9, r10); + masm.movq(r10, r8); + masm.cmpq(r9, 0); + masm.jcc(ConditionFlag.NotEqual, block8); + masm.xorpd(xmm0, xmm0); + masm.xorpd(xmm6, xmm6); + masm.jmp(block11); + + masm.bind(block9); + masm.jcc(ConditionFlag.Equal, block10); + masm.negl(rcx); + masm.shrq(r10); + masm.movq(rax, r9); + masm.shrq(r9); + masm.subl(rdx, rcx); + masm.negl(rcx); + masm.addl(rcx, 64); + masm.shlq(rax); + masm.orq(r10, rax); + masm.jmp(block10); + masm.bind(block3); + masm.negl(rcx); + masm.shlq(r9, 32); + masm.orq(r9, r11); + masm.shlq(r9); + masm.movq(rdi, r9); + masm.testl(r9, Integer.MIN_VALUE); + masm.jcc(ConditionFlag.NotEqual, block12); + masm.shrl(r9); + masm.movl(rbx, 0); + masm.shrq(rdi, 3); + masm.jmp(block6); + + masm.bind(block4); + masm.shrl(r9); + masm.movl(rbx, 536870912); + masm.shrl(rbx); + masm.shlq(r9, 32); + masm.orq(r9, r11); + masm.shlq(rbx, 32); + masm.addl(rdi, 536870912); + masm.movl(rcx, 0); + masm.movl(r11, 0); + masm.subq(rcx, r8); + masm.sbbq(r11, r10); + masm.sbbq(rbx, r9); + masm.movq(r8, rcx); + masm.movq(r10, r11); + masm.movq(r9, rbx); + masm.movl(rbx, 32768); + masm.jmp(block5); + + masm.bind(block12); + masm.shrl(r9); + masm.movq(rbx, 0x100000000L); + masm.shrq(rbx); + masm.movl(rcx, 0); + masm.movl(r11, 0); + masm.subq(rcx, r8); + masm.sbbq(r11, r10); + masm.sbbq(rbx, r9); + masm.movq(r8, rcx); + masm.movq(r10, r11); + masm.movq(r9, rbx); + masm.movl(rbx, 32768); + masm.shrq(rdi, 3); + masm.addl(rdi, 536870912); + masm.jmp(block6); + + masm.bind(block2); + masm.movsd(xmm0, new AMD64Address(rsp, 8)); + masm.mulsd(xmm0, recordExternalAddress(crb, negZero)); // 0x00000000, 0x80000000 + masm.movq(new AMD64Address(rsp, 0), xmm0); + + masm.bind(block13); + masm.addq(rsp, 16); + masm.pop(rbx); + } + +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathExpOp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathExpOp.java new file mode 100644 index 00000000000..e943817648e --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathExpOp.java @@ -0,0 +1,459 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, Intel Corporation. All rights reserved. + * Intel Math Library (LIBM) Source Code + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.amd64; + +import static jdk.vm.ci.amd64.AMD64.r11; +import static jdk.vm.ci.amd64.AMD64.rax; +import static jdk.vm.ci.amd64.AMD64.rcx; +import static jdk.vm.ci.amd64.AMD64.rdx; +import static jdk.vm.ci.amd64.AMD64.rsp; +import static jdk.vm.ci.amd64.AMD64.xmm0; +import static jdk.vm.ci.amd64.AMD64.xmm1; +import static jdk.vm.ci.amd64.AMD64.xmm2; +import static jdk.vm.ci.amd64.AMD64.xmm3; +import static jdk.vm.ci.amd64.AMD64.xmm4; +import static jdk.vm.ci.amd64.AMD64.xmm5; +import static jdk.vm.ci.amd64.AMD64.xmm6; +import static jdk.vm.ci.amd64.AMD64.xmm7; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.pointerConstant; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.recordExternalAddress; + +import org.graalvm.compiler.asm.Label; +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; +import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.lir.LIRInstructionClass; +import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; + +/** + * 
+ *                     ALGORITHM DESCRIPTION - EXP()
+ *                     ---------------------
+ *
+ * Description:
+ *  Let K = 64 (table size).
+ *        x    x/log(2)     n
+ *       e  = 2          = 2 * T[j] * (1 + P(y))
+ *  where
+ *       x = m*log(2)/K + y,    y in [-log(2)/K..log(2)/K]
+ *       m = n*K + j,           m,n,j - signed integer, j in [-K/2..K/2]
+ *                  j/K
+ *       values of 2   are tabulated as T[j] = T_hi[j] ( 1 + T_lo[j]).
+ *
+ *       P(y) is a minimax polynomial approximation of exp(x)-1
+ *       on small interval [-log(2)/K..log(2)/K] (were calculated by Maple V).
+ *
+ *  To avoid problems with arithmetic overflow and underflow,
+ *            n                        n1  n2
+ *  value of 2  is safely computed as 2 * 2 where n1 in [-BIAS/2..BIAS/2]
+ *  where BIAS is a value of exponent bias.
+ *
+ * Special cases:
+ *  exp(NaN) = NaN
+ *  exp(+INF) = +INF
+ *  exp(-INF) = 0
+ *  exp(x) = 1 for subnormals
+ *  for finite argument, only exp(0)=1 is exact
+ *  For IEEE double
+ *    if x >  709.782712893383973096 then exp(x) overflow
+ *    if x < -745.133219101941108420 then exp(x) underflow
+ *
+ */ +public final class AMD64MathExpOp extends AMD64MathIntrinsicUnaryOp { + + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64MathExpOp.class); + + public AMD64MathExpOp() { + super(TYPE, /* GPR */ rax, rcx, rdx, r11, + /* XMM */ xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7); + } + + private ArrayDataPointerConstant cv = pointerConstant(16, new int[]{ + // @formatter:off + 0x652b82fe, 0x40571547, 0x652b82fe, 0x40571547, + }); + private ArrayDataPointerConstant cv16 = pointerConstant(16, new int[]{ + 0xfefa0000, 0x3f862e42, 0xfefa0000, 0x3f862e42, + }); + private ArrayDataPointerConstant cv32 = pointerConstant(16, new int[]{ + 0xbc9e3b3a, 0x3d1cf79a, 0xbc9e3b3a, 0x3d1cf79a, + }); + private ArrayDataPointerConstant cv48 = pointerConstant(16, new int[]{ + 0xfffffffe, 0x3fdfffff, 0xfffffffe, 0x3fdfffff, + }); + private ArrayDataPointerConstant cv64 = pointerConstant(16, new int[]{ + 0xe3289860, 0x3f56c15c, 0x555b9e25, 0x3fa55555, + }); + private ArrayDataPointerConstant cv80 = pointerConstant(16, new int[]{ + 0xc090cf0f, 0x3f811115, 0x55548ba1, 0x3fc55555 + // @formatter:on + }); + + private ArrayDataPointerConstant shifter = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x43380000, 0x00000000, 0x43380000 + // @formatter:on + }); + + private ArrayDataPointerConstant mmask = pointerConstant(16, new int[]{ + // @formatter:off + 0xffffffc0, 0x00000000, 0xffffffc0, 0x00000000 + // @formatter:on + }); + + private ArrayDataPointerConstant bias = pointerConstant(16, new int[]{ + // @formatter:off + 0x0000ffc0, 0x00000000, 0x0000ffc0, 0x00000000 + // @formatter:on + }); + + private ArrayDataPointerConstant tblAddr = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x0e03754d, + 0x3cad7bbf, 0x3e778060, 0x00002c9a, 0x3567f613, 0x3c8cd252, + 0xd3158574, 0x000059b0, 0x61e6c861, 0x3c60f74e, 0x18759bc8, + 0x00008745, 0x5d837b6c, 0x3c979aa6, 0x6cf9890f, 0x0000b558, + 0x702f9cd1, 0x3c3ebe3d, 0x32d3d1a2, 0x0000e3ec, 0x1e63bcd8, + 0x3ca3516e, 0xd0125b50, 0x00011301, 0x26f0387b, 0x3ca4c554, + 0xaea92ddf, 0x0001429a, 0x62523fb6, 0x3ca95153, 0x3c7d517a, + 0x000172b8, 0x3f1353bf, 0x3c8b898c, 0xeb6fcb75, 0x0001a35b, + 0x3e3a2f5f, 0x3c9aecf7, 0x3168b9aa, 0x0001d487, 0x44a6c38d, + 0x3c8a6f41, 0x88628cd6, 0x0002063b, 0xe3a8a894, 0x3c968efd, + 0x6e756238, 0x0002387a, 0x981fe7f2, 0x3c80472b, 0x65e27cdd, + 0x00026b45, 0x6d09ab31, 0x3c82f7e1, 0xf51fdee1, 0x00029e9d, + 0x720c0ab3, 0x3c8b3782, 0xa6e4030b, 0x0002d285, 0x4db0abb6, + 0x3c834d75, 0x0a31b715, 0x000306fe, 0x5dd3f84a, 0x3c8fdd39, + 0xb26416ff, 0x00033c08, 0xcc187d29, 0x3ca12f8c, 0x373aa9ca, + 0x000371a7, 0x738b5e8b, 0x3ca7d229, 0x34e59ff6, 0x0003a7db, + 0xa72a4c6d, 0x3c859f48, 0x4c123422, 0x0003dea6, 0x259d9205, + 0x3ca8b846, 0x21f72e29, 0x0004160a, 0x60c2ac12, 0x3c4363ed, + 0x6061892d, 0x00044e08, 0xdaa10379, 0x3c6ecce1, 0xb5c13cd0, + 0x000486a2, 0xbb7aafb0, 0x3c7690ce, 0xd5362a27, 0x0004bfda, + 0x9b282a09, 0x3ca083cc, 0x769d2ca6, 0x0004f9b2, 0xc1aae707, + 0x3ca509b0, 0x569d4f81, 0x0005342b, 0x18fdd78e, 0x3c933505, + 0x36b527da, 0x00056f47, 0xe21c5409, 0x3c9063e1, 0xdd485429, + 0x0005ab07, 0x2b64c035, 0x3c9432e6, 0x15ad2148, 0x0005e76f, + 0x99f08c0a, 0x3ca01284, 0xb03a5584, 0x0006247e, 0x0073dc06, + 0x3c99f087, 0x82552224, 0x00066238, 0x0da05571, 0x3c998d4d, + 0x667f3bcc, 0x0006a09e, 0x86ce4786, 0x3ca52bb9, 0x3c651a2e, + 0x0006dfb2, 0x206f0dab, 0x3ca32092, 0xe8ec5f73, 0x00071f75, + 0x8e17a7a6, 0x3ca06122, 0x564267c8, 0x00075feb, 0x461e9f86, + 0x3ca244ac, 0x73eb0186, 0x0007a114, 0xabd66c55, 0x3c65ebe1, + 0x36cf4e62, 0x0007e2f3, 0xbbff67d0, 0x3c96fe9f, 0x994cce12, + 0x00082589, 0x14c801df, 0x3c951f14, 0x9b4492ec, 0x000868d9, + 0xc1f0eab4, 0x3c8db72f, 0x422aa0db, 0x0008ace5, 0x59f35f44, + 0x3c7bf683, 0x99157736, 0x0008f1ae, 0x9c06283c, 0x3ca360ba, + 0xb0cdc5e4, 0x00093737, 0x20f962aa, 0x3c95e8d1, 0x9fde4e4f, + 0x00097d82, 0x2b91ce27, 0x3c71affc, 0x82a3f090, 0x0009c491, + 0x589a2ebd, 0x3c9b6d34, 0x7b5de564, 0x000a0c66, 0x9ab89880, + 0x3c95277c, 0xb23e255c, 0x000a5503, 0x6e735ab3, 0x3c846984, + 0x5579fdbf, 0x000a9e6b, 0x92cb3387, 0x3c8c1a77, 0x995ad3ad, + 0x000ae89f, 0xdc2d1d96, 0x3ca22466, 0xb84f15fa, 0x000b33a2, + 0xb19505ae, 0x3ca1112e, 0xf2fb5e46, 0x000b7f76, 0x0a5fddcd, + 0x3c74ffd7, 0x904bc1d2, 0x000bcc1e, 0x30af0cb3, 0x3c736eae, + 0xdd85529c, 0x000c199b, 0xd10959ac, 0x3c84e08f, 0x2e57d14b, + 0x000c67f1, 0x6c921968, 0x3c676b2c, 0xdcef9069, 0x000cb720, + 0x36df99b3, 0x3c937009, 0x4a07897b, 0x000d072d, 0xa63d07a7, + 0x3c74a385, 0xdcfba487, 0x000d5818, 0xd5c192ac, 0x3c8e5a50, + 0x03db3285, 0x000da9e6, 0x1c4a9792, 0x3c98bb73, 0x337b9b5e, + 0x000dfc97, 0x603a88d3, 0x3c74b604, 0xe78b3ff6, 0x000e502e, + 0x92094926, 0x3c916f27, 0xa2a490d9, 0x000ea4af, 0x41aa2008, + 0x3c8ec3bc, 0xee615a27, 0x000efa1b, 0x31d185ee, 0x3c8a64a9, + 0x5b6e4540, 0x000f5076, 0x4d91cd9d, 0x3c77893b, 0x819e90d8, + 0x000fa7c1 + // @formatter:on + }); + + private ArrayDataPointerConstant allones = pointerConstant(16, new int[]{ + // @formatter:off + 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff + // @formatter:on + }); + + private ArrayDataPointerConstant ebias = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x3ff00000, 0x00000000, 0x3ff00000 + // @formatter:on + }); + + private ArrayDataPointerConstant xmax = pointerConstant(4, new int[]{ + // @formatter:off + 0xffffffff, 0x7fefffff + // @formatter:on + }); + + private ArrayDataPointerConstant xmin = pointerConstant(4, new int[]{ + // @formatter:off + 0x00000000, 0x00100000 + // @formatter:on + }); + + private ArrayDataPointerConstant inf = pointerConstant(4, new int[]{ + // @formatter:off + 0x00000000, 0x7ff00000 + // @formatter:on + }); + + private ArrayDataPointerConstant zero = pointerConstant(4, new int[]{ + // @formatter:off + 0x00000000, 0x00000000 + // @formatter:on + }); + + private ArrayDataPointerConstant oneVal = pointerConstant(4, new int[]{ + // @formatter:off + 0x00000000, 0x3ff00000 + // @formatter:on + }); + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + // Registers: + // input: xmm0 + // scratch: xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 + // rax, rdx, rcx, tmp - r11 + + // Code generated by Intel C compiler for LIBM library + Label block0 = new Label(); + Label block1 = new Label(); + Label block2 = new Label(); + Label block3 = new Label(); + Label block4 = new Label(); + Label block5 = new Label(); + Label block6 = new Label(); + Label block7 = new Label(); + Label block8 = new Label(); + Label block9 = new Label(); + Label block10 = new Label(); + Label block11 = new Label(); + Label block12 = new Label(); + Label block13 = new Label(); + + masm.subq(rsp, 24); + masm.movsd(new AMD64Address(rsp, 8), xmm0); + masm.unpcklpd(xmm0, xmm0); + masm.movdqu(xmm1, recordExternalAddress(crb, cv)); // 0x652b82fe, 0x40571547, + // 0x652b82fe, 0x40571547 + masm.movdqu(xmm6, recordExternalAddress(crb, shifter)); // 0x00000000, 0x43380000, + // 0x00000000, 0x43380000 + masm.movdqu(xmm2, recordExternalAddress(crb, cv16)); // 0xfefa0000, 0x3f862e42, + // 0xfefa0000, 0x3f862e42 + masm.movdqu(xmm3, recordExternalAddress(crb, cv32)); // 0xbc9e3b3a, 0x3d1cf79a, + // 0xbc9e3b3a, 0x3d1cf79a + masm.pextrw(rax, xmm0, 3); + masm.andl(rax, 32767); + masm.movl(rdx, 16527); + masm.subl(rdx, rax); + masm.subl(rax, 15504); + masm.orl(rdx, rax); + masm.cmpl(rdx, Integer.MIN_VALUE); + masm.jcc(ConditionFlag.AboveEqual, block0); + masm.mulpd(xmm1, xmm0); + masm.addpd(xmm1, xmm6); + masm.movapd(xmm7, xmm1); + masm.subpd(xmm1, xmm6); + masm.mulpd(xmm2, xmm1); + masm.movdqu(xmm4, recordExternalAddress(crb, cv64)); // 0xe3289860, 0x3f56c15c, + // 0x555b9e25, 0x3fa55555 + masm.mulpd(xmm3, xmm1); + masm.movdqu(xmm5, recordExternalAddress(crb, cv80)); // 0xc090cf0f, 0x3f811115, + // 0x55548ba1, 0x3fc55555 + masm.subpd(xmm0, xmm2); + masm.movdl(rax, xmm7); + masm.movl(rcx, rax); + masm.andl(rcx, 63); + masm.shll(rcx, 4); + masm.sarl(rax, 6); + masm.movl(rdx, rax); + masm.movdqu(xmm6, recordExternalAddress(crb, mmask)); // 0xffffffc0, 0x00000000, + // 0xffffffc0, 0x00000000 + masm.pand(xmm7, xmm6); + masm.movdqu(xmm6, recordExternalAddress(crb, bias)); // 0x0000ffc0, 0x00000000, + // 0x0000ffc0, 0x00000000 + masm.paddq(xmm7, xmm6); + masm.psllq(xmm7, 46); + masm.subpd(xmm0, xmm3); + masm.leaq(r11, recordExternalAddress(crb, tblAddr)); + masm.movdqu(xmm2, new AMD64Address(rcx, r11, AMD64Address.Scale.Times1)); + masm.mulpd(xmm4, xmm0); + masm.movapd(xmm6, xmm0); + masm.movapd(xmm1, xmm0); + masm.mulpd(xmm6, xmm6); + masm.mulpd(xmm0, xmm6); + masm.addpd(xmm5, xmm4); + masm.mulsd(xmm0, xmm6); + masm.mulpd(xmm6, recordExternalAddress(crb, cv48)); // 0xfffffffe, 0x3fdfffff, + // 0xfffffffe, 0x3fdfffff + masm.addsd(xmm1, xmm2); + masm.unpckhpd(xmm2, xmm2); + masm.mulpd(xmm0, xmm5); + masm.addsd(xmm1, xmm0); + masm.por(xmm2, xmm7); + masm.unpckhpd(xmm0, xmm0); + masm.addsd(xmm0, xmm1); + masm.addsd(xmm0, xmm6); + masm.addl(rdx, 894); + masm.cmpl(rdx, 1916); + masm.jcc(ConditionFlag.Above, block1); + masm.mulsd(xmm0, xmm2); + masm.addsd(xmm0, xmm2); + masm.jmp(block13); + + masm.bind(block1); + masm.xorpd(xmm3, xmm3); + masm.movdqu(xmm4, recordExternalAddress(crb, allones)); // 0xffffffff, 0xffffffff, + // 0xffffffff, 0xffffffff + masm.movl(rdx, -1022); + masm.subl(rdx, rax); + masm.movdl(xmm5, rdx); + masm.psllq(xmm4, xmm5); + masm.movl(rcx, rax); + masm.sarl(rax, 1); + masm.pinsrw(xmm3, rax, 3); + masm.movdqu(xmm6, recordExternalAddress(crb, ebias)); // 0x00000000, 0x3ff00000, + // 0x00000000, 0x3ff00000 + masm.psllq(xmm3, 4); + masm.psubd(xmm2, xmm3); + masm.mulsd(xmm0, xmm2); + masm.cmpl(rdx, 52); + masm.jcc(ConditionFlag.Greater, block2); + masm.pand(xmm4, xmm2); + masm.paddd(xmm3, xmm6); + masm.subsd(xmm2, xmm4); + masm.addsd(xmm0, xmm2); + masm.cmpl(rcx, 1023); + masm.jcc(ConditionFlag.GreaterEqual, block3); + masm.pextrw(rcx, xmm0, 3); + masm.andl(rcx, 32768); + masm.orl(rdx, rcx); + masm.cmpl(rdx, 0); + masm.jcc(ConditionFlag.Equal, block4); + masm.movapd(xmm6, xmm0); + masm.addsd(xmm0, xmm4); + masm.mulsd(xmm0, xmm3); + masm.pextrw(rcx, xmm0, 3); + masm.andl(rcx, 32752); + masm.cmpl(rcx, 0); + masm.jcc(ConditionFlag.Equal, block5); + masm.jmp(block13); + + masm.bind(block5); + masm.mulsd(xmm6, xmm3); + masm.mulsd(xmm4, xmm3); + masm.movdqu(xmm0, xmm6); + masm.pxor(xmm6, xmm4); + masm.psrad(xmm6, 31); + masm.pshufd(xmm6, xmm6, 85); + masm.psllq(xmm0, 1); + masm.psrlq(xmm0, 1); + masm.pxor(xmm0, xmm6); + masm.psrlq(xmm6, 63); + masm.paddq(xmm0, xmm6); + masm.paddq(xmm0, xmm4); + masm.movl(new AMD64Address(rsp, 0), 15); + masm.jmp(block6); + + masm.bind(block4); + masm.addsd(xmm0, xmm4); + masm.mulsd(xmm0, xmm3); + masm.jmp(block13); + + masm.bind(block3); + masm.addsd(xmm0, xmm4); + masm.mulsd(xmm0, xmm3); + masm.pextrw(rcx, xmm0, 3); + masm.andl(rcx, 32752); + masm.cmpl(rcx, 32752); + masm.jcc(ConditionFlag.AboveEqual, block7); + masm.jmp(block13); + + masm.bind(block2); + masm.paddd(xmm3, xmm6); + masm.addpd(xmm0, xmm2); + masm.mulsd(xmm0, xmm3); + masm.movl(new AMD64Address(rsp, 0), 15); + masm.jmp(block6); + + masm.bind(block8); + masm.cmpl(rax, 2146435072); + masm.jcc(ConditionFlag.AboveEqual, block9); + masm.movl(rax, new AMD64Address(rsp, 12)); + masm.cmpl(rax, Integer.MIN_VALUE); + masm.jcc(ConditionFlag.AboveEqual, block10); + masm.movsd(xmm0, recordExternalAddress(crb, xmax)); // 0xffffffff, 0x7fefffff + masm.mulsd(xmm0, xmm0); + + masm.bind(block7); + masm.movl(new AMD64Address(rsp, 0), 14); + masm.jmp(block6); + + masm.bind(block10); + masm.movsd(xmm0, recordExternalAddress(crb, xmin)); // 0x00000000, 0x00100000 + masm.mulsd(xmm0, xmm0); + masm.movl(new AMD64Address(rsp, 0), 15); + masm.jmp(block6); + + masm.bind(block9); + masm.movl(rdx, new AMD64Address(rsp, 8)); + masm.cmpl(rax, 2146435072); + masm.jcc(ConditionFlag.Above, block11); + masm.cmpl(rdx, 0); + masm.jcc(ConditionFlag.NotEqual, block11); + masm.movl(rax, new AMD64Address(rsp, 12)); + masm.cmpl(rax, 2146435072); + masm.jcc(ConditionFlag.NotEqual, block12); + masm.movsd(xmm0, recordExternalAddress(crb, inf)); // 0x00000000, 0x7ff00000 + masm.jmp(block13); + + masm.bind(block12); + masm.movsd(xmm0, recordExternalAddress(crb, zero)); // 0x00000000, 0x00000000 + masm.jmp(block13); + + masm.bind(block11); + masm.movsd(xmm0, new AMD64Address(rsp, 8)); + masm.addsd(xmm0, xmm0); + masm.jmp(block13); + + masm.bind(block0); + masm.movl(rax, new AMD64Address(rsp, 12)); + masm.andl(rax, 2147483647); + masm.cmpl(rax, 1083179008); + masm.jcc(ConditionFlag.AboveEqual, block8); + masm.movsd(new AMD64Address(rsp, 8), xmm0); + masm.addsd(xmm0, recordExternalAddress(crb, oneVal)); // 0x00000000, 0x3ff00000 + masm.jmp(block13); + + masm.bind(block6); + masm.movq(new AMD64Address(rsp, 16), xmm0); + + masm.movq(xmm0, new AMD64Address(rsp, 16)); + + masm.bind(block13); + masm.addq(rsp, 24); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathIntrinsicBinaryOp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathIntrinsicBinaryOp.java index cdf06fd0a80..ec9525fb9dd 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathIntrinsicBinaryOp.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathIntrinsicBinaryOp.java @@ -24,2043 +24,52 @@ package org.graalvm.compiler.lir.amd64; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.ILLEGAL; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; -import static jdk.vm.ci.code.ValueUtil.asRegister; +import static jdk.vm.ci.amd64.AMD64.xmm0; +import static jdk.vm.ci.amd64.AMD64.xmm1; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.registersToValues; -import org.graalvm.compiler.asm.Label; -import org.graalvm.compiler.asm.amd64.AMD64Address; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; -import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; -import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; import org.graalvm.compiler.core.common.LIRKind; -import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.lir.LIRInstructionClass; -import org.graalvm.compiler.lir.Opcode; -import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; -import org.graalvm.compiler.lir.asm.CompilationResultBuilder; -import org.graalvm.compiler.lir.gen.LIRGeneratorTool; +import org.graalvm.compiler.lir.Variable; +import org.graalvm.compiler.lir.gen.LIRGenerator; -import jdk.vm.ci.amd64.AMD64; import jdk.vm.ci.amd64.AMD64Kind; import jdk.vm.ci.code.Register; -import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.code.RegisterValue; import jdk.vm.ci.meta.Value; -public final class AMD64MathIntrinsicBinaryOp extends AMD64LIRInstruction { - public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64MathIntrinsicBinaryOp.class); +/** + * AMD64MathIntrinsicBinaryOp assumes that the input values are stored in the xmm0 and xmm1 + * registers, and it will emit the output value into the xmm0 register. + * {@link #emitLIRWrapper(LIRGenerator, Value, Value)} is provided for emitting necessary mov LIRs + * before and after this LIR instruction. + */ +public abstract class AMD64MathIntrinsicBinaryOp extends AMD64LIRInstruction { - public enum BinaryIntrinsicOpcode { - POW + @Def protected Value output; + @Use protected Value input0; + @Use protected Value input1; + @Temp protected Value[] temps; + + public AMD64MathIntrinsicBinaryOp(LIRInstructionClass type, Register... registers) { + super(type); + + input0 = xmm0.asValue(LIRKind.value(AMD64Kind.DOUBLE)); + input1 = xmm1.asValue(LIRKind.value(AMD64Kind.DOUBLE)); + output = xmm0.asValue(LIRKind.value(AMD64Kind.DOUBLE)); + + temps = registersToValues(registers); } - @Opcode private final BinaryIntrinsicOpcode opcode; - @Def protected Value result; - @Use protected Value input; - @Use protected Value secondInput; - @Temp({REG, ILLEGAL}) protected Value xmm1Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm2Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm3Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm4Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm5Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm6Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm7Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm8Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm9Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm10Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr1Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr2Temp = Value.ILLEGAL; - @Temp protected AllocatableValue rcxTemp; - @Temp({REG, ILLEGAL}) protected Value gpr4Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr5Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr6Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr7Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr8Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr9Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr10Temp = Value.ILLEGAL; - - CompilationResultBuilder internalCrb; - - public AMD64MathIntrinsicBinaryOp(LIRGeneratorTool tool, BinaryIntrinsicOpcode opcode, Value result, Value input, Value alternateInput) { - super(TYPE); - this.opcode = opcode; - this.result = result; - this.input = input; - this.secondInput = alternateInput; - if (opcode == BinaryIntrinsicOpcode.POW) { - this.gpr1Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr2Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.rcxTemp = AMD64.rcx.asValue(LIRKind.value(AMD64Kind.QWORD)); - this.gpr4Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr5Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr6Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr7Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr8Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - - this.xmm1Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm2Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm3Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm4Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm5Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm6Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm7Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm8Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm9Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm10Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - } - } - - private void setCrb(CompilationResultBuilder crb) { - internalCrb = crb; - } - - private AMD64Address externalAddress(ArrayDataPointerConstant curPtr) { - return (AMD64Address) internalCrb.recordDataReferenceInCode(curPtr); - } - - @Override - public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { - switch (opcode) { - case POW: - powIntrinsic(asRegister(result, AMD64Kind.DOUBLE), asRegister(input, AMD64Kind.DOUBLE), asRegister(secondInput, AMD64Kind.DOUBLE), crb, masm); - break; - default: - throw GraalError.shouldNotReachHere(); - } - } - - /* - * Copyright (c) 2014, 2016, Intel Corporation. All rights reserved. Intel Math Library (LIBM) - * Source Code - * - * ALGORITHM DESCRIPTION - POW() --------------------- - * - * Let x=2^k * mx, mx in [1,2) - * - * log2(x) calculation: - * - * Get B~1/mx based on the output of rcpps instruction (B0) B = int((B0*LH*2^9+0.5))/2^9 LH is a - * short approximation for log2(e) - * - * Reduced argument, scaled by LH: r=B*mx-LH (computed accurately in high and low parts) - * - * log2(x) result: k - log2(B) + p(r) p(r) is a degree 8 polynomial -log2(B) read from data - * table (high, low parts) log2(x) is formed from high and low parts For |x| in [1-1/32, - * 1+1/16), a slower but more accurate computation based om the same table design is performed. - * - * Main path is taken if | floor(log2(|log2(|x|)|) + floor(log2|y|) | < 8, to filter out all - * potential OF/UF cases. exp2(y*log2(x)) is computed using an 8-bit index table and a degree 5 - * polynomial - * - * Special cases: pow(-0,y) = -INF and raises the divide-by-zero exception for y an odd integer - * < 0. pow(-0,y) = +INF and raises the divide-by-zero exception for y < 0 and not an odd - * integer. pow(-0,y) = -0 for y an odd integer > 0. pow(-0,y) = +0 for y > 0 and not an odd - * integer. pow(-1,-INF) = NaN. pow(+1,y) = NaN for any y, even a NaN. pow(x,-0) = 1 for any x, - * even a NaN. pow(x,y) = a NaN and raises the invalid exception for finite x < 0 and finite - * non-integer y. pow(x,-INF) = +INF for |x|<1. pow(x,-INF) = +0 for |x|>1. pow(x,+INF) = +0 for - * |x|<1. pow(x,+INF) = +INF for |x|>1. pow(-INF,y) = -0 for y an odd integer < 0. pow(-INF,y) = - * +0 for y < 0 and not an odd integer. pow(-INF,y) = -INF for y an odd integer > 0. pow(-INF,y) - * = +INF for y > 0 and not an odd integer. pow(+INF,y) = +0 for y <0. pow(+INF,y) = +INF for y - * >0. - * - */ - - private static int[] highSigMask = { - 0x00000000, 0xfffff800, 0x00000000, 0xfffff800 - }; - - private static int[] logTwoE = { - 0x00000000, 0x3ff72000, 0x161bb241, 0xbf5dabe1 - }; - - private static int[] highmaskY = { - 0x00000000, 0xfffffff8, 0x00000000, 0xffffffff - }; - - private static int[] tExp = { - 0x00000000, 0x3ff00000, 0x00000000, 0x3b700000, 0xfa5abcbf, - 0x3ff00b1a, 0xa7609f71, 0xbc84f6b2, 0xa9fb3335, 0x3ff0163d, - 0x9ab8cdb7, 0x3c9b6129, 0x143b0281, 0x3ff02168, 0x0fc54eb6, - 0xbc82bf31, 0x3e778061, 0x3ff02c9a, 0x535b085d, 0xbc719083, - 0x2e11bbcc, 0x3ff037d4, 0xeeade11a, 0x3c656811, 0xe86e7f85, - 0x3ff04315, 0x1977c96e, 0xbc90a31c, 0x72f654b1, 0x3ff04e5f, - 0x3aa0d08c, 0x3c84c379, 0xd3158574, 0x3ff059b0, 0xa475b465, - 0x3c8d73e2, 0x0e3c1f89, 0x3ff0650a, 0x5799c397, 0xbc95cb7b, - 0x29ddf6de, 0x3ff0706b, 0xe2b13c27, 0xbc8c91df, 0x2b72a836, - 0x3ff07bd4, 0x54458700, 0x3c832334, 0x18759bc8, 0x3ff08745, - 0x4bb284ff, 0x3c6186be, 0xf66607e0, 0x3ff092bd, 0x800a3fd1, - 0xbc968063, 0xcac6f383, 0x3ff09e3e, 0x18316136, 0x3c914878, - 0x9b1f3919, 0x3ff0a9c7, 0x873d1d38, 0x3c85d16c, 0x6cf9890f, - 0x3ff0b558, 0x4adc610b, 0x3c98a62e, 0x45e46c85, 0x3ff0c0f1, - 0x06d21cef, 0x3c94f989, 0x2b7247f7, 0x3ff0cc92, 0x16e24f71, - 0x3c901edc, 0x23395dec, 0x3ff0d83b, 0xe43f316a, 0xbc9bc14d, - 0x32d3d1a2, 0x3ff0e3ec, 0x27c57b52, 0x3c403a17, 0x5fdfa9c5, - 0x3ff0efa5, 0xbc54021b, 0xbc949db9, 0xaffed31b, 0x3ff0fb66, - 0xc44ebd7b, 0xbc6b9bed, 0x28d7233e, 0x3ff10730, 0x1692fdd5, - 0x3c8d46eb, 0xd0125b51, 0x3ff11301, 0x39449b3a, 0xbc96c510, - 0xab5e2ab6, 0x3ff11edb, 0xf703fb72, 0xbc9ca454, 0xc06c31cc, - 0x3ff12abd, 0xb36ca5c7, 0xbc51b514, 0x14f204ab, 0x3ff136a8, - 0xba48dcf0, 0xbc67108f, 0xaea92de0, 0x3ff1429a, 0x9af1369e, - 0xbc932fbf, 0x934f312e, 0x3ff14e95, 0x39bf44ab, 0xbc8b91e8, - 0xc8a58e51, 0x3ff15a98, 0xb9eeab0a, 0x3c82406a, 0x5471c3c2, - 0x3ff166a4, 0x82ea1a32, 0x3c58f23b, 0x3c7d517b, 0x3ff172b8, - 0xb9d78a76, 0xbc819041, 0x8695bbc0, 0x3ff17ed4, 0xe2ac5a64, - 0x3c709e3f, 0x388c8dea, 0x3ff18af9, 0xd1970f6c, 0xbc911023, - 0x58375d2f, 0x3ff19726, 0x85f17e08, 0x3c94aadd, 0xeb6fcb75, - 0x3ff1a35b, 0x7b4968e4, 0x3c8e5b4c, 0xf8138a1c, 0x3ff1af99, - 0xa4b69280, 0x3c97bf85, 0x84045cd4, 0x3ff1bbe0, 0x352ef607, - 0xbc995386, 0x95281c6b, 0x3ff1c82f, 0x8010f8c9, 0x3c900977, - 0x3168b9aa, 0x3ff1d487, 0x00a2643c, 0x3c9e016e, 0x5eb44027, - 0x3ff1e0e7, 0x088cb6de, 0xbc96fdd8, 0x22fcd91d, 0x3ff1ed50, - 0x027bb78c, 0xbc91df98, 0x8438ce4d, 0x3ff1f9c1, 0xa097af5c, - 0xbc9bf524, 0x88628cd6, 0x3ff2063b, 0x814a8495, 0x3c8dc775, - 0x3578a819, 0x3ff212be, 0x2cfcaac9, 0x3c93592d, 0x917ddc96, - 0x3ff21f49, 0x9494a5ee, 0x3c82a97e, 0xa27912d1, 0x3ff22bdd, - 0x5577d69f, 0x3c8d34fb, 0x6e756238, 0x3ff2387a, 0xb6c70573, - 0x3c99b07e, 0xfb82140a, 0x3ff2451f, 0x911ca996, 0x3c8acfcc, - 0x4fb2a63f, 0x3ff251ce, 0xbef4f4a4, 0x3c8ac155, 0x711ece75, - 0x3ff25e85, 0x4ac31b2c, 0x3c93e1a2, 0x65e27cdd, 0x3ff26b45, - 0x9940e9d9, 0x3c82bd33, 0x341ddf29, 0x3ff2780e, 0x05f9e76c, - 0x3c9e067c, 0xe1f56381, 0x3ff284df, 0x8c3f0d7e, 0xbc9a4c3a, - 0x7591bb70, 0x3ff291ba, 0x28401cbd, 0xbc82cc72, 0xf51fdee1, - 0x3ff29e9d, 0xafad1255, 0x3c8612e8, 0x66d10f13, 0x3ff2ab8a, - 0x191690a7, 0xbc995743, 0xd0dad990, 0x3ff2b87f, 0xd6381aa4, - 0xbc410adc, 0x39771b2f, 0x3ff2c57e, 0xa6eb5124, 0xbc950145, - 0xa6e4030b, 0x3ff2d285, 0x54db41d5, 0x3c900247, 0x1f641589, - 0x3ff2df96, 0xfbbce198, 0x3c9d16cf, 0xa93e2f56, 0x3ff2ecaf, - 0x45d52383, 0x3c71ca0f, 0x4abd886b, 0x3ff2f9d2, 0x532bda93, - 0xbc653c55, 0x0a31b715, 0x3ff306fe, 0xd23182e4, 0x3c86f46a, - 0xedeeb2fd, 0x3ff31432, 0xf3f3fcd1, 0x3c8959a3, 0xfc4cd831, - 0x3ff32170, 0x8e18047c, 0x3c8a9ce7, 0x3ba8ea32, 0x3ff32eb8, - 0x3cb4f318, 0xbc9c45e8, 0xb26416ff, 0x3ff33c08, 0x843659a6, - 0x3c932721, 0x66e3fa2d, 0x3ff34962, 0x930881a4, 0xbc835a75, - 0x5f929ff1, 0x3ff356c5, 0x5c4e4628, 0xbc8b5cee, 0xa2de883b, - 0x3ff36431, 0xa06cb85e, 0xbc8c3144, 0x373aa9cb, 0x3ff371a7, - 0xbf42eae2, 0xbc963aea, 0x231e754a, 0x3ff37f26, 0x9eceb23c, - 0xbc99f5ca, 0x6d05d866, 0x3ff38cae, 0x3c9904bd, 0xbc9e958d, - 0x1b7140ef, 0x3ff39a40, 0xfc8e2934, 0xbc99a9a5, 0x34e59ff7, - 0x3ff3a7db, 0xd661f5e3, 0xbc75e436, 0xbfec6cf4, 0x3ff3b57f, - 0xe26fff18, 0x3c954c66, 0xc313a8e5, 0x3ff3c32d, 0x375d29c3, - 0xbc9efff8, 0x44ede173, 0x3ff3d0e5, 0x8c284c71, 0x3c7fe8d0, - 0x4c123422, 0x3ff3dea6, 0x11f09ebc, 0x3c8ada09, 0xdf1c5175, - 0x3ff3ec70, 0x7b8c9bca, 0xbc8af663, 0x04ac801c, 0x3ff3fa45, - 0xf956f9f3, 0xbc97d023, 0xc367a024, 0x3ff40822, 0xb6f4d048, - 0x3c8bddf8, 0x21f72e2a, 0x3ff4160a, 0x1c309278, 0xbc5ef369, - 0x2709468a, 0x3ff423fb, 0xc0b314dd, 0xbc98462d, 0xd950a897, - 0x3ff431f5, 0xe35f7999, 0xbc81c7dd, 0x3f84b9d4, 0x3ff43ffa, - 0x9704c003, 0x3c8880be, 0x6061892d, 0x3ff44e08, 0x04ef80d0, - 0x3c489b7a, 0x42a7d232, 0x3ff45c20, 0x82fb1f8e, 0xbc686419, - 0xed1d0057, 0x3ff46a41, 0xd1648a76, 0x3c9c944b, 0x668b3237, - 0x3ff4786d, 0xed445733, 0xbc9c20f0, 0xb5c13cd0, 0x3ff486a2, - 0xb69062f0, 0x3c73c1a3, 0xe192aed2, 0x3ff494e1, 0x5e499ea0, - 0xbc83b289, 0xf0d7d3de, 0x3ff4a32a, 0xf3d1be56, 0x3c99cb62, - 0xea6db7d7, 0x3ff4b17d, 0x7f2897f0, 0xbc8125b8, 0xd5362a27, - 0x3ff4bfda, 0xafec42e2, 0x3c7d4397, 0xb817c114, 0x3ff4ce41, - 0x690abd5d, 0x3c905e29, 0x99fddd0d, 0x3ff4dcb2, 0xbc6a7833, - 0x3c98ecdb, 0x81d8abff, 0x3ff4eb2d, 0x2e5d7a52, 0xbc95257d, - 0x769d2ca7, 0x3ff4f9b2, 0xd25957e3, 0xbc94b309, 0x7f4531ee, - 0x3ff50841, 0x49b7465f, 0x3c7a249b, 0xa2cf6642, 0x3ff516da, - 0x69bd93ef, 0xbc8f7685, 0xe83f4eef, 0x3ff5257d, 0x43efef71, - 0xbc7c998d, 0x569d4f82, 0x3ff5342b, 0x1db13cad, 0xbc807abe, - 0xf4f6ad27, 0x3ff542e2, 0x192d5f7e, 0x3c87926d, 0xca5d920f, - 0x3ff551a4, 0xefede59b, 0xbc8d689c, 0xdde910d2, 0x3ff56070, - 0x168eebf0, 0xbc90fb6e, 0x36b527da, 0x3ff56f47, 0x011d93ad, - 0x3c99bb2c, 0xdbe2c4cf, 0x3ff57e27, 0x8a57b9c4, 0xbc90b98c, - 0xd497c7fd, 0x3ff58d12, 0x5b9a1de8, 0x3c8295e1, 0x27ff07cc, - 0x3ff59c08, 0xe467e60f, 0xbc97e2ce, 0xdd485429, 0x3ff5ab07, - 0x054647ad, 0x3c96324c, 0xfba87a03, 0x3ff5ba11, 0x4c233e1a, - 0xbc9b77a1, 0x8a5946b7, 0x3ff5c926, 0x816986a2, 0x3c3c4b1b, - 0x90998b93, 0x3ff5d845, 0xa8b45643, 0xbc9cd6a7, 0x15ad2148, - 0x3ff5e76f, 0x3080e65e, 0x3c9ba6f9, 0x20dceb71, 0x3ff5f6a3, - 0xe3cdcf92, 0xbc89eadd, 0xb976dc09, 0x3ff605e1, 0x9b56de47, - 0xbc93e242, 0xe6cdf6f4, 0x3ff6152a, 0x4ab84c27, 0x3c9e4b3e, - 0xb03a5585, 0x3ff6247e, 0x7e40b497, 0xbc9383c1, 0x1d1929fd, - 0x3ff633dd, 0xbeb964e5, 0x3c984710, 0x34ccc320, 0x3ff64346, - 0x759d8933, 0xbc8c483c, 0xfebc8fb7, 0x3ff652b9, 0xc9a73e09, - 0xbc9ae3d5, 0x82552225, 0x3ff66238, 0x87591c34, 0xbc9bb609, - 0xc70833f6, 0x3ff671c1, 0x586c6134, 0xbc8e8732, 0xd44ca973, - 0x3ff68155, 0x44f73e65, 0x3c6038ae, 0xb19e9538, 0x3ff690f4, - 0x9aeb445d, 0x3c8804bd, 0x667f3bcd, 0x3ff6a09e, 0x13b26456, - 0xbc9bdd34, 0xfa75173e, 0x3ff6b052, 0x2c9a9d0e, 0x3c7a38f5, - 0x750bdabf, 0x3ff6c012, 0x67ff0b0d, 0xbc728956, 0xddd47645, - 0x3ff6cfdc, 0xb6f17309, 0x3c9c7aa9, 0x3c651a2f, 0x3ff6dfb2, - 0x683c88ab, 0xbc6bbe3a, 0x98593ae5, 0x3ff6ef92, 0x9e1ac8b2, - 0xbc90b974, 0xf9519484, 0x3ff6ff7d, 0x25860ef6, 0xbc883c0f, - 0x66f42e87, 0x3ff70f74, 0xd45aa65f, 0x3c59d644, 0xe8ec5f74, - 0x3ff71f75, 0x86887a99, 0xbc816e47, 0x86ead08a, 0x3ff72f82, - 0x2cd62c72, 0xbc920aa0, 0x48a58174, 0x3ff73f9a, 0x6c65d53c, - 0xbc90a8d9, 0x35d7cbfd, 0x3ff74fbd, 0x618a6e1c, 0x3c9047fd, - 0x564267c9, 0x3ff75feb, 0x57316dd3, 0xbc902459, 0xb1ab6e09, - 0x3ff77024, 0x169147f8, 0x3c9b7877, 0x4fde5d3f, 0x3ff78069, - 0x0a02162d, 0x3c9866b8, 0x38ac1cf6, 0x3ff790b9, 0x62aadd3e, - 0x3c9349a8, 0x73eb0187, 0x3ff7a114, 0xee04992f, 0xbc841577, - 0x0976cfdb, 0x3ff7b17b, 0x8468dc88, 0xbc9bebb5, 0x0130c132, - 0x3ff7c1ed, 0xd1164dd6, 0x3c9f124c, 0x62ff86f0, 0x3ff7d26a, - 0xfb72b8b4, 0x3c91bddb, 0x36cf4e62, 0x3ff7e2f3, 0xba15797e, - 0x3c705d02, 0x8491c491, 0x3ff7f387, 0xcf9311ae, 0xbc807f11, - 0x543e1a12, 0x3ff80427, 0x626d972b, 0xbc927c86, 0xadd106d9, - 0x3ff814d2, 0x0d151d4d, 0x3c946437, 0x994cce13, 0x3ff82589, - 0xd41532d8, 0xbc9d4c1d, 0x1eb941f7, 0x3ff8364c, 0x31df2bd5, - 0x3c999b9a, 0x4623c7ad, 0x3ff8471a, 0xa341cdfb, 0xbc88d684, - 0x179f5b21, 0x3ff857f4, 0xf8b216d0, 0xbc5ba748, 0x9b4492ed, - 0x3ff868d9, 0x9bd4f6ba, 0xbc9fc6f8, 0xd931a436, 0x3ff879ca, - 0xd2db47bd, 0x3c85d2d7, 0xd98a6699, 0x3ff88ac7, 0xf37cb53a, - 0x3c9994c2, 0xa478580f, 0x3ff89bd0, 0x4475202a, 0x3c9d5395, - 0x422aa0db, 0x3ff8ace5, 0x56864b27, 0x3c96e9f1, 0xbad61778, - 0x3ff8be05, 0xfc43446e, 0x3c9ecb5e, 0x16b5448c, 0x3ff8cf32, - 0x32e9e3aa, 0xbc70d55e, 0x5e0866d9, 0x3ff8e06a, 0x6fc9b2e6, - 0xbc97114a, 0x99157736, 0x3ff8f1ae, 0xa2e3976c, 0x3c85cc13, - 0xd0282c8a, 0x3ff902fe, 0x85fe3fd2, 0x3c9592ca, 0x0b91ffc6, - 0x3ff9145b, 0x2e582524, 0xbc9dd679, 0x53aa2fe2, 0x3ff925c3, - 0xa639db7f, 0xbc83455f, 0xb0cdc5e5, 0x3ff93737, 0x81b57ebc, - 0xbc675fc7, 0x2b5f98e5, 0x3ff948b8, 0x797d2d99, 0xbc8dc3d6, - 0xcbc8520f, 0x3ff95a44, 0x96a5f039, 0xbc764b7c, 0x9a7670b3, - 0x3ff96bdd, 0x7f19c896, 0xbc5ba596, 0x9fde4e50, 0x3ff97d82, - 0x7c1b85d1, 0xbc9d185b, 0xe47a22a2, 0x3ff98f33, 0xa24c78ec, - 0x3c7cabda, 0x70ca07ba, 0x3ff9a0f1, 0x91cee632, 0xbc9173bd, - 0x4d53fe0d, 0x3ff9b2bb, 0x4df6d518, 0xbc9dd84e, 0x82a3f090, - 0x3ff9c491, 0xb071f2be, 0x3c7c7c46, 0x194bb8d5, 0x3ff9d674, - 0xa3dd8233, 0xbc9516be, 0x19e32323, 0x3ff9e863, 0x78e64c6e, - 0x3c7824ca, 0x8d07f29e, 0x3ff9fa5e, 0xaaf1face, 0xbc84a9ce, - 0x7b5de565, 0x3ffa0c66, 0x5d1cd533, 0xbc935949, 0xed8eb8bb, - 0x3ffa1e7a, 0xee8be70e, 0x3c9c6618, 0xec4a2d33, 0x3ffa309b, - 0x7ddc36ab, 0x3c96305c, 0x80460ad8, 0x3ffa42c9, 0x589fb120, - 0xbc9aa780, 0xb23e255d, 0x3ffa5503, 0xdb8d41e1, 0xbc9d2f6e, - 0x8af46052, 0x3ffa674a, 0x30670366, 0x3c650f56, 0x1330b358, - 0x3ffa799e, 0xcac563c7, 0x3c9bcb7e, 0x53c12e59, 0x3ffa8bfe, - 0xb2ba15a9, 0xbc94f867, 0x5579fdbf, 0x3ffa9e6b, 0x0ef7fd31, - 0x3c90fac9, 0x21356eba, 0x3ffab0e5, 0xdae94545, 0x3c889c31, - 0xbfd3f37a, 0x3ffac36b, 0xcae76cd0, 0xbc8f9234, 0x3a3c2774, - 0x3ffad5ff, 0xb6b1b8e5, 0x3c97ef3b, 0x995ad3ad, 0x3ffae89f, - 0x345dcc81, 0x3c97a1cd, 0xe622f2ff, 0x3ffafb4c, 0x0f315ecd, - 0xbc94b2fc, 0x298db666, 0x3ffb0e07, 0x4c80e425, 0xbc9bdef5, - 0x6c9a8952, 0x3ffb20ce, 0x4a0756cc, 0x3c94dd02, 0xb84f15fb, - 0x3ffb33a2, 0x3084d708, 0xbc62805e, 0x15b749b1, 0x3ffb4684, - 0xe9df7c90, 0xbc7f763d, 0x8de5593a, 0x3ffb5972, 0xbbba6de3, - 0xbc9c71df, 0x29f1c52a, 0x3ffb6c6e, 0x52883f6e, 0x3c92a8f3, - 0xf2fb5e47, 0x3ffb7f76, 0x7e54ac3b, 0xbc75584f, 0xf22749e4, - 0x3ffb928c, 0x54cb65c6, 0xbc9b7216, 0x30a1064a, 0x3ffba5b0, - 0x0e54292e, 0xbc9efcd3, 0xb79a6f1f, 0x3ffbb8e0, 0xc9696205, - 0xbc3f52d1, 0x904bc1d2, 0x3ffbcc1e, 0x7a2d9e84, 0x3c823dd0, - 0xc3f3a207, 0x3ffbdf69, 0x60ea5b53, 0xbc3c2623, 0x5bd71e09, - 0x3ffbf2c2, 0x3f6b9c73, 0xbc9efdca, 0x6141b33d, 0x3ffc0628, - 0xa1fbca34, 0xbc8d8a5a, 0xdd85529c, 0x3ffc199b, 0x895048dd, - 0x3c811065, 0xd9fa652c, 0x3ffc2d1c, 0x17c8a5d7, 0xbc96e516, - 0x5fffd07a, 0x3ffc40ab, 0xe083c60a, 0x3c9b4537, 0x78fafb22, - 0x3ffc5447, 0x2493b5af, 0x3c912f07, 0x2e57d14b, 0x3ffc67f1, - 0xff483cad, 0x3c92884d, 0x8988c933, 0x3ffc7ba8, 0xbe255559, - 0xbc8e76bb, 0x9406e7b5, 0x3ffc8f6d, 0x48805c44, 0x3c71acbc, - 0x5751c4db, 0x3ffca340, 0xd10d08f5, 0xbc87f2be, 0xdcef9069, - 0x3ffcb720, 0xd1e949db, 0x3c7503cb, 0x2e6d1675, 0x3ffccb0f, - 0x86009092, 0xbc7d220f, 0x555dc3fa, 0x3ffcdf0b, 0x53829d72, - 0xbc8dd83b, 0x5b5bab74, 0x3ffcf315, 0xb86dff57, 0xbc9a08e9, - 0x4a07897c, 0x3ffd072d, 0x43797a9c, 0xbc9cbc37, 0x2b08c968, - 0x3ffd1b53, 0x219a36ee, 0x3c955636, 0x080d89f2, 0x3ffd2f87, - 0x719d8578, 0xbc9d487b, 0xeacaa1d6, 0x3ffd43c8, 0xbf5a1614, - 0x3c93db53, 0xdcfba487, 0x3ffd5818, 0xd75b3707, 0x3c82ed02, - 0xe862e6d3, 0x3ffd6c76, 0x4a8165a0, 0x3c5fe87a, 0x16c98398, - 0x3ffd80e3, 0x8beddfe8, 0xbc911ec1, 0x71ff6075, 0x3ffd955d, - 0xbb9af6be, 0x3c9a052d, 0x03db3285, 0x3ffda9e6, 0x696db532, - 0x3c9c2300, 0xd63a8315, 0x3ffdbe7c, 0x926b8be4, 0xbc9b76f1, - 0xf301b460, 0x3ffdd321, 0x78f018c3, 0x3c92da57, 0x641c0658, - 0x3ffde7d5, 0x8e79ba8f, 0xbc9ca552, 0x337b9b5f, 0x3ffdfc97, - 0x4f184b5c, 0xbc91a5cd, 0x6b197d17, 0x3ffe1167, 0xbd5c7f44, - 0xbc72b529, 0x14f5a129, 0x3ffe2646, 0x817a1496, 0xbc97b627, - 0x3b16ee12, 0x3ffe3b33, 0x31fdc68b, 0xbc99f4a4, 0xe78b3ff6, - 0x3ffe502e, 0x80a9cc8f, 0x3c839e89, 0x24676d76, 0x3ffe6539, - 0x7522b735, 0xbc863ff8, 0xfbc74c83, 0x3ffe7a51, 0xca0c8de2, - 0x3c92d522, 0x77cdb740, 0x3ffe8f79, 0x80b054b1, 0xbc910894, - 0xa2a490da, 0x3ffea4af, 0x179c2893, 0xbc9e9c23, 0x867cca6e, - 0x3ffeb9f4, 0x2293e4f2, 0x3c94832f, 0x2d8e67f1, 0x3ffecf48, - 0xb411ad8c, 0xbc9c93f3, 0xa2188510, 0x3ffee4aa, 0xa487568d, - 0x3c91c68d, 0xee615a27, 0x3ffefa1b, 0x86a4b6b0, 0x3c9dc7f4, - 0x1cb6412a, 0x3fff0f9c, 0x65181d45, 0xbc932200, 0x376bba97, - 0x3fff252b, 0xbf0d8e43, 0x3c93a1a5, 0x48dd7274, 0x3fff3ac9, - 0x3ed837de, 0xbc795a5a, 0x5b6e4540, 0x3fff5076, 0x2dd8a18b, - 0x3c99d3e1, 0x798844f8, 0x3fff6632, 0x3539343e, 0x3c9fa37b, - 0xad9cbe14, 0x3fff7bfd, 0xd006350a, 0xbc9dbb12, 0x02243c89, - 0x3fff91d8, 0xa779f689, 0xbc612ea8, 0x819e90d8, 0x3fffa7c1, - 0xf3a5931e, 0x3c874853, 0x3692d514, 0x3fffbdba, 0x15098eb6, - 0xbc796773, 0x2b8f71f1, 0x3fffd3c2, 0x966579e7, 0x3c62eb74, - 0x6b2a23d9, 0x3fffe9d9, 0x7442fde3, 0x3c74a603 - }; - - private static int[] eCoeff = { - 0xe78a6731, 0x3f55d87f, 0xd704a0c0, 0x3fac6b08, 0x6fba4e77, - 0x3f83b2ab, 0xff82c58f, 0x3fcebfbd, 0xfefa39ef, 0x3fe62e42, - 0x00000000, 0x00000000 - }; - - private static int[] coeffH = { - 0x00000000, 0xbfd61a00, 0x00000000, 0xbf5dabe1 - }; - - private static int[] highmaskLogX = { - 0xf8000000, 0xffffffff, 0x00000000, 0xfffff800 - }; - - private static int[] halfmask = { - 0xf8000000, 0xffffffff, 0xf8000000, 0xffffffff - }; - - private static int[] coeffPow = { - 0x6dc96112, 0xbf836578, 0xee241472, 0xbf9b0301, 0x9f95985a, - 0xbfb528db, 0xb3841d2a, 0xbfd619b6, 0x518775e3, 0x3f9004f2, - 0xac8349bb, 0x3fa76c9b, 0x486ececc, 0x3fc4635e, 0x161bb241, - 0xbf5dabe1, 0x9f95985a, 0xbfb528db, 0xf8b5787d, 0x3ef2531e, - 0x486ececb, 0x3fc4635e, 0x412055cc, 0xbdd61bb2 - }; - - private static int[] lTblPow = { - 0x00000000, 0x3ff00000, 0x00000000, 0x00000000, 0x20000000, - 0x3feff00a, 0x96621f95, 0x3e5b1856, 0xe0000000, 0x3fefe019, - 0xe5916f9e, 0xbe325278, 0x00000000, 0x3fefd02f, 0x859a1062, - 0x3e595fb7, 0xc0000000, 0x3fefc049, 0xb245f18f, 0xbe529c38, - 0xe0000000, 0x3fefb069, 0xad2880a7, 0xbe501230, 0x60000000, - 0x3fefa08f, 0xc8e72420, 0x3e597bd1, 0x80000000, 0x3fef90ba, - 0xc30c4500, 0xbe5d6c75, 0xe0000000, 0x3fef80ea, 0x02c63f43, - 0x3e2e1318, 0xc0000000, 0x3fef7120, 0xb3d4cccc, 0xbe44c52a, - 0x00000000, 0x3fef615c, 0xdbd91397, 0xbe4e7d6c, 0xa0000000, - 0x3fef519c, 0x65c5cd68, 0xbe522dc8, 0xa0000000, 0x3fef41e2, - 0x46d1306c, 0xbe5a840e, 0xe0000000, 0x3fef322d, 0xd2980e94, - 0x3e5071af, 0xa0000000, 0x3fef227e, 0x773abade, 0xbe5891e5, - 0xa0000000, 0x3fef12d4, 0xdc6bf46b, 0xbe5cccbe, 0xe0000000, - 0x3fef032f, 0xbc7247fa, 0xbe2bab83, 0x80000000, 0x3feef390, - 0xbcaa1e46, 0xbe53bb3b, 0x60000000, 0x3feee3f6, 0x5f6c682d, - 0xbe54c619, 0x80000000, 0x3feed461, 0x5141e368, 0xbe4b6d86, - 0xe0000000, 0x3feec4d1, 0xec678f76, 0xbe369af6, 0x80000000, - 0x3feeb547, 0x41301f55, 0xbe2d4312, 0x60000000, 0x3feea5c2, - 0x676da6bd, 0xbe4d8dd0, 0x60000000, 0x3fee9642, 0x57a891c4, - 0x3e51f991, 0xa0000000, 0x3fee86c7, 0xe4eb491e, 0x3e579bf9, - 0x20000000, 0x3fee7752, 0xfddc4a2c, 0xbe3356e6, 0xc0000000, - 0x3fee67e1, 0xd75b5bf1, 0xbe449531, 0x80000000, 0x3fee5876, - 0xbd423b8e, 0x3df54fe4, 0x60000000, 0x3fee4910, 0x330e51b9, - 0x3e54289c, 0x80000000, 0x3fee39af, 0x8651a95f, 0xbe55aad6, - 0xa0000000, 0x3fee2a53, 0x5e98c708, 0xbe2fc4a9, 0xe0000000, - 0x3fee1afc, 0x0989328d, 0x3e23958c, 0x40000000, 0x3fee0bab, - 0xee642abd, 0xbe425dd8, 0xa0000000, 0x3fedfc5e, 0xc394d236, - 0x3e526362, 0x20000000, 0x3feded17, 0xe104aa8e, 0x3e4ce247, - 0xc0000000, 0x3fedddd4, 0x265a9be4, 0xbe5bb77a, 0x40000000, - 0x3fedce97, 0x0ecac52f, 0x3e4a7cb1, 0xe0000000, 0x3fedbf5e, - 0x124cb3b8, 0x3e257024, 0x80000000, 0x3fedb02b, 0xe6d4febe, - 0xbe2033ee, 0x20000000, 0x3feda0fd, 0x39cca00e, 0xbe3ddabc, - 0xc0000000, 0x3fed91d3, 0xef8a552a, 0xbe543390, 0x40000000, - 0x3fed82af, 0xb8e85204, 0x3e513850, 0xe0000000, 0x3fed738f, - 0x3d59fe08, 0xbe5db728, 0x40000000, 0x3fed6475, 0x3aa7ead1, - 0x3e58804b, 0xc0000000, 0x3fed555f, 0xf8a35ba9, 0xbe5298b0, - 0x00000000, 0x3fed464f, 0x9a88dd15, 0x3e5a8cdb, 0x40000000, - 0x3fed3743, 0xb0b0a190, 0x3e598635, 0x80000000, 0x3fed283c, - 0xe2113295, 0xbe5c1119, 0x80000000, 0x3fed193a, 0xafbf1728, - 0xbe492e9c, 0x60000000, 0x3fed0a3d, 0xe4a4ccf3, 0x3e19b90e, - 0x20000000, 0x3fecfb45, 0xba3cbeb8, 0x3e406b50, 0xc0000000, - 0x3fecec51, 0x110f7ddd, 0x3e0d6806, 0x40000000, 0x3fecdd63, - 0x7dd7d508, 0xbe5a8943, 0x80000000, 0x3fecce79, 0x9b60f271, - 0xbe50676a, 0x80000000, 0x3fecbf94, 0x0b9ad660, 0x3e59174f, - 0x60000000, 0x3fecb0b4, 0x00823d9c, 0x3e5bbf72, 0x20000000, - 0x3feca1d9, 0x38a6ec89, 0xbe4d38f9, 0x80000000, 0x3fec9302, - 0x3a0b7d8e, 0x3e53dbfd, 0xc0000000, 0x3fec8430, 0xc6826b34, - 0xbe27c5c9, 0xc0000000, 0x3fec7563, 0x0c706381, 0xbe593653, - 0x60000000, 0x3fec669b, 0x7df34ec7, 0x3e461ab5, 0xe0000000, - 0x3fec57d7, 0x40e5e7e8, 0xbe5c3dae, 0x00000000, 0x3fec4919, - 0x5602770f, 0xbe55219d, 0xc0000000, 0x3fec3a5e, 0xec7911eb, - 0x3e5a5d25, 0x60000000, 0x3fec2ba9, 0xb39ea225, 0xbe53c00b, - 0x80000000, 0x3fec1cf8, 0x967a212e, 0x3e5a8ddf, 0x60000000, - 0x3fec0e4c, 0x580798bd, 0x3e5f53ab, 0x00000000, 0x3febffa5, - 0xb8282df6, 0xbe46b874, 0x20000000, 0x3febf102, 0xe33a6729, - 0x3e54963f, 0x00000000, 0x3febe264, 0x3b53e88a, 0xbe3adce1, - 0x60000000, 0x3febd3ca, 0xc2585084, 0x3e5cde9f, 0x80000000, - 0x3febc535, 0xa335c5ee, 0xbe39fd9c, 0x20000000, 0x3febb6a5, - 0x7325b04d, 0x3e42ba15, 0x60000000, 0x3feba819, 0x1564540f, - 0x3e3a9f35, 0x40000000, 0x3feb9992, 0x83fff592, 0xbe5465ce, - 0xa0000000, 0x3feb8b0f, 0xb9da63d3, 0xbe4b1a0a, 0x80000000, - 0x3feb7c91, 0x6d6f1ea4, 0x3e557657, 0x00000000, 0x3feb6e18, - 0x5e80a1bf, 0x3e4ddbb6, 0x00000000, 0x3feb5fa3, 0x1c9eacb5, - 0x3e592877, 0xa0000000, 0x3feb5132, 0x6d40beb3, 0xbe51858c, - 0xa0000000, 0x3feb42c6, 0xd740c67b, 0x3e427ad2, 0x40000000, - 0x3feb345f, 0xa3e0ccee, 0xbe5c2fc4, 0x40000000, 0x3feb25fc, - 0x8e752b50, 0xbe3da3c2, 0xc0000000, 0x3feb179d, 0xa892e7de, - 0x3e1fb481, 0xc0000000, 0x3feb0943, 0x21ed71e9, 0xbe365206, - 0x20000000, 0x3feafaee, 0x0e1380a3, 0x3e5c5b7b, 0x20000000, - 0x3feaec9d, 0x3c3d640e, 0xbe5dbbd0, 0x60000000, 0x3feade50, - 0x8f97a715, 0x3e3a8ec5, 0x20000000, 0x3fead008, 0x23ab2839, - 0x3e2fe98a, 0x40000000, 0x3feac1c4, 0xf4bbd50f, 0x3e54d8f6, - 0xe0000000, 0x3feab384, 0x14757c4d, 0xbe48774c, 0xc0000000, - 0x3feaa549, 0x7c7b0eea, 0x3e5b51bb, 0x20000000, 0x3fea9713, - 0xf56f7013, 0x3e386200, 0xe0000000, 0x3fea88e0, 0xbe428ebe, - 0xbe514af5, 0xe0000000, 0x3fea7ab2, 0x8d0e4496, 0x3e4f9165, - 0x60000000, 0x3fea6c89, 0xdbacc5d5, 0xbe5c063b, 0x20000000, - 0x3fea5e64, 0x3f19d970, 0xbe5a0c8c, 0x20000000, 0x3fea5043, - 0x09ea3e6b, 0x3e5065dc, 0x80000000, 0x3fea4226, 0x78df246c, - 0x3e5e05f6, 0x40000000, 0x3fea340e, 0x4057d4a0, 0x3e431b2b, - 0x40000000, 0x3fea25fa, 0x82867bb5, 0x3e4b76be, 0xa0000000, - 0x3fea17ea, 0x9436f40a, 0xbe5aad39, 0x20000000, 0x3fea09df, - 0x4b5253b3, 0x3e46380b, 0x00000000, 0x3fe9fbd8, 0x8fc52466, - 0xbe386f9b, 0x20000000, 0x3fe9edd5, 0x22d3f344, 0xbe538347, - 0x60000000, 0x3fe9dfd6, 0x1ac33522, 0x3e5dbc53, 0x00000000, - 0x3fe9d1dc, 0xeabdff1d, 0x3e40fc0c, 0xe0000000, 0x3fe9c3e5, - 0xafd30e73, 0xbe585e63, 0xe0000000, 0x3fe9b5f3, 0xa52f226a, - 0xbe43e8f9, 0x20000000, 0x3fe9a806, 0xecb8698d, 0xbe515b36, - 0x80000000, 0x3fe99a1c, 0xf2b4e89d, 0x3e48b62b, 0x20000000, - 0x3fe98c37, 0x7c9a88fb, 0x3e44414c, 0x00000000, 0x3fe97e56, - 0xda015741, 0xbe5d13ba, 0xe0000000, 0x3fe97078, 0x5fdace06, - 0x3e51b947, 0x00000000, 0x3fe962a0, 0x956ca094, 0x3e518785, - 0x40000000, 0x3fe954cb, 0x01164c1d, 0x3e5d5b57, 0xc0000000, - 0x3fe946fa, 0xe63b3767, 0xbe4f84e7, 0x40000000, 0x3fe9392e, - 0xe57cc2a9, 0x3e34eda3, 0xe0000000, 0x3fe92b65, 0x8c75b544, - 0x3e5766a0, 0xc0000000, 0x3fe91da1, 0x37d1d087, 0xbe5e2ab1, - 0x80000000, 0x3fe90fe1, 0xa953dc20, 0x3e5fa1f3, 0x80000000, - 0x3fe90225, 0xdbd3f369, 0x3e47d6db, 0xa0000000, 0x3fe8f46d, - 0x1c9be989, 0xbe5e2b0a, 0xa0000000, 0x3fe8e6b9, 0x3c93d76a, - 0x3e5c8618, 0xe0000000, 0x3fe8d909, 0x2182fc9a, 0xbe41aa9e, - 0x20000000, 0x3fe8cb5e, 0xe6b3539d, 0xbe530d19, 0x60000000, - 0x3fe8bdb6, 0x49e58cc3, 0xbe3bb374, 0xa0000000, 0x3fe8b012, - 0xa7cfeb8f, 0x3e56c412, 0x00000000, 0x3fe8a273, 0x8d52bc19, - 0x3e1429b8, 0x60000000, 0x3fe894d7, 0x4dc32c6c, 0xbe48604c, - 0xc0000000, 0x3fe8873f, 0x0c868e56, 0xbe564ee5, 0x00000000, - 0x3fe879ac, 0x56aee828, 0x3e5e2fd8, 0x60000000, 0x3fe86c1c, - 0x7ceab8ec, 0x3e493365, 0xc0000000, 0x3fe85e90, 0x78d4dadc, - 0xbe4f7f25, 0x00000000, 0x3fe85109, 0x0ccd8280, 0x3e31e7a2, - 0x40000000, 0x3fe84385, 0x34ba4e15, 0x3e328077, 0x80000000, - 0x3fe83605, 0xa670975a, 0xbe53eee5, 0xa0000000, 0x3fe82889, - 0xf61b77b2, 0xbe43a20a, 0xa0000000, 0x3fe81b11, 0x13e6643b, - 0x3e5e5fe5, 0xc0000000, 0x3fe80d9d, 0x82cc94e8, 0xbe5ff1f9, - 0xa0000000, 0x3fe8002d, 0x8a0c9c5d, 0xbe42b0e7, 0x60000000, - 0x3fe7f2c1, 0x22a16f01, 0x3e5d9ea0, 0x20000000, 0x3fe7e559, - 0xc38cd451, 0x3e506963, 0xc0000000, 0x3fe7d7f4, 0x9902bc71, - 0x3e4503d7, 0x40000000, 0x3fe7ca94, 0xdef2a3c0, 0x3e3d98ed, - 0xa0000000, 0x3fe7bd37, 0xed49abb0, 0x3e24c1ff, 0xe0000000, - 0x3fe7afde, 0xe3b0be70, 0xbe40c467, 0x00000000, 0x3fe7a28a, - 0xaf9f193c, 0xbe5dff6c, 0xe0000000, 0x3fe79538, 0xb74cf6b6, - 0xbe258ed0, 0xa0000000, 0x3fe787eb, 0x1d9127c7, 0x3e345fb0, - 0x40000000, 0x3fe77aa2, 0x1028c21d, 0xbe4619bd, 0xa0000000, - 0x3fe76d5c, 0x7cb0b5e4, 0x3e40f1a2, 0xe0000000, 0x3fe7601a, - 0x2b1bc4ad, 0xbe32e8bb, 0xe0000000, 0x3fe752dc, 0x6839f64e, - 0x3e41f57b, 0xc0000000, 0x3fe745a2, 0xc4121f7e, 0xbe52c40a, - 0x60000000, 0x3fe7386c, 0xd6852d72, 0xbe5c4e6b, 0xc0000000, - 0x3fe72b39, 0x91d690f7, 0xbe57f88f, 0xe0000000, 0x3fe71e0a, - 0x627a2159, 0xbe4425d5, 0xc0000000, 0x3fe710df, 0x50a54033, - 0x3e422b7e, 0x60000000, 0x3fe703b8, 0x3b0b5f91, 0x3e5d3857, - 0xe0000000, 0x3fe6f694, 0x84d628a2, 0xbe51f090, 0x00000000, - 0x3fe6e975, 0x306d8894, 0xbe414d83, 0xe0000000, 0x3fe6dc58, - 0x30bf24aa, 0xbe4650ca, 0x80000000, 0x3fe6cf40, 0xd4628d69, - 0xbe5db007, 0xc0000000, 0x3fe6c22b, 0xa2aae57b, 0xbe31d279, - 0xc0000000, 0x3fe6b51a, 0x860edf7e, 0xbe2d4c4a, 0x80000000, - 0x3fe6a80d, 0xf3559341, 0xbe5f7e98, 0xe0000000, 0x3fe69b03, - 0xa885899e, 0xbe5c2011, 0xe0000000, 0x3fe68dfd, 0x2bdc6d37, - 0x3e224a82, 0xa0000000, 0x3fe680fb, 0xc12ad1b9, 0xbe40cf56, - 0x00000000, 0x3fe673fd, 0x1bcdf659, 0xbdf52f2d, 0x00000000, - 0x3fe66702, 0x5df10408, 0x3e5663e0, 0xc0000000, 0x3fe65a0a, - 0xa4070568, 0xbe40b12f, 0x00000000, 0x3fe64d17, 0x71c54c47, - 0x3e5f5e8b, 0x00000000, 0x3fe64027, 0xbd4b7e83, 0x3e42ead6, - 0xa0000000, 0x3fe6333a, 0x61598bd2, 0xbe4c48d4, 0xc0000000, - 0x3fe62651, 0x6f538d61, 0x3e548401, 0xa0000000, 0x3fe6196c, - 0x14344120, 0xbe529af6, 0x00000000, 0x3fe60c8b, 0x5982c587, - 0xbe3e1e4f, 0x00000000, 0x3fe5ffad, 0xfe51d4ea, 0xbe4c897a, - 0x80000000, 0x3fe5f2d2, 0xfd46ebe1, 0x3e552e00, 0xa0000000, - 0x3fe5e5fb, 0xa4695699, 0x3e5ed471, 0x60000000, 0x3fe5d928, - 0x80d118ae, 0x3e456b61, 0xa0000000, 0x3fe5cc58, 0x304c330b, - 0x3e54dc29, 0x80000000, 0x3fe5bf8c, 0x0af2dedf, 0xbe3aa9bd, - 0xe0000000, 0x3fe5b2c3, 0x15fc9258, 0xbe479a37, 0xc0000000, - 0x3fe5a5fe, 0x9292c7ea, 0x3e188650, 0x20000000, 0x3fe5993d, - 0x33b4d380, 0x3e5d6d93, 0x20000000, 0x3fe58c7f, 0x02fd16c7, - 0x3e2fe961, 0xa0000000, 0x3fe57fc4, 0x4a05edb6, 0xbe4d55b4, - 0xa0000000, 0x3fe5730d, 0x3d443abb, 0xbe5e6954, 0x00000000, - 0x3fe5665a, 0x024acfea, 0x3e50e61b, 0x00000000, 0x3fe559aa, - 0xcc9edd09, 0xbe325403, 0x60000000, 0x3fe54cfd, 0x1fe26950, - 0x3e5d500e, 0x60000000, 0x3fe54054, 0x6c5ae164, 0xbe4a79b4, - 0xc0000000, 0x3fe533ae, 0x154b0287, 0xbe401571, 0xa0000000, - 0x3fe5270c, 0x0673f401, 0xbe56e56b, 0xe0000000, 0x3fe51a6d, - 0x751b639c, 0x3e235269, 0xa0000000, 0x3fe50dd2, 0x7c7b2bed, - 0x3ddec887, 0xc0000000, 0x3fe5013a, 0xafab4e17, 0x3e5e7575, - 0x60000000, 0x3fe4f4a6, 0x2e308668, 0x3e59aed6, 0x80000000, - 0x3fe4e815, 0xf33e2a76, 0xbe51f184, 0xe0000000, 0x3fe4db87, - 0x839f3e3e, 0x3e57db01, 0xc0000000, 0x3fe4cefd, 0xa9eda7bb, - 0x3e535e0f, 0x00000000, 0x3fe4c277, 0x2a8f66a5, 0x3e5ce451, - 0xc0000000, 0x3fe4b5f3, 0x05192456, 0xbe4e8518, 0xc0000000, - 0x3fe4a973, 0x4aa7cd1d, 0x3e46784a, 0x40000000, 0x3fe49cf7, - 0x8e23025e, 0xbe5749f2, 0x00000000, 0x3fe4907e, 0x18d30215, - 0x3e360f39, 0x20000000, 0x3fe48408, 0x63dcf2f3, 0x3e5e00fe, - 0xc0000000, 0x3fe47795, 0x46182d09, 0xbe5173d9, 0xa0000000, - 0x3fe46b26, 0x8f0e62aa, 0xbe48f281, 0xe0000000, 0x3fe45eba, - 0x5775c40c, 0xbe56aad4, 0x60000000, 0x3fe45252, 0x0fe25f69, - 0x3e48bd71, 0x40000000, 0x3fe445ed, 0xe9989ec5, 0x3e590d97, - 0x80000000, 0x3fe4398b, 0xb3d9ffe3, 0x3e479dbc, 0x20000000, - 0x3fe42d2d, 0x388e4d2e, 0xbe5eed80, 0xe0000000, 0x3fe420d1, - 0x6f797c18, 0x3e554b4c, 0x20000000, 0x3fe4147a, 0x31048bb4, - 0xbe5b1112, 0x80000000, 0x3fe40825, 0x2efba4f9, 0x3e48ebc7, - 0x40000000, 0x3fe3fbd4, 0x50201119, 0x3e40b701, 0x40000000, - 0x3fe3ef86, 0x0a4db32c, 0x3e551de8, 0xa0000000, 0x3fe3e33b, - 0x0c9c148b, 0xbe50c1f6, 0x20000000, 0x3fe3d6f4, 0xc9129447, - 0x3e533fa0, 0x00000000, 0x3fe3cab0, 0xaae5b5a0, 0xbe22b68e, - 0x20000000, 0x3fe3be6f, 0x02305e8a, 0xbe54fc08, 0x60000000, - 0x3fe3b231, 0x7f908258, 0x3e57dc05, 0x00000000, 0x3fe3a5f7, - 0x1a09af78, 0x3e08038b, 0xe0000000, 0x3fe399bf, 0x490643c1, - 0xbe5dbe42, 0xe0000000, 0x3fe38d8b, 0x5e8ad724, 0xbe3c2b72, - 0x20000000, 0x3fe3815b, 0xc67196b6, 0x3e1713cf, 0xa0000000, - 0x3fe3752d, 0x6182e429, 0xbe3ec14c, 0x40000000, 0x3fe36903, - 0xab6eb1ae, 0x3e5a2cc5, 0x40000000, 0x3fe35cdc, 0xfe5dc064, - 0xbe5c5878, 0x40000000, 0x3fe350b8, 0x0ba6b9e4, 0x3e51619b, - 0x80000000, 0x3fe34497, 0x857761aa, 0x3e5fff53, 0x00000000, - 0x3fe3387a, 0xf872d68c, 0x3e484f4d, 0xa0000000, 0x3fe32c5f, - 0x087e97c2, 0x3e52842e, 0x80000000, 0x3fe32048, 0x73d6d0c0, - 0xbe503edf, 0x80000000, 0x3fe31434, 0x0c1456a1, 0xbe5f72ad, - 0xa0000000, 0x3fe30823, 0x83a1a4d5, 0xbe5e65cc, 0xe0000000, - 0x3fe2fc15, 0x855a7390, 0xbe506438, 0x40000000, 0x3fe2f00b, - 0xa2898287, 0x3e3d22a2, 0xe0000000, 0x3fe2e403, 0x8b56f66f, - 0xbe5aa5fd, 0x80000000, 0x3fe2d7ff, 0x52db119a, 0x3e3a2e3d, - 0x60000000, 0x3fe2cbfe, 0xe2ddd4c0, 0xbe586469, 0x40000000, - 0x3fe2c000, 0x6b01bf10, 0x3e352b9d, 0x40000000, 0x3fe2b405, - 0xb07a1cdf, 0x3e5c5cda, 0x80000000, 0x3fe2a80d, 0xc7b5f868, - 0xbe5668b3, 0xc0000000, 0x3fe29c18, 0x185edf62, 0xbe563d66, - 0x00000000, 0x3fe29027, 0xf729e1cc, 0x3e59a9a0, 0x80000000, - 0x3fe28438, 0x6433c727, 0xbe43cc89, 0x00000000, 0x3fe2784d, - 0x41782631, 0xbe30750c, 0xa0000000, 0x3fe26c64, 0x914911b7, - 0xbe58290e, 0x40000000, 0x3fe2607f, 0x3dcc73e1, 0xbe4269cd, - 0x00000000, 0x3fe2549d, 0x2751bf70, 0xbe5a6998, 0xc0000000, - 0x3fe248bd, 0x4248b9fb, 0xbe4ddb00, 0x80000000, 0x3fe23ce1, - 0xf35cf82f, 0x3e561b71, 0x60000000, 0x3fe23108, 0x8e481a2d, - 0x3e518fb9, 0x60000000, 0x3fe22532, 0x5ab96edc, 0xbe5fafc5, - 0x40000000, 0x3fe2195f, 0x80943911, 0xbe07f819, 0x40000000, - 0x3fe20d8f, 0x386f2d6c, 0xbe54ba8b, 0x40000000, 0x3fe201c2, - 0xf29664ac, 0xbe5eb815, 0x20000000, 0x3fe1f5f8, 0x64f03390, - 0x3e5e320c, 0x20000000, 0x3fe1ea31, 0x747ff696, 0x3e5ef0a5, - 0x40000000, 0x3fe1de6d, 0x3e9ceb51, 0xbe5f8d27, 0x20000000, - 0x3fe1d2ac, 0x4ae0b55e, 0x3e5faa21, 0x20000000, 0x3fe1c6ee, - 0x28569a5e, 0x3e598a4f, 0x20000000, 0x3fe1bb33, 0x54b33e07, - 0x3e46130a, 0x20000000, 0x3fe1af7b, 0x024f1078, 0xbe4dbf93, - 0x00000000, 0x3fe1a3c6, 0xb0783bfa, 0x3e419248, 0xe0000000, - 0x3fe19813, 0x2f02b836, 0x3e4e02b7, 0xc0000000, 0x3fe18c64, - 0x28dec9d4, 0x3e09064f, 0x80000000, 0x3fe180b8, 0x45cbf406, - 0x3e5b1f46, 0x40000000, 0x3fe1750f, 0x03d9964c, 0x3e5b0a79, - 0x00000000, 0x3fe16969, 0x8b5b882b, 0xbe238086, 0xa0000000, - 0x3fe15dc5, 0x73bad6f8, 0xbdf1fca4, 0x20000000, 0x3fe15225, - 0x5385769c, 0x3e5e8d76, 0xa0000000, 0x3fe14687, 0x1676dc6b, - 0x3e571d08, 0x20000000, 0x3fe13aed, 0xa8c41c7f, 0xbe598a25, - 0x60000000, 0x3fe12f55, 0xc4e1aaf0, 0x3e435277, 0xa0000000, - 0x3fe123c0, 0x403638e1, 0xbe21aa7c, 0xc0000000, 0x3fe1182e, - 0x557a092b, 0xbdd0116b, 0xc0000000, 0x3fe10c9f, 0x7d779f66, - 0x3e4a61ba, 0xc0000000, 0x3fe10113, 0x2b09c645, 0xbe5d586e, - 0x20000000, 0x3fe0ea04, 0xea2cad46, 0x3e5aa97c, 0x20000000, - 0x3fe0d300, 0x23190e54, 0x3e50f1a7, 0xa0000000, 0x3fe0bc07, - 0x1379a5a6, 0xbe51619d, 0x60000000, 0x3fe0a51a, 0x926a3d4a, - 0x3e5cf019, 0xa0000000, 0x3fe08e38, 0xa8c24358, 0x3e35241e, - 0x20000000, 0x3fe07762, 0x24317e7a, 0x3e512cfa, 0x00000000, - 0x3fe06097, 0xfd9cf274, 0xbe55bef3, 0x00000000, 0x3fe049d7, - 0x3689b49d, 0xbe36d26d, 0x40000000, 0x3fe03322, 0xf72ef6c4, - 0xbe54cd08, 0xa0000000, 0x3fe01c78, 0x23702d2d, 0xbe5900bf, - 0x00000000, 0x3fe005da, 0x3f59c14c, 0x3e57d80b, 0x40000000, - 0x3fdfde8d, 0xad67766d, 0xbe57fad4, 0x40000000, 0x3fdfb17c, - 0x644f4ae7, 0x3e1ee43b, 0x40000000, 0x3fdf8481, 0x903234d2, - 0x3e501a86, 0x40000000, 0x3fdf579c, 0xafe9e509, 0xbe267c3e, - 0x00000000, 0x3fdf2acd, 0xb7dfda0b, 0xbe48149b, 0x40000000, - 0x3fdefe13, 0x3b94305e, 0x3e5f4ea7, 0x80000000, 0x3fded16f, - 0x5d95da61, 0xbe55c198, 0x00000000, 0x3fdea4e1, 0x406960c9, - 0xbdd99a19, 0x00000000, 0x3fde7868, 0xd22f3539, 0x3e470c78, - 0x80000000, 0x3fde4c04, 0x83eec535, 0xbe3e1232, 0x40000000, - 0x3fde1fb6, 0x3dfbffcb, 0xbe4b7d71, 0x40000000, 0x3fddf37d, - 0x7e1be4e0, 0xbe5b8f8f, 0x40000000, 0x3fddc759, 0x46dae887, - 0xbe350458, 0x80000000, 0x3fdd9b4a, 0xed6ecc49, 0xbe5f0045, - 0x80000000, 0x3fdd6f50, 0x2e9e883c, 0x3e2915da, 0x80000000, - 0x3fdd436b, 0xf0bccb32, 0x3e4a68c9, 0x80000000, 0x3fdd179b, - 0x9bbfc779, 0xbe54a26a, 0x00000000, 0x3fdcebe0, 0x7cea33ab, - 0x3e43c6b7, 0x40000000, 0x3fdcc039, 0xe740fd06, 0x3e5526c2, - 0x40000000, 0x3fdc94a7, 0x9eadeb1a, 0xbe396d8d, 0xc0000000, - 0x3fdc6929, 0xf0a8f95a, 0xbe5c0ab2, 0x80000000, 0x3fdc3dc0, - 0x6ee2693b, 0x3e0992e6, 0xc0000000, 0x3fdc126b, 0x5ac6b581, - 0xbe2834b6, 0x40000000, 0x3fdbe72b, 0x8cc226ff, 0x3e3596a6, - 0x00000000, 0x3fdbbbff, 0xf92a74bb, 0x3e3c5813, 0x00000000, - 0x3fdb90e7, 0x479664c0, 0xbe50d644, 0x00000000, 0x3fdb65e3, - 0x5004975b, 0xbe55258f, 0x00000000, 0x3fdb3af3, 0xe4b23194, - 0xbe588407, 0xc0000000, 0x3fdb1016, 0xe65d4d0a, 0x3e527c26, - 0x80000000, 0x3fdae54e, 0x814fddd6, 0x3e5962a2, 0x40000000, - 0x3fdaba9a, 0xe19d0913, 0xbe562f4e, 0x80000000, 0x3fda8ff9, - 0x43cfd006, 0xbe4cfdeb, 0x40000000, 0x3fda656c, 0x686f0a4e, - 0x3e5e47a8, 0xc0000000, 0x3fda3af2, 0x7200d410, 0x3e5e1199, - 0xc0000000, 0x3fda108c, 0xabd2266e, 0x3e5ee4d1, 0x40000000, - 0x3fd9e63a, 0x396f8f2c, 0x3e4dbffb, 0x00000000, 0x3fd9bbfb, - 0xe32b25dd, 0x3e5c3a54, 0x40000000, 0x3fd991cf, 0x431e4035, - 0xbe457925, 0x80000000, 0x3fd967b6, 0x7bed3dd3, 0x3e40c61d, - 0x00000000, 0x3fd93db1, 0xd7449365, 0x3e306419, 0x80000000, - 0x3fd913be, 0x1746e791, 0x3e56fcfc, 0x40000000, 0x3fd8e9df, - 0xf3a9028b, 0xbe5041b9, 0xc0000000, 0x3fd8c012, 0x56840c50, - 0xbe26e20a, 0x40000000, 0x3fd89659, 0x19763102, 0xbe51f466, - 0x80000000, 0x3fd86cb2, 0x7032de7c, 0xbe4d298a, 0x80000000, - 0x3fd8431e, 0xdeb39fab, 0xbe4361eb, 0x40000000, 0x3fd8199d, - 0x5d01cbe0, 0xbe5425b3, 0x80000000, 0x3fd7f02e, 0x3ce99aa9, - 0x3e146fa8, 0x80000000, 0x3fd7c6d2, 0xd1a262b9, 0xbe5a1a69, - 0xc0000000, 0x3fd79d88, 0x8606c236, 0x3e423a08, 0x80000000, - 0x3fd77451, 0x8fd1e1b7, 0x3e5a6a63, 0xc0000000, 0x3fd74b2c, - 0xe491456a, 0x3e42c1ca, 0x40000000, 0x3fd7221a, 0x4499a6d7, - 0x3e36a69a, 0x00000000, 0x3fd6f91a, 0x5237df94, 0xbe0f8f02, - 0x00000000, 0x3fd6d02c, 0xb6482c6e, 0xbe5abcf7, 0x00000000, - 0x3fd6a750, 0x1919fd61, 0xbe57ade2, 0x00000000, 0x3fd67e86, - 0xaa7a994d, 0xbe3f3fbd, 0x00000000, 0x3fd655ce, 0x67db014c, - 0x3e33c550, 0x00000000, 0x3fd62d28, 0xa82856b7, 0xbe1409d1, - 0xc0000000, 0x3fd60493, 0x1e6a300d, 0x3e55d899, 0x80000000, - 0x3fd5dc11, 0x1222bd5c, 0xbe35bfc0, 0xc0000000, 0x3fd5b3a0, - 0x6e8dc2d3, 0x3e5d4d79, 0x00000000, 0x3fd58b42, 0xe0e4ace6, - 0xbe517303, 0x80000000, 0x3fd562f4, 0xb306e0a8, 0x3e5edf0f, - 0xc0000000, 0x3fd53ab8, 0x6574bc54, 0x3e5ee859, 0x80000000, - 0x3fd5128e, 0xea902207, 0x3e5f6188, 0xc0000000, 0x3fd4ea75, - 0x9f911d79, 0x3e511735, 0x80000000, 0x3fd4c26e, 0xf9c77397, - 0xbe5b1643, 0x40000000, 0x3fd49a78, 0x15fc9258, 0x3e479a37, - 0x80000000, 0x3fd47293, 0xd5a04dd9, 0xbe426e56, 0xc0000000, - 0x3fd44abf, 0xe04042f5, 0x3e56f7c6, 0x40000000, 0x3fd422fd, - 0x1d8bf2c8, 0x3e5d8810, 0x00000000, 0x3fd3fb4c, 0x88a8ddee, - 0xbe311454, 0xc0000000, 0x3fd3d3ab, 0x3e3b5e47, 0xbe5d1b72, - 0x40000000, 0x3fd3ac1c, 0xc2ab5d59, 0x3e31b02b, 0xc0000000, - 0x3fd3849d, 0xd4e34b9e, 0x3e51cb2f, 0x40000000, 0x3fd35d30, - 0x177204fb, 0xbe2b8cd7, 0x80000000, 0x3fd335d3, 0xfcd38c82, - 0xbe4356e1, 0x80000000, 0x3fd30e87, 0x64f54acc, 0xbe4e6224, - 0x00000000, 0x3fd2e74c, 0xaa7975d9, 0x3e5dc0fe, 0x80000000, - 0x3fd2c021, 0x516dab3f, 0xbe50ffa3, 0x40000000, 0x3fd29907, - 0x2bfb7313, 0x3e5674a2, 0xc0000000, 0x3fd271fd, 0x0549fc99, - 0x3e385d29, 0xc0000000, 0x3fd24b04, 0x55b63073, 0xbe500c6d, - 0x00000000, 0x3fd2241c, 0x3f91953a, 0x3e389977, 0xc0000000, - 0x3fd1fd43, 0xa1543f71, 0xbe3487ab, 0xc0000000, 0x3fd1d67b, - 0x4ec8867c, 0x3df6a2dc, 0x00000000, 0x3fd1afc4, 0x4328e3bb, - 0x3e41d9c0, 0x80000000, 0x3fd1891c, 0x2e1cda84, 0x3e3bdd87, - 0x40000000, 0x3fd16285, 0x4b5331ae, 0xbe53128e, 0x00000000, - 0x3fd13bfe, 0xb9aec164, 0xbe52ac98, 0xc0000000, 0x3fd11586, - 0xd91e1316, 0xbe350630, 0x80000000, 0x3fd0ef1f, 0x7cacc12c, - 0x3e3f5219, 0x40000000, 0x3fd0c8c8, 0xbce277b7, 0x3e3d30c0, - 0x00000000, 0x3fd0a281, 0x2a63447d, 0xbe541377, 0x80000000, - 0x3fd07c49, 0xfac483b5, 0xbe5772ec, 0xc0000000, 0x3fd05621, - 0x36b8a570, 0xbe4fd4bd, 0xc0000000, 0x3fd03009, 0xbae505f7, - 0xbe450388, 0x80000000, 0x3fd00a01, 0x3e35aead, 0xbe5430fc, - 0x80000000, 0x3fcfc811, 0x707475ac, 0x3e38806e, 0x80000000, - 0x3fcf7c3f, 0xc91817fc, 0xbe40ccea, 0x80000000, 0x3fcf308c, - 0xae05d5e9, 0xbe4919b8, 0x80000000, 0x3fcee4f8, 0xae6cc9e6, - 0xbe530b94, 0x00000000, 0x3fce9983, 0x1efe3e8e, 0x3e57747e, - 0x00000000, 0x3fce4e2d, 0xda78d9bf, 0xbe59a608, 0x00000000, - 0x3fce02f5, 0x8abe2c2e, 0x3e4a35ad, 0x00000000, 0x3fcdb7dc, - 0x1495450d, 0xbe0872cc, 0x80000000, 0x3fcd6ce1, 0x86ee0ba0, - 0xbe4f59a0, 0x00000000, 0x3fcd2205, 0xe81ca888, 0x3e5402c3, - 0x00000000, 0x3fccd747, 0x3b4424b9, 0x3e5dfdc3, 0x80000000, - 0x3fcc8ca7, 0xd305b56c, 0x3e202da6, 0x00000000, 0x3fcc4226, - 0x399a6910, 0xbe482a1c, 0x80000000, 0x3fcbf7c2, 0x747f7938, - 0xbe587372, 0x80000000, 0x3fcbad7c, 0x6fc246a0, 0x3e50d83d, - 0x00000000, 0x3fcb6355, 0xee9e9be5, 0xbe5c35bd, 0x80000000, - 0x3fcb194a, 0x8416c0bc, 0x3e546d4f, 0x00000000, 0x3fcacf5e, - 0x49f7f08f, 0x3e56da76, 0x00000000, 0x3fca858f, 0x5dc30de2, - 0x3e5f390c, 0x00000000, 0x3fca3bde, 0x950583b6, 0xbe5e4169, - 0x80000000, 0x3fc9f249, 0x33631553, 0x3e52aeb1, 0x00000000, - 0x3fc9a8d3, 0xde8795a6, 0xbe59a504, 0x00000000, 0x3fc95f79, - 0x076bf41e, 0x3e5122fe, 0x80000000, 0x3fc9163c, 0x2914c8e7, - 0x3e3dd064, 0x00000000, 0x3fc8cd1d, 0x3a30eca3, 0xbe21b4aa, - 0x80000000, 0x3fc8841a, 0xb2a96650, 0xbe575444, 0x80000000, - 0x3fc83b34, 0x2376c0cb, 0xbe2a74c7, 0x80000000, 0x3fc7f26b, - 0xd8a0b653, 0xbe5181b6, 0x00000000, 0x3fc7a9bf, 0x32257882, - 0xbe4a78b4, 0x00000000, 0x3fc7612f, 0x1eee8bd9, 0xbe1bfe9d, - 0x80000000, 0x3fc718bb, 0x0c603cc4, 0x3e36fdc9, 0x80000000, - 0x3fc6d064, 0x3728b8cf, 0xbe1e542e, 0x80000000, 0x3fc68829, - 0xc79a4067, 0x3e5c380f, 0x00000000, 0x3fc6400b, 0xf69eac69, - 0x3e550a84, 0x80000000, 0x3fc5f808, 0xb7a780a4, 0x3e5d9224, - 0x80000000, 0x3fc5b022, 0xad9dfb1e, 0xbe55242f, 0x00000000, - 0x3fc56858, 0x659b18be, 0xbe4bfda3, 0x80000000, 0x3fc520a9, - 0x66ee3631, 0xbe57d769, 0x80000000, 0x3fc4d916, 0x1ec62819, - 0x3e2427f7, 0x80000000, 0x3fc4919f, 0xdec25369, 0xbe435431, - 0x00000000, 0x3fc44a44, 0xa8acfc4b, 0xbe3c62e8, 0x00000000, - 0x3fc40304, 0xcf1d3eab, 0xbdfba29f, 0x80000000, 0x3fc3bbdf, - 0x79aba3ea, 0xbdf1b7c8, 0x80000000, 0x3fc374d6, 0xb8d186da, - 0xbe5130cf, 0x80000000, 0x3fc32de8, 0x9d74f152, 0x3e2285b6, - 0x00000000, 0x3fc2e716, 0x50ae7ca9, 0xbe503920, 0x80000000, - 0x3fc2a05e, 0x6caed92e, 0xbe533924, 0x00000000, 0x3fc259c2, - 0x9cb5034e, 0xbe510e31, 0x80000000, 0x3fc21340, 0x12c4d378, - 0xbe540b43, 0x80000000, 0x3fc1ccd9, 0xcc418706, 0x3e59887a, - 0x00000000, 0x3fc1868e, 0x921f4106, 0xbe528e67, 0x80000000, - 0x3fc1405c, 0x3969441e, 0x3e5d8051, 0x00000000, 0x3fc0fa46, - 0xd941ef5b, 0x3e5f9079, 0x80000000, 0x3fc0b44a, 0x5a3e81b2, - 0xbe567691, 0x00000000, 0x3fc06e69, 0x9d66afe7, 0xbe4d43fb, - 0x00000000, 0x3fc028a2, 0x0a92a162, 0xbe52f394, 0x00000000, - 0x3fbfc5ea, 0x209897e5, 0x3e529e37, 0x00000000, 0x3fbf3ac5, - 0x8458bd7b, 0x3e582831, 0x00000000, 0x3fbeafd5, 0xb8d8b4b8, - 0xbe486b4a, 0x00000000, 0x3fbe2518, 0xe0a3b7b6, 0x3e5bafd2, - 0x00000000, 0x3fbd9a90, 0x2bf2710e, 0x3e383b2b, 0x00000000, - 0x3fbd103c, 0x73eb6ab7, 0xbe56d78d, 0x00000000, 0x3fbc861b, - 0x32ceaff5, 0xbe32dc5a, 0x00000000, 0x3fbbfc2e, 0xbee04cb7, - 0xbe4a71a4, 0x00000000, 0x3fbb7274, 0x35ae9577, 0x3e38142f, - 0x00000000, 0x3fbae8ee, 0xcbaddab4, 0xbe5490f0, 0x00000000, - 0x3fba5f9a, 0x95ce1114, 0x3e597c71, 0x00000000, 0x3fb9d67a, - 0x6d7c0f78, 0x3e3abc2d, 0x00000000, 0x3fb94d8d, 0x2841a782, - 0xbe566cbc, 0x00000000, 0x3fb8c4d2, 0x6ed429c6, 0xbe3cfff9, - 0x00000000, 0x3fb83c4a, 0xe4a49fbb, 0xbe552964, 0x00000000, - 0x3fb7b3f4, 0x2193d81e, 0xbe42fa72, 0x00000000, 0x3fb72bd0, - 0xdd70c122, 0x3e527a8c, 0x00000000, 0x3fb6a3df, 0x03108a54, - 0xbe450393, 0x00000000, 0x3fb61c1f, 0x30ff7954, 0x3e565840, - 0x00000000, 0x3fb59492, 0xdedd460c, 0xbe5422b5, 0x00000000, - 0x3fb50d36, 0x950f9f45, 0xbe5313f6, 0x00000000, 0x3fb4860b, - 0x582cdcb1, 0x3e506d39, 0x00000000, 0x3fb3ff12, 0x7216d3a6, - 0x3e4aa719, 0x00000000, 0x3fb3784a, 0x57a423fd, 0x3e5a9b9f, - 0x00000000, 0x3fb2f1b4, 0x7a138b41, 0xbe50b418, 0x00000000, - 0x3fb26b4e, 0x2fbfd7ea, 0x3e23a53e, 0x00000000, 0x3fb1e519, - 0x18913ccb, 0x3e465fc1, 0x00000000, 0x3fb15f15, 0x7ea24e21, - 0x3e042843, 0x00000000, 0x3fb0d941, 0x7c6d9c77, 0x3e59f61e, - 0x00000000, 0x3fb0539e, 0x114efd44, 0x3e4ccab7, 0x00000000, - 0x3faf9c56, 0x1777f657, 0x3e552f65, 0x00000000, 0x3fae91d2, - 0xc317b86a, 0xbe5a61e0, 0x00000000, 0x3fad87ac, 0xb7664efb, - 0xbe41f64e, 0x00000000, 0x3fac7de6, 0x5d3d03a9, 0x3e0807a0, - 0x00000000, 0x3fab7480, 0x743c38eb, 0xbe3726e1, 0x00000000, - 0x3faa6b78, 0x06a253f1, 0x3e5ad636, 0x00000000, 0x3fa962d0, - 0xa35f541b, 0x3e5a187a, 0x00000000, 0x3fa85a88, 0x4b86e446, - 0xbe508150, 0x00000000, 0x3fa7529c, 0x2589cacf, 0x3e52938a, - 0x00000000, 0x3fa64b10, 0xaf6b11f2, 0xbe3454cd, 0x00000000, - 0x3fa543e2, 0x97506fef, 0xbe5fdec5, 0x00000000, 0x3fa43d10, - 0xe75f7dd9, 0xbe388dd3, 0x00000000, 0x3fa3369c, 0xa4139632, - 0xbdea5177, 0x00000000, 0x3fa23086, 0x352d6f1e, 0xbe565ad6, - 0x00000000, 0x3fa12acc, 0x77449eb7, 0xbe50d5c7, 0x00000000, - 0x3fa0256e, 0x7478da78, 0x3e404724, 0x00000000, 0x3f9e40dc, - 0xf59cef7f, 0xbe539d0a, 0x00000000, 0x3f9c3790, 0x1511d43c, - 0x3e53c2c8, 0x00000000, 0x3f9a2f00, 0x9b8bff3c, 0xbe43b3e1, - 0x00000000, 0x3f982724, 0xad1e22a5, 0x3e46f0bd, 0x00000000, - 0x3f962000, 0x130d9356, 0x3e475ba0, 0x00000000, 0x3f941994, - 0x8f86f883, 0xbe513d0b, 0x00000000, 0x3f9213dc, 0x914d0dc8, - 0xbe534335, 0x00000000, 0x3f900ed8, 0x2d73e5e7, 0xbe22ba75, - 0x00000000, 0x3f8c1510, 0xc5b7d70e, 0x3e599c5d, 0x00000000, - 0x3f880de0, 0x8a27857e, 0xbe3d28c8, 0x00000000, 0x3f840810, - 0xda767328, 0x3e531b3d, 0x00000000, 0x3f8003b0, 0x77bacaf3, - 0xbe5f04e3, 0x00000000, 0x3f780150, 0xdf4b0720, 0x3e5a8bff, - 0x00000000, 0x3f6ffc40, 0x34c48e71, 0xbe3fcd99, 0x00000000, - 0x3f5ff6c0, 0x1ad218af, 0xbe4c78a7, 0x00000000, 0x00000000, - 0x00000000, 0x80000000 - }; - - private static int[] logTwoPow = { - 0xfefa39ef, 0x3fe62e42, 0xfefa39ef, 0xbfe62e42 - }; - - public void powIntrinsic(Register dest, Register value1, Register value2, CompilationResultBuilder crb, AMD64MacroAssembler masm) { - ArrayDataPointerConstant highSigMaskPtr = new ArrayDataPointerConstant(highSigMask, 16); - ArrayDataPointerConstant logTwoEPtr = new ArrayDataPointerConstant(logTwoE, 16); - ArrayDataPointerConstant highmaskYPtr = new ArrayDataPointerConstant(highmaskY, 16); - ArrayDataPointerConstant tExpPtr = new ArrayDataPointerConstant(tExp, 16); - ArrayDataPointerConstant eCoeffPtr = new ArrayDataPointerConstant(eCoeff, 16); - ArrayDataPointerConstant coeffHPtr = new ArrayDataPointerConstant(coeffH, 16); - ArrayDataPointerConstant highmaskLogXPtr = new ArrayDataPointerConstant(highmaskLogX, 16); - ArrayDataPointerConstant halfmaskPtr = new ArrayDataPointerConstant(halfmask, 8); - ArrayDataPointerConstant coeffPowPtr = new ArrayDataPointerConstant(coeffPow, 16); - ArrayDataPointerConstant lTblPowPtr = new ArrayDataPointerConstant(lTblPow, 16); - ArrayDataPointerConstant logTwoPowPtr = new ArrayDataPointerConstant(logTwoPow, 8); - - Label bb0 = new Label(); - Label bb1 = new Label(); - Label bb2 = new Label(); - Label bb3 = new Label(); - Label bb4 = new Label(); - Label bb5 = new Label(); - Label bb6 = new Label(); - Label bb7 = new Label(); - Label bb8 = new Label(); - Label bb9 = new Label(); - Label bb10 = new Label(); - Label bb11 = new Label(); - Label bb12 = new Label(); - Label bb13 = new Label(); - Label bb14 = new Label(); - Label bb15 = new Label(); - Label bb16 = new Label(); - Label bb18 = new Label(); - Label bb19 = new Label(); - Label bb20 = new Label(); - Label bb21 = new Label(); - Label bb22 = new Label(); - Label bb23 = new Label(); - Label bb24 = new Label(); - Label bb25 = new Label(); - Label bb26 = new Label(); - Label bb27 = new Label(); - Label bb28 = new Label(); - Label bb29 = new Label(); - Label bb30 = new Label(); - Label bb31 = new Label(); - Label bb32 = new Label(); - Label bb33 = new Label(); - Label bb34 = new Label(); - Label bb35 = new Label(); - Label bb36 = new Label(); - Label bb37 = new Label(); - Label bb38 = new Label(); - Label bb39 = new Label(); - Label bb40 = new Label(); - Label bb41 = new Label(); - Label bb42 = new Label(); - Label bb43 = new Label(); - Label bb44 = new Label(); - Label bb45 = new Label(); - Label bb46 = new Label(); - Label bb47 = new Label(); - Label bb48 = new Label(); - Label bb49 = new Label(); - Label bb50 = new Label(); - Label bb51 = new Label(); - Label bb53 = new Label(); - Label bb54 = new Label(); - Label bb55 = new Label(); - Label bb56 = new Label(); - - Register gpr1 = asRegister(gpr1Temp, AMD64Kind.QWORD); - Register gpr2 = asRegister(gpr2Temp, AMD64Kind.QWORD); - Register gpr3 = asRegister(rcxTemp, AMD64Kind.QWORD); - Register gpr4 = asRegister(gpr4Temp, AMD64Kind.QWORD); - Register gpr5 = asRegister(gpr5Temp, AMD64Kind.QWORD); - Register gpr6 = asRegister(gpr6Temp, AMD64Kind.QWORD); - Register gpr7 = asRegister(gpr7Temp, AMD64Kind.QWORD); - Register gpr8 = asRegister(gpr8Temp, AMD64Kind.QWORD); - - Register temp1 = asRegister(xmm1Temp, AMD64Kind.DOUBLE); - Register temp2 = asRegister(xmm2Temp, AMD64Kind.DOUBLE); - Register temp3 = asRegister(xmm3Temp, AMD64Kind.DOUBLE); - Register temp4 = asRegister(xmm4Temp, AMD64Kind.DOUBLE); - Register temp5 = asRegister(xmm5Temp, AMD64Kind.DOUBLE); - Register temp6 = asRegister(xmm6Temp, AMD64Kind.DOUBLE); - Register temp7 = asRegister(xmm7Temp, AMD64Kind.DOUBLE); - Register temp8 = asRegister(xmm8Temp, AMD64Kind.DOUBLE); - Register temp9 = asRegister(xmm9Temp, AMD64Kind.DOUBLE); - Register temp10 = asRegister(xmm10Temp, AMD64Kind.DOUBLE); - - setCrb(crb); - masm.movdqu(temp10, value1); - masm.movsd(temp8, value2); - if (dest.encoding != value1.encoding) { - masm.movdqu(dest, value1); - } - - masm.movq(temp9, externalAddress(logTwoEPtr)); // 0x00000000, - // 0x3ff72000 - masm.pextrw(gpr1, dest, 3); - masm.xorpd(temp2, temp2); - masm.movq(gpr2, 0x3ff0000000000000L); - masm.movdq(temp2, gpr2); - masm.movl(gpr5, 1069088768); - masm.movdq(temp7, gpr5); - masm.xorpd(temp1, temp1); - masm.movq(gpr6, 0x77f0000000000000L); - masm.movdq(temp1, gpr6); - masm.movdqu(temp3, dest); - masm.movl(gpr4, 32752); - masm.andl(gpr4, gpr1); - masm.subl(gpr4, 16368); - masm.movl(gpr3, gpr4); - masm.sarl(gpr4, 31); - masm.addl(gpr3, gpr4); - masm.xorl(gpr3, gpr4); - masm.por(dest, temp2); - masm.movdqu(temp6, externalAddress(highSigMaskPtr)); // 0x00000000, - // 0xfffff800, - // 0x00000000, - // 0xfffff800 - masm.psrlq(dest, 27); - masm.psrld(dest, 2); - masm.addl(gpr3, 16); - masm.bsrl(gpr3, gpr3); - masm.rcpps(dest, dest); - masm.psllq(temp3, 12); - masm.movl(gpr7, 8192); - masm.movdq(temp4, gpr7); - masm.psrlq(temp3, 12); - masm.subl(gpr1, 16); - masm.cmpl(gpr1, 32736); - masm.jcc(ConditionFlag.AboveEqual, bb0); - - masm.movl(gpr5, 0); - - masm.bind(bb1); - masm.mulss(dest, temp7); - masm.movl(gpr4, -1); - masm.subl(gpr3, 4); - masm.shll(gpr4); - masm.shlq(gpr4, 32); - masm.movdq(temp5, gpr4); - masm.por(temp3, temp1); - masm.subl(gpr1, 16351); - masm.cmpl(gpr1, 1); - masm.jcc(ConditionFlag.BelowEqual, bb2); - - masm.paddd(dest, temp4); - masm.pand(temp5, temp3); - masm.movdl(gpr4, dest); - masm.psllq(dest, 29); - - masm.bind(bb3); - masm.subsd(temp3, temp5); - masm.pand(dest, temp6); - masm.subl(gpr1, 1); - masm.sarl(gpr1, 4); - masm.cvtsi2sdl(temp7, gpr1); - masm.mulpd(temp5, dest); - - masm.bind(bb4); - masm.mulsd(temp3, dest); - masm.leaq(gpr8, externalAddress(coeffPowPtr)); - masm.movdqu(temp1, new AMD64Address(gpr8, 0)); // 0x6dc96112, - // 0xbf836578, - // 0xee241472, - // 0xbf9b0301 - masm.movdqu(temp4, new AMD64Address(gpr8, 16)); // 0x9f95985a, - // 0xbfb528db, - // 0xb3841d2a, - // 0xbfd619b6 - masm.movdqu(temp6, new AMD64Address(gpr8, 32)); // 0x518775e3, - // 0x3f9004f2, - // 0xac8349bb, - // 0x3fa76c9b - masm.movdqu(dest, new AMD64Address(gpr8, 48)); // 0x486ececc, - // 0x3fc4635e, - // 0x161bb241, - // 0xbf5dabe1 - masm.subsd(temp5, temp9); - masm.movl(gpr3, gpr1); - masm.sarl(gpr1, 31); - masm.addl(gpr3, gpr1); - masm.xorl(gpr1, gpr3); - masm.addl(gpr1, 1); - masm.bsrl(gpr1, gpr1); - masm.unpcklpd(temp5, temp3); - masm.addsd(temp3, temp5); - masm.leaq(gpr7, externalAddress(lTblPowPtr)); - masm.andl(gpr4, 16760832); - masm.shrl(gpr4, 10); - masm.addpd(temp5, new AMD64Address(gpr7, gpr4, Scale.Times1, -3648)); - masm.pshufd(temp2, temp3, 0x44); - masm.mulsd(temp3, temp3); - masm.mulpd(temp1, temp2); - masm.mulpd(temp4, temp2); - masm.addsd(temp5, temp7); - masm.mulsd(temp2, temp3); - masm.addpd(temp6, temp1); - masm.mulsd(temp3, temp3); - masm.addpd(dest, temp4); - masm.movdqu(temp1, temp8); - masm.pextrw(gpr3, temp8, 3); - masm.pshufd(temp7, temp5, 0xEE); - masm.movq(temp4, externalAddress(highmaskYPtr)); // 0x00000000, - // 0xfffffff8 - masm.mulpd(temp6, temp2); - masm.pshufd(temp3, temp3, 0x44); - masm.mulpd(dest, temp2); - masm.shll(gpr1, 4); - masm.subl(gpr1, 15872); - masm.andl(gpr3, 32752); - masm.addl(gpr1, gpr3); - masm.mulpd(temp3, temp6); - masm.cmpl(gpr1, 624); - masm.jcc(ConditionFlag.AboveEqual, bb5); - - masm.xorpd(temp6, temp6); - masm.movl(gpr4, 17080); - masm.pinsrw(temp6, gpr4, 3); - masm.movdqu(temp2, temp1); - masm.pand(temp4, temp1); - masm.subsd(temp1, temp4); - masm.mulsd(temp4, temp5); - masm.addsd(dest, temp7); - masm.mulsd(temp1, temp5); - masm.movdqu(temp7, temp6); - masm.addsd(temp6, temp4); - masm.leaq(gpr7, externalAddress(tExpPtr)); - masm.addpd(temp3, dest); - masm.movdl(gpr4, temp6); - masm.movl(gpr3, gpr4); - masm.andl(gpr4, 255); - masm.addl(gpr4, gpr4); - masm.movdqu(temp5, new AMD64Address(gpr7, gpr4, Scale.Times8, 0)); - masm.subsd(temp6, temp7); - masm.pshufd(dest, temp3, 0xEE); - masm.subsd(temp4, temp6); - masm.addsd(dest, temp3); - masm.addsd(temp4, temp1); - masm.mulsd(temp2, dest); - masm.leaq(gpr8, externalAddress(eCoeffPtr)); - masm.movdqu(temp7, new AMD64Address(gpr8, 0)); // 0xe78a6731, - // 0x3f55d87f, - // 0xd704a0c0, - // 0x3fac6b08 - masm.movdqu(temp3, new AMD64Address(gpr8, 16)); // 0x6fba4e77, - // 0x3f83b2ab, - // 0xff82c58f, - // 0x3fcebfbd - masm.shll(gpr3, 12); - masm.xorl(gpr3, gpr5); - masm.andl(gpr3, -1048576); - masm.movdq(temp6, gpr3); - masm.addsd(temp2, temp4); - masm.movq(gpr2, 0x3fe62e42fefa39efL); - masm.movdq(temp1, gpr2); - masm.pshufd(dest, temp2, 0x44); - masm.pshufd(temp4, temp2, 0x44); - masm.mulsd(temp1, temp2); - masm.pshufd(temp6, temp6, 0x11); - masm.mulpd(dest, dest); - masm.mulpd(temp7, temp4); - masm.paddd(temp5, temp6); - masm.mulsd(temp1, temp5); - masm.pshufd(temp6, temp5, 0xEE); - masm.mulsd(dest, dest); - masm.addpd(temp3, temp7); - masm.addsd(temp1, temp6); - masm.mulpd(dest, temp3); - masm.pshufd(temp3, dest, 0xEE); - masm.mulsd(dest, temp5); - masm.mulsd(temp3, temp5); - masm.addsd(dest, temp1); - masm.addsd(dest, temp3); - masm.addsd(dest, temp5); - masm.jmp(bb56); - - masm.bind(bb0); - masm.addl(gpr1, 16); - masm.movl(gpr4, 32752); - masm.andl(gpr4, gpr1); - masm.cmpl(gpr4, 32752); - masm.jcc(ConditionFlag.Equal, bb6); - - masm.testl(gpr1, 32768); - masm.jcc(ConditionFlag.NotEqual, bb7); - - masm.bind(bb8); - masm.movdqu(dest, temp10); - masm.movdqu(temp3, temp10); - masm.movdl(gpr4, temp3); - masm.psrlq(temp3, 32); - masm.movdl(gpr3, temp3); - masm.orl(gpr4, gpr3); - masm.cmpl(gpr4, 0); - masm.jcc(ConditionFlag.Equal, bb9); - - masm.xorpd(temp3, temp3); - masm.movl(gpr1, 18416); - masm.pinsrw(temp3, gpr1, 3); - masm.mulsd(dest, temp3); - masm.xorpd(temp2, temp2); - masm.movl(gpr1, 16368); - masm.pinsrw(temp2, gpr1, 3); - masm.movdqu(temp3, dest); - masm.pextrw(gpr1, dest, 3); - masm.por(dest, temp2); - masm.movl(gpr3, 18416); - masm.psrlq(dest, 27); - masm.psrld(dest, 2); - masm.rcpps(dest, dest); - masm.psllq(temp3, 12); - masm.movdqu(temp6, externalAddress(highSigMaskPtr)); // 0x00000000, - // 0xfffff800, - // 0x00000000, - // 0xfffff800 - masm.psrlq(temp3, 12); - masm.mulss(dest, temp7); - masm.movl(gpr4, -1024); - masm.movdl(temp5, gpr4); - masm.por(temp3, temp1); - masm.paddd(dest, temp4); - masm.psllq(temp5, 32); - masm.movdl(gpr4, dest); - masm.psllq(dest, 29); - masm.pand(temp5, temp3); - masm.movl(gpr5, 0); - masm.pand(dest, temp6); - masm.subsd(temp3, temp5); - masm.andl(gpr1, 32752); - masm.subl(gpr1, 18416); - masm.sarl(gpr1, 4); - masm.cvtsi2sdl(temp7, gpr1); - masm.mulpd(temp5, dest); - masm.jmp(bb4); - - masm.bind(bb10); - masm.movdqu(dest, temp10); - masm.movdqu(temp3, temp10); - masm.movdl(gpr4, temp3); - masm.psrlq(temp3, 32); - masm.movdl(gpr3, temp3); - masm.orl(gpr4, gpr3); - masm.cmpl(gpr4, 0); - masm.jcc(ConditionFlag.Equal, bb9); - - masm.xorpd(temp3, temp3); - masm.movl(gpr1, 18416); - masm.pinsrw(temp3, gpr1, 3); - masm.mulsd(dest, temp3); - masm.xorpd(temp2, temp2); - masm.movl(gpr1, 16368); - masm.pinsrw(temp2, gpr1, 3); - masm.movdqu(temp3, dest); - masm.pextrw(gpr1, dest, 3); - masm.por(dest, temp2); - masm.movl(gpr3, 18416); - masm.psrlq(dest, 27); - masm.psrld(dest, 2); - masm.rcpps(dest, dest); - masm.psllq(temp3, 12); - masm.movdqu(temp6, externalAddress(highSigMaskPtr)); // 0x00000000, - // 0xfffff800, - // 0x00000000, - // 0xfffff800 - masm.psrlq(temp3, 12); - masm.mulss(dest, temp7); - masm.movl(gpr4, -1024); - masm.movdl(temp5, gpr4); - masm.por(temp3, temp1); - masm.paddd(dest, temp4); - masm.psllq(temp5, 32); - masm.movdl(gpr4, dest); - masm.psllq(dest, 29); - masm.pand(temp5, temp3); - masm.movl(gpr5, Integer.MIN_VALUE); - masm.pand(dest, temp6); - masm.subsd(temp3, temp5); - masm.andl(gpr1, 32752); - masm.subl(gpr1, 18416); - masm.sarl(gpr1, 4); - masm.cvtsi2sdl(temp7, gpr1); - masm.mulpd(temp5, dest); - masm.jmp(bb4); - - masm.bind(bb5); - masm.cmpl(gpr1, 0); - masm.jcc(ConditionFlag.Less, bb11); - - masm.cmpl(gpr1, 752); - masm.jcc(ConditionFlag.AboveEqual, bb12); - - masm.addsd(dest, temp7); - masm.movq(temp4, externalAddress(halfmaskPtr)); // 0xf8000000, - // 0xffffffff - masm.addpd(temp3, dest); - masm.xorpd(temp6, temp6); - masm.movl(gpr1, 17080); - masm.pinsrw(temp6, gpr1, 3); - masm.pshufd(dest, temp3, 0xEE); - masm.addsd(dest, temp3); - masm.movdqu(temp3, temp5); - masm.addsd(temp5, dest); - masm.subsd(temp3, temp5); - masm.movdqu(temp7, temp5); - masm.pand(temp5, temp4); - masm.movdqu(temp2, temp1); - masm.pand(temp4, temp1); - masm.subsd(temp7, temp5); - masm.addsd(dest, temp3); - masm.subsd(temp1, temp4); - masm.mulsd(temp4, temp5); - masm.addsd(dest, temp7); - masm.mulsd(temp2, dest); - masm.movdqu(temp7, temp6); - masm.mulsd(temp1, temp5); - masm.addsd(temp6, temp4); - masm.movdl(gpr1, temp6); - masm.subsd(temp6, temp7); - masm.leaq(gpr7, externalAddress(tExpPtr)); - masm.movl(gpr3, gpr1); - masm.andl(gpr1, 255); - masm.addl(gpr1, gpr1); - masm.movdqu(temp5, new AMD64Address(gpr7, gpr1, Scale.Times8, 0)); - masm.addsd(temp2, temp1); - masm.leaq(gpr8, externalAddress(eCoeffPtr)); - masm.movdqu(temp7, new AMD64Address(gpr8, 0)); // 0xe78a6731, - // 0x3f55d87f, - // 0xd704a0c0, - // 0x3fac6b08 - masm.movdqu(temp3, new AMD64Address(gpr8, 16)); // 0x6fba4e77, - // 0x3f83b2ab, - // 0xff82c58f, - // 0x3fcebfbd - masm.subsd(temp4, temp6); - masm.pextrw(gpr4, temp6, 3); - masm.addsd(temp2, temp4); - masm.sarl(gpr3, 8); - masm.movl(gpr1, gpr3); - masm.sarl(gpr3, 1); - masm.subl(gpr1, gpr3); - masm.shll(gpr3, 20); - masm.xorl(gpr3, gpr5); - masm.movdl(temp6, gpr3); - masm.movq(temp1, new AMD64Address(gpr8, 32)); // 0xfefa39ef, - // 0x3fe62e42 - masm.andl(gpr4, 32767); - masm.cmpl(gpr4, 16529); - masm.jcc(ConditionFlag.Above, bb12); - - masm.pshufd(dest, temp2, 0x44); - masm.pshufd(temp4, temp2, 0x44); - masm.mulpd(dest, dest); - masm.mulpd(temp7, temp4); - masm.pshufd(temp6, temp6, 0x11); - masm.mulsd(temp1, temp2); - masm.mulsd(dest, dest); - masm.paddd(temp5, temp6); - masm.addpd(temp3, temp7); - masm.mulsd(temp1, temp5); - masm.pshufd(temp6, temp5, 0xEE); - masm.mulpd(dest, temp3); - masm.addsd(temp1, temp6); - masm.pshufd(temp3, dest, 0xEE); - masm.mulsd(dest, temp5); - masm.mulsd(temp3, temp5); - masm.shll(gpr1, 4); - masm.xorpd(temp4, temp4); - masm.addl(gpr1, 16368); - masm.pinsrw(temp4, gpr1, 3); - masm.addsd(dest, temp1); - masm.addsd(dest, temp3); - masm.movdqu(temp1, dest); - masm.addsd(dest, temp5); - masm.mulsd(dest, temp4); - masm.pextrw(gpr1, dest, 3); - masm.andl(gpr1, 32752); - masm.jcc(ConditionFlag.Equal, bb13); - - masm.cmpl(gpr1, 32752); - masm.jcc(ConditionFlag.Equal, bb14); - - masm.jmp(bb56); - - masm.bind(bb6); - masm.movdqu(temp1, temp8); - masm.movdqu(dest, temp10); - masm.movdqu(temp2, dest); - masm.movdl(gpr1, temp2); - masm.psrlq(temp2, 20); - masm.movdl(gpr4, temp2); - masm.orl(gpr1, gpr4); - masm.jcc(ConditionFlag.Equal, bb15); - - masm.movdl(gpr1, temp1); - masm.psrlq(temp1, 32); - masm.movdl(gpr4, temp1); - masm.movl(gpr3, gpr4); - masm.addl(gpr4, gpr4); - masm.orl(gpr1, gpr4); - masm.jcc(ConditionFlag.Equal, bb16); - - masm.addsd(dest, dest); - masm.jmp(bb56); - - masm.bind(bb16); - masm.xorpd(dest, dest); - masm.movl(gpr1, 16368); - masm.pinsrw(dest, gpr1, 3); - masm.jmp(bb56); - - masm.bind(bb18); - masm.addpd(dest, temp8); - masm.jmp(bb56); - - masm.bind(bb15); - masm.movdl(gpr1, temp1); - masm.movdqu(temp2, temp1); - masm.psrlq(temp1, 32); - masm.movdl(gpr4, temp1); - masm.movl(gpr3, gpr4); - masm.addl(gpr4, gpr4); - masm.orl(gpr1, gpr4); - masm.jcc(ConditionFlag.Equal, bb19); - - masm.pextrw(gpr1, temp2, 3); - masm.andl(gpr1, 32752); - masm.cmpl(gpr1, 32752); - masm.jcc(ConditionFlag.NotEqual, bb20); - - masm.movdl(gpr1, temp2); - masm.psrlq(temp2, 20); - masm.movdl(gpr4, temp2); - masm.orl(gpr1, gpr4); - masm.jcc(ConditionFlag.NotEqual, bb18); - - masm.bind(bb20); - masm.pextrw(gpr1, dest, 3); - masm.testl(gpr1, 32768); - masm.jcc(ConditionFlag.NotEqual, bb21); - - masm.testl(gpr3, Integer.MIN_VALUE); - masm.jcc(ConditionFlag.NotZero, bb22); - - masm.jmp(bb56); - - masm.bind(bb23); - masm.movdl(gpr1, temp8); - masm.testl(gpr1, 1); - masm.jcc(ConditionFlag.NotEqual, bb24); - - masm.testl(gpr1, 2); - masm.jcc(ConditionFlag.NotEqual, bb25); - - masm.jmp(bb24); - - masm.bind(bb21); - masm.shrl(gpr3, 20); - masm.andl(gpr3, 2047); - masm.cmpl(gpr3, 1075); - masm.jcc(ConditionFlag.Above, bb24); - - masm.jcc(ConditionFlag.Equal, bb26); - - masm.cmpl(gpr3, 1074); - masm.jcc(ConditionFlag.Above, bb23); - - masm.cmpl(gpr3, 1023); - masm.jcc(ConditionFlag.Below, bb24); - - masm.movdqu(temp1, temp8); - masm.movl(gpr1, 17208); - masm.xorpd(temp3, temp3); - masm.pinsrw(temp3, gpr1, 3); - masm.movdqu(temp4, temp3); - masm.addsd(temp3, temp1); - masm.subsd(temp4, temp3); - masm.addsd(temp1, temp4); - masm.pextrw(gpr1, temp1, 3); - masm.andl(gpr1, 32752); - masm.jcc(ConditionFlag.NotEqual, bb24); - - masm.movdl(gpr1, temp3); - masm.andl(gpr1, 1); - masm.jcc(ConditionFlag.Equal, bb24); - - masm.bind(bb25); - masm.pextrw(gpr1, temp8, 3); - masm.andl(gpr1, 32768); - masm.jcc(ConditionFlag.NotEqual, bb27); - - masm.jmp(bb56); - - masm.bind(bb27); - masm.xorpd(dest, dest); - masm.movl(gpr1, 32768); - masm.pinsrw(dest, gpr1, 3); - masm.jmp(bb56); - - masm.bind(bb24); - masm.pextrw(gpr1, temp8, 3); - masm.andl(gpr1, 32768); - masm.jcc(ConditionFlag.NotEqual, bb22); - - masm.xorpd(dest, dest); - masm.movl(gpr1, 32752); - masm.pinsrw(dest, gpr1, 3); - masm.jmp(bb56); - - masm.bind(bb26); - masm.movdl(gpr1, temp8); - masm.andl(gpr1, 1); - masm.jcc(ConditionFlag.Equal, bb24); - - masm.jmp(bb25); - - masm.bind(bb28); - masm.movdl(gpr1, temp1); - masm.psrlq(temp1, 20); - masm.movdl(gpr4, temp1); - masm.orl(gpr1, gpr4); - masm.jcc(ConditionFlag.Equal, bb29); - - masm.addsd(dest, temp8); - masm.jmp(bb56); - - masm.bind(bb29); - masm.movdqu(dest, temp10); - masm.pextrw(gpr1, dest, 3); - masm.cmpl(gpr1, 49136); - masm.jcc(ConditionFlag.NotEqual, bb30); - - masm.movdl(gpr3, dest); - masm.psrlq(dest, 20); - masm.movdl(gpr4, dest); - masm.orl(gpr3, gpr4); - masm.jcc(ConditionFlag.NotEqual, bb30); - - masm.xorpd(dest, dest); - masm.movl(gpr1, 32760); - masm.pinsrw(dest, gpr1, 3); - masm.jmp(bb56); - - masm.bind(bb30); - masm.andl(gpr1, 32752); - masm.subl(gpr1, 16368); - masm.pextrw(gpr4, temp8, 3); - masm.xorpd(dest, dest); - masm.xorl(gpr1, gpr4); - masm.andl(gpr1, 32768); - masm.jcc(ConditionFlag.Equal, bb31); - - masm.jmp(bb56); - - masm.bind(bb31); - masm.movl(gpr3, 32752); - masm.pinsrw(dest, gpr3, 3); - masm.jmp(bb56); - - masm.bind(bb32); - masm.movdl(gpr1, temp1); - masm.cmpl(gpr4, 17184); - masm.jcc(ConditionFlag.Above, bb33); - - masm.testl(gpr1, 1); - masm.jcc(ConditionFlag.NotEqual, bb34); - - masm.testl(gpr1, 2); - masm.jcc(ConditionFlag.Equal, bb35); - - masm.jmp(bb36); - - masm.bind(bb33); - masm.testl(gpr1, 1); - masm.jcc(ConditionFlag.Equal, bb35); - - masm.jmp(bb36); - - masm.bind(bb7); - masm.movdqu(temp2, temp10); - masm.movdl(gpr1, temp2); - masm.psrlq(temp2, 31); - masm.movdl(gpr3, temp2); - masm.orl(gpr1, gpr3); - masm.jcc(ConditionFlag.Equal, bb9); - - masm.pextrw(gpr4, temp8, 3); - masm.movdl(gpr1, temp8); - masm.movdqu(temp2, temp8); - masm.psrlq(temp2, 32); - masm.movdl(gpr3, temp2); - masm.addl(gpr3, gpr3); - masm.orl(gpr3, gpr1); - masm.jcc(ConditionFlag.Equal, bb37); - - masm.andl(gpr4, 32752); - masm.cmpl(gpr4, 32752); - masm.jcc(ConditionFlag.Equal, bb28); - - masm.cmpl(gpr4, 17200); - masm.jcc(ConditionFlag.Above, bb35); - - masm.cmpl(gpr4, 17184); - masm.jcc(ConditionFlag.AboveEqual, bb32); - - masm.cmpl(gpr4, 16368); - masm.jcc(ConditionFlag.Below, bb34); - - masm.movl(gpr1, 17208); - masm.xorpd(temp2, temp2); - masm.pinsrw(temp2, gpr1, 3); - masm.movdqu(temp4, temp2); - masm.addsd(temp2, temp1); - masm.subsd(temp4, temp2); - masm.addsd(temp1, temp4); - masm.pextrw(gpr1, temp1, 3); - masm.andl(gpr1, 32767); - masm.jcc(ConditionFlag.NotEqual, bb34); - - masm.movdl(gpr1, temp2); - masm.andl(gpr1, 1); - masm.jcc(ConditionFlag.Equal, bb35); - - masm.bind(bb36); - masm.xorpd(temp1, temp1); - masm.movl(gpr4, 30704); - masm.pinsrw(temp1, gpr4, 3); - masm.pextrw(gpr1, temp10, 3); - masm.movl(gpr4, 8192); - masm.movdl(temp4, gpr4); - masm.andl(gpr1, 32767); - masm.subl(gpr1, 16); - masm.jcc(ConditionFlag.Less, bb10); - - masm.movl(gpr4, gpr1); - masm.andl(gpr4, 32752); - masm.subl(gpr4, 16368); - masm.movl(gpr3, gpr4); - masm.sarl(gpr4, 31); - masm.addl(gpr3, gpr4); - masm.xorl(gpr3, gpr4); - masm.addl(gpr3, 16); - masm.bsrl(gpr3, gpr3); - masm.movl(gpr5, Integer.MIN_VALUE); - masm.jmp(bb1); - - masm.bind(bb34); - masm.xorpd(temp1, temp1); - masm.movl(gpr1, 32752); - masm.pinsrw(temp1, gpr1, 3); - masm.xorpd(dest, dest); - masm.mulsd(dest, temp1); - masm.jmp(bb56); - - masm.bind(bb35); - masm.xorpd(temp1, temp1); - masm.movl(gpr4, 30704); - masm.pinsrw(temp1, gpr4, 3); - masm.pextrw(gpr1, temp10, 3); - masm.movl(gpr4, 8192); - masm.movdl(temp4, gpr4); - masm.andl(gpr1, 32767); - masm.subl(gpr1, 16); - masm.jcc(ConditionFlag.Less, bb8); - - masm.movl(gpr4, gpr1); - masm.andl(gpr4, 32752); - masm.subl(gpr4, 16368); - masm.movl(gpr3, gpr4); - masm.sarl(gpr4, 31); - masm.addl(gpr3, gpr4); - masm.xorl(gpr3, gpr4); - masm.addl(gpr3, 16); - masm.bsrl(gpr3, gpr3); - masm.movl(gpr5, 0); - masm.jmp(bb1); - - masm.bind(bb19); - masm.xorpd(dest, dest); - masm.movl(gpr1, 16368); - masm.pinsrw(dest, gpr1, 3); - masm.jmp(bb56); - - masm.bind(bb22); - masm.xorpd(dest, dest); - masm.jmp(bb56); - - masm.bind(bb11); - masm.addl(gpr1, 384); - masm.cmpl(gpr1, 0); - masm.jcc(ConditionFlag.Less, bb38); - - masm.mulsd(temp5, temp1); - masm.addsd(dest, temp7); - masm.shrl(gpr5, 31); - masm.addpd(temp3, dest); - masm.pshufd(dest, temp3, 0xEE); - masm.addsd(temp3, dest); - masm.leaq(gpr7, externalAddress(logTwoPowPtr)); // 0xfefa39ef, - // 0x3fe62e42, - // 0xfefa39ef, - // 0xbfe62e42 - masm.movq(temp4, new AMD64Address(gpr7, gpr5, Scale.Times8, 0)); - masm.mulsd(temp1, temp3); - masm.xorpd(dest, dest); - masm.movl(gpr1, 16368); - masm.shll(gpr5, 15); - masm.orl(gpr1, gpr5); - masm.pinsrw(dest, gpr1, 3); - masm.addsd(temp5, temp1); - masm.mulsd(temp5, temp4); - masm.addsd(dest, temp5); - masm.jmp(bb56); - - masm.bind(bb38); - - masm.bind(bb37); - masm.xorpd(dest, dest); - masm.movl(gpr1, 16368); - masm.pinsrw(dest, gpr1, 3); - masm.jmp(bb56); - - masm.bind(bb39); - masm.xorpd(dest, dest); - masm.movl(gpr1, 16368); - masm.pinsrw(dest, gpr1, 3); - masm.jmp(bb56); - - masm.bind(bb9); - masm.movdqu(temp2, temp8); - masm.pextrw(gpr1, temp8, 3); - masm.andl(gpr1, 32752); - masm.cmpl(gpr1, 32752); - masm.jcc(ConditionFlag.NotEqual, bb40); - - masm.movdl(gpr1, temp2); - masm.psrlq(temp2, 20); - masm.movdl(gpr4, temp2); - masm.orl(gpr1, gpr4); - masm.jcc(ConditionFlag.NotEqual, bb18); - - masm.bind(bb40); - masm.movdl(gpr1, temp1); - masm.psrlq(temp1, 32); - masm.movdl(gpr4, temp1); - masm.movl(gpr3, gpr4); - masm.addl(gpr4, gpr4); - masm.orl(gpr1, gpr4); - masm.jcc(ConditionFlag.Equal, bb39); - - masm.shrl(gpr4, 21); - masm.cmpl(gpr4, 1075); - masm.jcc(ConditionFlag.Above, bb41); - - masm.jcc(ConditionFlag.Equal, bb42); - - masm.cmpl(gpr4, 1023); - masm.jcc(ConditionFlag.Below, bb41); - - masm.movdqu(temp1, temp8); - masm.movl(gpr1, 17208); - masm.xorpd(temp3, temp3); - masm.pinsrw(temp3, gpr1, 3); - masm.movdqu(temp4, temp3); - masm.addsd(temp3, temp1); - masm.subsd(temp4, temp3); - masm.addsd(temp1, temp4); - masm.pextrw(gpr1, temp1, 3); - masm.andl(gpr1, 32752); - masm.jcc(ConditionFlag.NotEqual, bb41); - - masm.movdl(gpr1, temp3); - masm.andl(gpr1, 1); - masm.jcc(ConditionFlag.Equal, bb41); - - masm.bind(bb43); - masm.movdqu(dest, temp10); - masm.testl(gpr3, Integer.MIN_VALUE); - masm.jcc(ConditionFlag.NotEqual, bb44); - - masm.jmp(bb56); - - masm.bind(bb42); - masm.movdl(gpr1, temp8); - masm.testl(gpr1, 1); - masm.jcc(ConditionFlag.NotEqual, bb43); - - masm.bind(bb41); - masm.testl(gpr3, Integer.MIN_VALUE); - masm.jcc(ConditionFlag.Equal, bb22); - - masm.xorpd(dest, dest); - - masm.bind(bb44); - masm.movl(gpr1, 16368); - masm.xorpd(temp1, temp1); - masm.pinsrw(temp1, gpr1, 3); - masm.divsd(temp1, dest); - masm.movdqu(dest, temp1); - masm.jmp(bb56); - - masm.bind(bb12); - masm.pextrw(gpr1, temp10, 3); - masm.pextrw(gpr4, temp8, 3); - masm.movl(gpr3, 32752); - masm.andl(gpr3, gpr4); - masm.cmpl(gpr3, 32752); - masm.jcc(ConditionFlag.Equal, bb45); - - masm.andl(gpr1, 32752); - masm.subl(gpr1, 16368); - masm.xorl(gpr4, gpr1); - masm.testl(gpr4, 32768); - masm.jcc(ConditionFlag.NotEqual, bb46); - - masm.bind(bb47); - masm.movl(gpr1, 32736); - masm.pinsrw(dest, gpr1, 3); - masm.shrl(gpr5, 16); - masm.orl(gpr1, gpr5); - masm.pinsrw(temp1, gpr1, 3); - masm.mulsd(dest, temp1); - - masm.bind(bb14); - masm.jmp(bb56); - - masm.bind(bb46); - masm.movl(gpr1, 16); - masm.pinsrw(dest, gpr1, 3); - masm.mulsd(dest, dest); - masm.testl(gpr3, Integer.MIN_VALUE); - masm.jcc(ConditionFlag.Equal, bb48); - - masm.movq(gpr2, 0x8000000000000000L); - masm.movdq(temp2, gpr2); - masm.xorpd(dest, temp2); - - masm.bind(bb48); - masm.jmp(bb56); - - masm.bind(bb13); - masm.pextrw(gpr3, temp5, 3); - masm.pextrw(gpr4, temp4, 3); - masm.movl(gpr1, -1); - masm.andl(gpr3, 32752); - masm.subl(gpr3, 16368); - masm.andl(gpr4, 32752); - masm.addl(gpr4, gpr3); - masm.movl(gpr3, -31); - masm.sarl(gpr4, 4); - masm.subl(gpr3, gpr4); - masm.jcc(ConditionFlag.LessEqual, bb49); - - masm.cmpl(gpr3, 20); - masm.jcc(ConditionFlag.Above, bb50); - - masm.shll(gpr1); - - masm.bind(bb49); - masm.movdl(dest, gpr1); - masm.psllq(dest, 32); - masm.pand(dest, temp5); - masm.subsd(temp5, dest); - masm.addsd(temp5, temp1); - masm.mulsd(dest, temp4); - masm.mulsd(temp5, temp4); - masm.addsd(dest, temp5); - - masm.bind(bb50); - masm.jmp(bb48); - - masm.bind(bb2); - masm.pextrw(gpr3, temp8, 3); - masm.movl(gpr4, Integer.MIN_VALUE); - masm.movdl(temp1, gpr4); - masm.xorpd(temp7, temp7); - masm.paddd(dest, temp4); - masm.movdl(gpr4, dest); - masm.psllq(dest, 29); - masm.paddq(temp1, temp3); - masm.pand(temp5, temp1); - masm.andl(gpr3, 32752); - masm.cmpl(gpr3, 16560); - masm.jcc(ConditionFlag.Less, bb3); - - masm.leaq(gpr7, externalAddress(lTblPowPtr)); - masm.leaq(gpr8, externalAddress(coeffHPtr)); - masm.movdqu(temp4, new AMD64Address(gpr8, 0)); // 0x00000000, - // 0xbfd61a00, - // 0x00000000, - // 0xbf5dabe1 - masm.pand(dest, temp6); - masm.subsd(temp3, temp5); - masm.addl(gpr1, 16351); - masm.shrl(gpr1, 4); - masm.subl(gpr1, 1022); - masm.cvtsi2sdl(temp7, gpr1); - masm.mulpd(temp5, dest); - masm.mulsd(temp3, dest); - masm.subsd(temp5, temp9); - masm.pshufd(temp1, temp4, 0xE); - masm.pshufd(temp2, temp3, 0x44); - masm.unpcklpd(temp5, temp3); - masm.addsd(temp3, temp5); - masm.andl(gpr4, 16760832); - masm.shrl(gpr4, 10); - masm.addpd(temp7, new AMD64Address(gpr7, gpr4, Scale.Times1, -3648)); - masm.movdqu(temp6, temp4); - masm.mulsd(temp4, temp5); - masm.movdqu(dest, temp1); - masm.mulsd(dest, temp5); - masm.mulsd(temp6, temp2); - masm.mulsd(temp1, temp2); - masm.movdqu(temp2, temp5); - masm.mulsd(temp4, temp5); - masm.addsd(temp5, dest); - masm.movdqu(dest, temp7); - masm.addsd(temp2, temp3); - masm.addsd(temp7, temp5); - masm.mulsd(temp6, temp2); - masm.subsd(dest, temp7); - masm.movdqu(temp2, temp7); - masm.addsd(temp7, temp4); - masm.addsd(dest, temp5); - masm.subsd(temp2, temp7); - masm.addsd(temp4, temp2); - masm.pshufd(temp2, temp5, 0xEE); - masm.movdqu(temp5, temp7); - masm.addsd(temp7, temp2); - masm.addsd(temp4, dest); - masm.leaq(gpr8, externalAddress(coeffPowPtr)); - masm.movdqu(dest, new AMD64Address(gpr8, 0)); // 0x6dc96112, - // 0xbf836578, - // 0xee241472, - // 0xbf9b0301 - masm.subsd(temp5, temp7); - masm.addsd(temp6, temp4); - masm.movdqu(temp4, temp7); - masm.addsd(temp5, temp2); - masm.addsd(temp7, temp1); - masm.movdqu(temp2, new AMD64Address(gpr8, 64)); // 0x486ececc, - // 0x3fc4635e, - // 0x161bb241, - // 0xbf5dabe1 - masm.subsd(temp4, temp7); - masm.addsd(temp6, temp5); - masm.addsd(temp4, temp1); - masm.pshufd(temp5, temp7, 0xEE); - masm.movapd(temp1, temp7); - masm.addsd(temp7, temp5); - masm.subsd(temp1, temp7); - masm.addsd(temp1, temp5); - masm.movdqu(temp5, new AMD64Address(gpr8, 80)); // 0x9f95985a, - // 0xbfb528db, - // 0xf8b5787d, - // 0x3ef2531e - masm.pshufd(temp3, temp3, 0x44); - masm.addsd(temp6, temp4); - masm.addsd(temp6, temp1); - masm.movdqu(temp1, new AMD64Address(gpr8, 32)); // 0x9f95985a, - // 0xbfb528db, - // 0xb3841d2a, - // 0xbfd619b6 - masm.mulpd(dest, temp3); - masm.mulpd(temp2, temp3); - masm.pshufd(temp4, temp3, 0x44); - masm.mulpd(temp3, temp3); - masm.addpd(dest, temp1); - masm.addpd(temp5, temp2); - masm.mulsd(temp4, temp3); - masm.movq(temp2, externalAddress(highmaskLogXPtr)); // 0xf8000000, - // 0xffffffff - masm.mulpd(temp3, temp3); - masm.movdqu(temp1, temp8); - masm.pextrw(gpr3, temp8, 3); - masm.mulpd(dest, temp4); - masm.pextrw(gpr1, temp7, 3); - masm.mulpd(temp5, temp4); - masm.mulpd(dest, temp3); - masm.leaq(gpr8, externalAddress(highmaskYPtr)); - masm.movq(temp4, new AMD64Address(gpr8, 8)); // 0x00000000, - // 0xffffffff - masm.pand(temp2, temp7); - masm.addsd(temp5, temp6); - masm.subsd(temp7, temp2); - masm.addpd(temp5, dest); - masm.andl(gpr1, 32752); - masm.subl(gpr1, 16368); - masm.andl(gpr3, 32752); - masm.cmpl(gpr3, 32752); - masm.jcc(ConditionFlag.Equal, bb45); - - masm.addl(gpr3, gpr1); - masm.cmpl(gpr3, 16576); - masm.jcc(ConditionFlag.AboveEqual, bb51); - - masm.pshufd(dest, temp5, 0xEE); - masm.pand(temp4, temp1); - masm.movdqu(temp3, temp1); - masm.addsd(temp5, dest); - masm.subsd(temp1, temp4); - masm.xorpd(temp6, temp6); - masm.movl(gpr4, 17080); - masm.pinsrw(temp6, gpr4, 3); - masm.addsd(temp7, temp5); - masm.mulsd(temp4, temp2); - masm.mulsd(temp1, temp2); - masm.movdqu(temp5, temp6); - masm.mulsd(temp3, temp7); - masm.addsd(temp6, temp4); - masm.addsd(temp1, temp3); - masm.leaq(gpr8, externalAddress(eCoeffPtr)); - masm.movdqu(temp7, new AMD64Address(gpr8, 0)); // 0xe78a6731, - // 0x3f55d87f, - // 0xd704a0c0, - // 0x3fac6b08 - masm.movdl(gpr4, temp6); - masm.subsd(temp6, temp5); - masm.leaq(gpr7, externalAddress(tExpPtr)); - masm.movl(gpr3, gpr4); - masm.andl(gpr4, 255); - masm.addl(gpr4, gpr4); - masm.movdqu(temp5, new AMD64Address(gpr7, gpr4, Scale.Times8, 0)); - masm.movdqu(temp3, new AMD64Address(gpr8, 16)); // 0x6fba4e77, - // 0x3f83b2ab, - // 0xff82c58f, - // 0x3fcebfbd - masm.movq(temp2, new AMD64Address(gpr8, 32)); // 0xfefa39ef, - // 0x3fe62e42 - masm.subsd(temp4, temp6); - masm.addsd(temp4, temp1); - masm.pextrw(gpr4, temp6, 3); - masm.shrl(gpr3, 8); - masm.movl(gpr1, gpr3); - masm.shrl(gpr3, 1); - masm.subl(gpr1, gpr3); - masm.shll(gpr3, 20); - masm.movdl(temp6, gpr3); - masm.pshufd(dest, temp4, 0x44); - masm.pshufd(temp1, temp4, 0x44); - masm.mulpd(dest, dest); - masm.mulpd(temp7, temp1); - masm.pshufd(temp6, temp6, 0x11); - masm.mulsd(temp2, temp4); - masm.andl(gpr4, 32767); - masm.cmpl(gpr4, 16529); - masm.jcc(ConditionFlag.Above, bb12); - - masm.mulsd(dest, dest); - masm.paddd(temp5, temp6); - masm.addpd(temp3, temp7); - masm.mulsd(temp2, temp5); - masm.pshufd(temp6, temp5, 0xEE); - masm.mulpd(dest, temp3); - masm.addsd(temp2, temp6); - masm.pshufd(temp3, dest, 0xEE); - masm.addl(gpr1, 1023); - masm.shll(gpr1, 20); - masm.orl(gpr1, gpr5); - masm.movdl(temp4, gpr1); - masm.mulsd(dest, temp5); - masm.mulsd(temp3, temp5); - masm.addsd(dest, temp2); - masm.psllq(temp4, 32); - masm.addsd(dest, temp3); - masm.movdqu(temp1, dest); - masm.addsd(dest, temp5); - masm.mulsd(dest, temp4); - masm.pextrw(gpr1, dest, 3); - masm.andl(gpr1, 32752); - masm.jcc(ConditionFlag.Equal, bb13); - - masm.cmpl(gpr1, 32752); - masm.jcc(ConditionFlag.Equal, bb14); - - masm.jmp(bb56); - - masm.bind(bb45); - masm.movdqu(dest, temp10); - masm.xorpd(temp2, temp2); - masm.movl(gpr1, 49136); - masm.pinsrw(temp2, gpr1, 3); - masm.addsd(temp2, dest); - masm.pextrw(gpr1, temp2, 3); - masm.cmpl(gpr1, 0); - masm.jcc(ConditionFlag.NotEqual, bb53); - - masm.xorpd(dest, dest); - masm.movl(gpr1, 32760); - masm.pinsrw(dest, gpr1, 3); - masm.jmp(bb56); - - masm.bind(bb53); - masm.movdqu(temp1, temp8); - masm.movdl(gpr4, temp1); - masm.movdqu(temp3, temp1); - masm.psrlq(temp3, 20); - masm.movdl(gpr3, temp3); - masm.orl(gpr3, gpr4); - masm.jcc(ConditionFlag.Equal, bb54); - - masm.addsd(temp1, temp1); - masm.movdqu(dest, temp1); - masm.jmp(bb56); - - masm.bind(bb51); - masm.pextrw(gpr1, temp1, 3); - masm.pextrw(gpr3, temp2, 3); - masm.xorl(gpr1, gpr3); - masm.testl(gpr1, 32768); - masm.jcc(ConditionFlag.Equal, bb47); - - masm.jmp(bb46); - - masm.bind(bb54); - masm.pextrw(gpr1, dest, 3); - masm.andl(gpr1, 32752); - masm.pextrw(gpr4, temp1, 3); - masm.xorpd(dest, dest); - masm.subl(gpr1, 16368); - masm.xorl(gpr1, gpr4); - masm.testl(gpr1, 32768); - masm.jcc(ConditionFlag.Equal, bb55); - - masm.jmp(bb56); - - masm.bind(bb55); - masm.movl(gpr4, 32752); - masm.pinsrw(dest, gpr4, 3); - masm.jmp(bb56); - - masm.bind(bb56); + public final Variable emitLIRWrapper(LIRGenerator gen, Value x, Value y) { + LIRKind kind = LIRKind.combine(x, y); + RegisterValue xmm0Value = xmm0.asValue(kind); + gen.emitMove(xmm0Value, x); + RegisterValue xmm1Value = xmm1.asValue(kind); + gen.emitMove(xmm1Value, y); + gen.append(this); + Variable result = gen.newVariable(kind); + gen.emitMove(result, xmm0Value); + return result; } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathIntrinsicUnaryOp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathIntrinsicUnaryOp.java index b28c65def47..70942479d04 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathIntrinsicUnaryOp.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathIntrinsicUnaryOp.java @@ -24,3822 +24,48 @@ package org.graalvm.compiler.lir.amd64; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.ILLEGAL; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.REG; -import static org.graalvm.compiler.lir.LIRInstruction.OperandFlag.STACK; -import static jdk.vm.ci.code.ValueUtil.asRegister; +import static jdk.vm.ci.amd64.AMD64.xmm0; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.registersToValues; -import org.graalvm.compiler.asm.Label; -import org.graalvm.compiler.asm.amd64.AMD64Address; -import org.graalvm.compiler.asm.amd64.AMD64Address.Scale; -import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; -import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; import org.graalvm.compiler.core.common.LIRKind; -import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.lir.LIRInstructionClass; -import org.graalvm.compiler.lir.Opcode; -import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; -import org.graalvm.compiler.lir.asm.CompilationResultBuilder; +import org.graalvm.compiler.lir.Variable; import org.graalvm.compiler.lir.gen.LIRGeneratorTool; -import jdk.vm.ci.amd64.AMD64; -import jdk.vm.ci.amd64.AMD64.CPUFeature; import jdk.vm.ci.amd64.AMD64Kind; import jdk.vm.ci.code.Register; -import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.code.RegisterValue; import jdk.vm.ci.meta.Value; -public final class AMD64MathIntrinsicUnaryOp extends AMD64LIRInstruction { - public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64MathIntrinsicUnaryOp.class); +/** + * AMD64MathIntrinsicUnaryOp assumes that the input value is stored at the xmm0 register, and will + * emit the output value into the xmm0 register as well. + * {@link #emitLIRWrapper(LIRGeneratorTool, Value)} is provided for emitting necessary mov LIRs + * before and after this LIR instruction. + */ +public abstract class AMD64MathIntrinsicUnaryOp extends AMD64LIRInstruction { - public enum UnaryIntrinsicOpcode { - LOG, - LOG10, - SIN, - COS, - TAN, - EXP - } - - @Opcode private final UnaryIntrinsicOpcode opcode; - @Def protected Value result; + @Def protected Value output; @Use protected Value input; - @Temp({REG, ILLEGAL}) protected Value xmm1Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm2Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm3Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm4Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm5Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm6Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm7Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm8Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm9Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value xmm10Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr1Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr2Temp = Value.ILLEGAL; - @Temp protected AllocatableValue rcxTemp; - @Temp({REG, ILLEGAL}) protected Value gpr4Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr5Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr6Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr7Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr8Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr9Temp = Value.ILLEGAL; - @Temp({REG, ILLEGAL}) protected Value gpr10Temp = Value.ILLEGAL; - @Temp({STACK, ILLEGAL}) protected Value stackTemp = Value.ILLEGAL; + @Temp protected Value[] temps; - CompilationResultBuilder internalCrb; + public AMD64MathIntrinsicUnaryOp(LIRInstructionClass type, Register... registers) { + super(type); - public AMD64MathIntrinsicUnaryOp(LIRGeneratorTool tool, UnaryIntrinsicOpcode opcode, Value result, Value input, Value stackTemp) { - super(TYPE); - this.opcode = opcode; - this.result = result; - this.input = input; - if (opcode == UnaryIntrinsicOpcode.LOG || opcode == UnaryIntrinsicOpcode.LOG10 || - opcode == UnaryIntrinsicOpcode.SIN || opcode == UnaryIntrinsicOpcode.COS || - opcode == UnaryIntrinsicOpcode.TAN || opcode == UnaryIntrinsicOpcode.EXP) { - this.gpr1Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr2Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.rcxTemp = AMD64.rcx.asValue(LIRKind.value(AMD64Kind.QWORD)); - this.gpr4Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.xmm1Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm2Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm3Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm4Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm5Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm6Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm7Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); + input = xmm0.asValue(LIRKind.value(AMD64Kind.DOUBLE)); + output = xmm0.asValue(LIRKind.value(AMD64Kind.DOUBLE)); - if (opcode == UnaryIntrinsicOpcode.EXP) { - this.gpr5Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.xmm8Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm9Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm10Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - } - - if (opcode == UnaryIntrinsicOpcode.TAN) { - this.gpr5Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr6Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr7Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr8Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr9Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr10Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - } - - if (opcode == UnaryIntrinsicOpcode.SIN || opcode == UnaryIntrinsicOpcode.COS) { - this.gpr5Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr6Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr7Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr8Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr9Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.gpr10Temp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); - this.xmm8Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - this.xmm9Temp = tool.newVariable(LIRKind.value(AMD64Kind.DOUBLE)); - } - - this.stackTemp = stackTemp; - } + temps = registersToValues(registers); } - public AMD64MathIntrinsicUnaryOp(LIRGeneratorTool tool, UnaryIntrinsicOpcode opcode, Value result, Value input) { - this(tool, opcode, result, input, Value.ILLEGAL); + public final Variable emitLIRWrapper(LIRGeneratorTool gen, Value value) { + LIRKind kind = LIRKind.combine(value); + RegisterValue xmm0Value = xmm0.asValue(kind); + gen.emitMove(xmm0Value, value); + gen.append(this); + Variable result = gen.newVariable(kind); + gen.emitMove(result, xmm0Value); + return result; } - private void setCrb(CompilationResultBuilder crb) { - internalCrb = crb; - } - - private AMD64Address externalAddress(ArrayDataPointerConstant curPtr) { - return (AMD64Address) internalCrb.recordDataReferenceInCode(curPtr); - } - - @Override - public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { - switch (opcode) { - case LOG: - logIntrinsic(asRegister(result, AMD64Kind.DOUBLE), asRegister(input, AMD64Kind.DOUBLE), crb, masm); - break; - case LOG10: - log10Intrinsic(asRegister(result, AMD64Kind.DOUBLE), asRegister(input, AMD64Kind.DOUBLE), crb, masm); - break; - case SIN: - sinIntrinsic(asRegister(result, AMD64Kind.DOUBLE), asRegister(input, AMD64Kind.DOUBLE), crb, masm); - break; - case COS: - cosIntrinsic(asRegister(result, AMD64Kind.DOUBLE), asRegister(input, AMD64Kind.DOUBLE), crb, masm); - break; - case TAN: - tanIntrinsic(asRegister(result, AMD64Kind.DOUBLE), asRegister(input, AMD64Kind.DOUBLE), crb, masm); - break; - case EXP: - expIntrinsic(asRegister(result, AMD64Kind.DOUBLE), asRegister(input, AMD64Kind.DOUBLE), crb, masm); - break; - default: - throw GraalError.shouldNotReachHere(); - } - } - - private static int[] logTwoTable = { - 0xfefa3800, 0x3fe62e42, 0x93c76730, 0x3d2ef357, 0xaa241800, - 0x3fe5ee82, 0x0cda46be, 0x3d220238, 0x5c364800, 0x3fe5af40, - 0xac10c9fb, 0x3d2dfa63, 0x26bb8c00, 0x3fe5707a, 0xff3303dd, - 0x3d09980b, 0x26867800, 0x3fe5322e, 0x5d257531, 0x3d05ccc4, - 0x835a5000, 0x3fe4f45a, 0x6d93b8fb, 0xbd2e6c51, 0x6f970c00, - 0x3fe4b6fd, 0xed4c541c, 0x3cef7115, 0x27e8a400, 0x3fe47a15, - 0xf94d60aa, 0xbd22cb6a, 0xf2f92400, 0x3fe43d9f, 0x481051f7, - 0xbcfd984f, 0x2125cc00, 0x3fe4019c, 0x30f0c74c, 0xbd26ce79, - 0x0c36c000, 0x3fe3c608, 0x7cfe13c2, 0xbd02b736, 0x17197800, - 0x3fe38ae2, 0xbb5569a4, 0xbd218b7a, 0xad9d8c00, 0x3fe35028, - 0x9527e6ac, 0x3d10b83f, 0x44340800, 0x3fe315da, 0xc5a0ed9c, - 0xbd274e93, 0x57b0e000, 0x3fe2dbf5, 0x07b9dc11, 0xbd17a6e5, - 0x6d0ec000, 0x3fe2a278, 0xe797882d, 0x3d206d2b, 0x1134dc00, - 0x3fe26962, 0x05226250, 0xbd0b61f1, 0xd8bebc00, 0x3fe230b0, - 0x6e48667b, 0x3d12fc06, 0x5fc61800, 0x3fe1f863, 0xc9fe81d3, - 0xbd2a7242, 0x49ae6000, 0x3fe1c078, 0xed70e667, 0x3cccacde, - 0x40f23c00, 0x3fe188ee, 0xf8ab4650, 0x3d14cc4e, 0xf6f29800, - 0x3fe151c3, 0xa293ae49, 0xbd2edd97, 0x23c75c00, 0x3fe11af8, - 0xbb9ddcb2, 0xbd258647, 0x8611cc00, 0x3fe0e489, 0x07801742, - 0x3d1c2998, 0xe2d05400, 0x3fe0ae76, 0x887e7e27, 0x3d1f486b, - 0x0533c400, 0x3fe078bf, 0x41edf5fd, 0x3d268122, 0xbe760400, - 0x3fe04360, 0xe79539e0, 0xbd04c45f, 0xe5b20800, 0x3fe00e5a, - 0xb1727b1c, 0xbd053ba3, 0xaf7a4800, 0x3fdfb358, 0x3c164935, - 0x3d0085fa, 0xee031800, 0x3fdf4aa7, 0x6f014a8b, 0x3d12cde5, - 0x56b41000, 0x3fdee2a1, 0x5a470251, 0x3d2f27f4, 0xc3ddb000, - 0x3fde7b42, 0x5372bd08, 0xbd246550, 0x1a272800, 0x3fde148a, - 0x07322938, 0xbd1326b2, 0x484c9800, 0x3fddae75, 0x60dc616a, - 0xbd1ea42d, 0x46def800, 0x3fdd4902, 0xe9a767a8, 0x3d235baf, - 0x18064800, 0x3fdce42f, 0x3ec7a6b0, 0xbd0797c3, 0xc7455800, - 0x3fdc7ff9, 0xc15249ae, 0xbd29b6dd, 0x693fa000, 0x3fdc1c60, - 0x7fe8e180, 0x3d2cec80, 0x1b80e000, 0x3fdbb961, 0xf40a666d, - 0x3d27d85b, 0x04462800, 0x3fdb56fa, 0x2d841995, 0x3d109525, - 0x5248d000, 0x3fdaf529, 0x52774458, 0xbd217cc5, 0x3c8ad800, - 0x3fda93ed, 0xbea77a5d, 0x3d1e36f2, 0x0224f800, 0x3fda3344, - 0x7f9d79f5, 0x3d23c645, 0xea15f000, 0x3fd9d32b, 0x10d0c0b0, - 0xbd26279e, 0x43135800, 0x3fd973a3, 0xa502d9f0, 0xbd152313, - 0x635bf800, 0x3fd914a8, 0x2ee6307d, 0xbd1766b5, 0xa88b3000, - 0x3fd8b639, 0xe5e70470, 0xbd205ae1, 0x776dc800, 0x3fd85855, - 0x3333778a, 0x3d2fd56f, 0x3bd81800, 0x3fd7fafa, 0xc812566a, - 0xbd272090, 0x687cf800, 0x3fd79e26, 0x2efd1778, 0x3d29ec7d, - 0x76c67800, 0x3fd741d8, 0x49dc60b3, 0x3d2d8b09, 0xe6af1800, - 0x3fd6e60e, 0x7c222d87, 0x3d172165, 0x3e9c6800, 0x3fd68ac8, - 0x2756eba0, 0x3d20a0d3, 0x0b3ab000, 0x3fd63003, 0xe731ae00, - 0xbd2db623, 0xdf596000, 0x3fd5d5bd, 0x08a465dc, 0xbd0a0b2a, - 0x53c8d000, 0x3fd57bf7, 0xee5d40ef, 0x3d1faded, 0x0738a000, - 0x3fd522ae, 0x8164c759, 0x3d2ebe70, 0x9e173000, 0x3fd4c9e0, - 0x1b0ad8a4, 0xbd2e2089, 0xc271c800, 0x3fd4718d, 0x0967d675, - 0xbd2f27ce, 0x23d5e800, 0x3fd419b4, 0xec90e09d, 0x3d08e436, - 0x77333000, 0x3fd3c252, 0xb606bd5c, 0x3d183b54, 0x76be1000, - 0x3fd36b67, 0xb0f177c8, 0x3d116ecd, 0xe1d36000, 0x3fd314f1, - 0xd3213cb8, 0xbd28e27a, 0x7cdc9000, 0x3fd2bef0, 0x4a5004f4, - 0x3d2a9cfa, 0x1134d800, 0x3fd26962, 0xdf5bb3b6, 0x3d2c93c1, - 0x6d0eb800, 0x3fd21445, 0xba46baea, 0x3d0a87de, 0x635a6800, - 0x3fd1bf99, 0x5147bdb7, 0x3d2ca6ed, 0xcbacf800, 0x3fd16b5c, - 0xf7a51681, 0x3d2b9acd, 0x8227e800, 0x3fd1178e, 0x63a5f01c, - 0xbd2c210e, 0x67616000, 0x3fd0c42d, 0x163ceae9, 0x3d27188b, - 0x604d5800, 0x3fd07138, 0x16ed4e91, 0x3cf89cdb, 0x5626c800, - 0x3fd01eae, 0x1485e94a, 0xbd16f08c, 0x6cb3b000, 0x3fcf991c, - 0xca0cdf30, 0x3d1bcbec, 0xe4dd0000, 0x3fcef5ad, 0x65bb8e11, - 0xbcca2115, 0xffe71000, 0x3fce530e, 0x6041f430, 0x3cc21227, - 0xb0d49000, 0x3fcdb13d, 0xf715b035, 0xbd2aff2a, 0xf2656000, - 0x3fcd1037, 0x75b6f6e4, 0xbd084a7e, 0xc6f01000, 0x3fcc6ffb, - 0xc5962bd2, 0xbcf1ec72, 0x383be000, 0x3fcbd087, 0x595412b6, - 0xbd2d4bc4, 0x575bd000, 0x3fcb31d8, 0x4eace1aa, 0xbd0c358d, - 0x3c8ae000, 0x3fca93ed, 0x50562169, 0xbd287243, 0x07089000, - 0x3fc9f6c4, 0x6865817a, 0x3d29904d, 0xdcf70000, 0x3fc95a5a, - 0x58a0ff6f, 0x3d07f228, 0xeb390000, 0x3fc8beaf, 0xaae92cd1, - 0xbd073d54, 0x6551a000, 0x3fc823c1, 0x9a631e83, 0x3d1e0ddb, - 0x85445000, 0x3fc7898d, 0x70914305, 0xbd1c6610, 0x8b757000, - 0x3fc6f012, 0xe59c21e1, 0xbd25118d, 0xbe8c1000, 0x3fc6574e, - 0x2c3c2e78, 0x3d19cf8b, 0x6b544000, 0x3fc5bf40, 0xeb68981c, - 0xbd127023, 0xe4a1b000, 0x3fc527e5, 0xe5697dc7, 0x3d2633e8, - 0x8333b000, 0x3fc4913d, 0x54fdb678, 0x3d258379, 0xa5993000, - 0x3fc3fb45, 0x7e6a354d, 0xbd2cd1d8, 0xb0159000, 0x3fc365fc, - 0x234b7289, 0x3cc62fa8, 0x0c868000, 0x3fc2d161, 0xcb81b4a1, - 0x3d039d6c, 0x2a49c000, 0x3fc23d71, 0x8fd3df5c, 0x3d100d23, - 0x7e23f000, 0x3fc1aa2b, 0x44389934, 0x3d2ca78e, 0x8227e000, - 0x3fc1178e, 0xce2d07f2, 0x3d21ef78, 0xb59e4000, 0x3fc08598, - 0x7009902c, 0xbd27e5dd, 0x39dbe000, 0x3fbfe891, 0x4fa10afd, - 0xbd2534d6, 0x830a2000, 0x3fbec739, 0xafe645e0, 0xbd2dc068, - 0x63844000, 0x3fbda727, 0x1fa71733, 0x3d1a8940, 0x01bc4000, - 0x3fbc8858, 0xc65aacd3, 0x3d2646d1, 0x8dad6000, 0x3fbb6ac8, - 0x2bf768e5, 0xbd139080, 0x40b1c000, 0x3fba4e76, 0xb94407c8, - 0xbd0e42b6, 0x5d594000, 0x3fb9335e, 0x3abd47da, 0x3d23115c, - 0x2f40e000, 0x3fb8197e, 0xf96ffdf7, 0x3d0f80dc, 0x0aeac000, - 0x3fb700d3, 0xa99ded32, 0x3cec1e8d, 0x4d97a000, 0x3fb5e95a, - 0x3c5d1d1e, 0xbd2c6906, 0x5d208000, 0x3fb4d311, 0x82f4e1ef, - 0xbcf53a25, 0xa7d1e000, 0x3fb3bdf5, 0xa5db4ed7, 0x3d2cc85e, - 0xa4472000, 0x3fb2aa04, 0xae9c697d, 0xbd20b6e8, 0xd1466000, - 0x3fb1973b, 0x560d9e9b, 0xbd25325d, 0xb59e4000, 0x3fb08598, - 0x7009902c, 0xbd17e5dd, 0xc006c000, 0x3faeea31, 0x4fc93b7b, - 0xbd0e113e, 0xcdddc000, 0x3faccb73, 0x47d82807, 0xbd1a68f2, - 0xd0fb0000, 0x3faaaef2, 0x353bb42e, 0x3d20fc1a, 0x149fc000, - 0x3fa894aa, 0xd05a267d, 0xbd197995, 0xf2d4c000, 0x3fa67c94, - 0xec19afa2, 0xbd029efb, 0xd42e0000, 0x3fa466ae, 0x75bdfd28, - 0xbd2c1673, 0x2f8d0000, 0x3fa252f3, 0xe021b67b, 0x3d283e9a, - 0x89e74000, 0x3fa0415d, 0x5cf1d753, 0x3d0111c0, 0xec148000, - 0x3f9c63d2, 0x3f9eb2f3, 0x3d2578c6, 0x28c90000, 0x3f984925, - 0x325a0c34, 0xbd2aa0ba, 0x25980000, 0x3f9432a9, 0x928637fe, - 0x3d098139, 0x58938000, 0x3f902056, 0x06e2f7d2, 0xbd23dc5b, - 0xa3890000, 0x3f882448, 0xda74f640, 0xbd275577, 0x75890000, - 0x3f801015, 0x999d2be8, 0xbd10c76b, 0x59580000, 0x3f700805, - 0xcb31c67b, 0x3d2166af, 0x00000000, 0x00000000, 0x00000000, - 0x80000000 - }; - - private static int[] logTwoData = { - 0xfefa3800, 0x3fa62e42, 0x93c76730, 0x3ceef357 - }; - - private static int[] coeffLogTwoData = { - 0x92492492, 0x3fc24924, 0x00000000, 0xbfd00000, 0x3d6fb175, - 0xbfc5555e, 0x55555555, 0x3fd55555, 0x9999999a, 0x3fc99999, - 0x00000000, 0xbfe00000 - }; - - /* - * Copyright (c) 2014, 2016, Intel Corporation. All rights reserved. Intel Math Library (LIBM) - * Source Code - * - * ALGORITHM DESCRIPTION - LOG() --------------------- - * - * x=2^k * mx, mx in [1,2) - * - * Get B~1/mx based on the output of rcpps instruction (B0) B = int((B0*2^7+0.5))/2^7 - * - * Reduced argument: r=B*mx-1.0 (computed accurately in high and low parts) - * - * Result: k*log(2) - log(B) + p(r) if |x-1| >= small value (2^-6) and p(r) is a degree 7 - * polynomial -log(B) read from data table (high, low parts) Result is formed from high and low - * parts. - * - * Special cases: log(NaN) = quiet NaN, and raise invalid exception log(+INF) = that INF log(0) - * = -INF with divide-by-zero exception raised log(1) = +0 log(x) = NaN with invalid exception - * raised if x < -0, including -INF - * - */ - - public void logIntrinsic(Register dest, Register value, CompilationResultBuilder crb, AMD64MacroAssembler masm) { - ArrayDataPointerConstant logTwoTablePtr = new ArrayDataPointerConstant(logTwoTable, 16); - ArrayDataPointerConstant logTwoDataPtr = new ArrayDataPointerConstant(logTwoData, 16); - ArrayDataPointerConstant coeffLogTwoDataPtr = new ArrayDataPointerConstant(coeffLogTwoData, 16); - - Label bb0 = new Label(); - Label bb1 = new Label(); - Label bb2 = new Label(); - Label bb3 = new Label(); - Label bb4 = new Label(); - Label bb5 = new Label(); - Label bb6 = new Label(); - Label bb7 = new Label(); - Label bb8 = new Label(); - - Register gpr1 = asRegister(gpr1Temp, AMD64Kind.QWORD); - Register gpr2 = asRegister(gpr2Temp, AMD64Kind.QWORD); - Register gpr3 = asRegister(rcxTemp, AMD64Kind.QWORD); - Register gpr4 = asRegister(gpr4Temp, AMD64Kind.QWORD); - - Register temp1 = asRegister(xmm1Temp, AMD64Kind.DOUBLE); - Register temp2 = asRegister(xmm2Temp, AMD64Kind.DOUBLE); - Register temp3 = asRegister(xmm3Temp, AMD64Kind.DOUBLE); - Register temp4 = asRegister(xmm4Temp, AMD64Kind.DOUBLE); - Register temp5 = asRegister(xmm5Temp, AMD64Kind.DOUBLE); - Register temp6 = asRegister(xmm6Temp, AMD64Kind.DOUBLE); - Register temp7 = asRegister(xmm7Temp, AMD64Kind.DOUBLE); - - AMD64Address stackSlot = (AMD64Address) crb.asAddress(stackTemp); - - setCrb(crb); - masm.movdq(stackSlot, value); - if (dest.encoding != value.encoding) { - masm.movdqu(dest, value); - } - masm.movq(gpr1, 0x3ff0000000000000L); - masm.movdq(temp2, gpr1); - masm.movq(gpr3, 0x77f0000000000000L); - masm.movdq(temp3, gpr3); - masm.movl(gpr2, 32768); - masm.movdl(temp4, gpr2); - masm.movq(gpr2, 0xffffe00000000000L); - masm.movdq(temp5, gpr2); - masm.movdqu(temp1, value); - masm.pextrw(gpr1, dest, 3); - masm.por(dest, temp2); - masm.movl(gpr2, 16352); - masm.psrlq(dest, 27); - masm.leaq(gpr4, externalAddress(logTwoTablePtr)); - masm.psrld(dest, 2); - masm.rcpps(dest, dest); - masm.psllq(temp1, 12); - masm.pshufd(temp6, temp5, 0xE4); - masm.psrlq(temp1, 12); - masm.subl(gpr1, 16); - masm.cmpl(gpr1, 32736); - masm.jcc(ConditionFlag.AboveEqual, bb0); - - masm.bind(bb1); - masm.paddd(dest, temp4); - masm.por(temp1, temp3); - masm.movdl(gpr3, dest); - masm.psllq(dest, 29); - masm.pand(temp5, temp1); - masm.pand(dest, temp6); - masm.subsd(temp1, temp5); - masm.mulpd(temp5, dest); - masm.andl(gpr1, 32752); - masm.subl(gpr1, gpr2); - masm.cvtsi2sdl(temp7, gpr1); - masm.mulsd(temp1, dest); - masm.movdq(temp6, externalAddress(logTwoDataPtr)); // 0xfefa3800, - // 0x3fa62e42 - masm.movdqu(temp3, externalAddress(coeffLogTwoDataPtr)); // 0x92492492, - // 0x3fc24924, - // 0x00000000, - // 0xbfd00000 - masm.subsd(temp5, temp2); - masm.andl(gpr3, 16711680); - masm.shrl(gpr3, 12); - masm.movdqu(dest, new AMD64Address(gpr4, gpr3, Scale.Times1, 0)); - masm.leaq(gpr4, externalAddress(coeffLogTwoDataPtr)); - masm.movdqu(temp4, new AMD64Address(gpr4, 16)); // 0x3d6fb175, - // 0xbfc5555e, - // 0x55555555, - // 0x3fd55555 - masm.addsd(temp1, temp5); - masm.movdqu(temp2, new AMD64Address(gpr4, 32)); // 0x9999999a, - // 0x3fc99999, - // 0x00000000, - // 0xbfe00000 - masm.mulsd(temp6, temp7); - if (masm.supports(CPUFeature.SSE3)) { - masm.movddup(temp5, temp1); - } else { - masm.movdqu(temp5, temp1); - masm.movlhps(temp5, temp5); - } - masm.leaq(gpr4, externalAddress(logTwoDataPtr)); - masm.mulsd(temp7, new AMD64Address(gpr4, 8)); // 0x93c76730, - // 0x3ceef357 - masm.mulsd(temp3, temp1); - masm.addsd(dest, temp6); - masm.mulpd(temp4, temp5); - masm.mulpd(temp5, temp5); - if (masm.supports(CPUFeature.SSE3)) { - masm.movddup(temp6, dest); - } else { - masm.movdqu(temp6, dest); - masm.movlhps(temp6, temp6); - } - masm.addsd(dest, temp1); - masm.addpd(temp4, temp2); - masm.mulpd(temp3, temp5); - masm.subsd(temp6, dest); - masm.mulsd(temp4, temp1); - masm.pshufd(temp2, dest, 0xEE); - masm.addsd(temp1, temp6); - masm.mulsd(temp5, temp5); - masm.addsd(temp7, temp2); - masm.addpd(temp4, temp3); - masm.addsd(temp1, temp7); - masm.mulpd(temp4, temp5); - masm.addsd(temp1, temp4); - masm.pshufd(temp5, temp4, 0xEE); - masm.addsd(temp1, temp5); - masm.addsd(dest, temp1); - masm.jmp(bb8); - - masm.bind(bb0); - masm.movdq(dest, stackSlot); - masm.movdq(temp1, stackSlot); - masm.addl(gpr1, 16); - masm.cmpl(gpr1, 32768); - masm.jcc(ConditionFlag.AboveEqual, bb2); - - masm.cmpl(gpr1, 16); - masm.jcc(ConditionFlag.Below, bb3); - - masm.bind(bb4); - masm.addsd(dest, dest); - masm.jmp(bb8); - - masm.bind(bb5); - masm.jcc(ConditionFlag.Above, bb4); - - masm.cmpl(gpr3, 0); - masm.jcc(ConditionFlag.Above, bb4); - - masm.jmp(bb6); - - masm.bind(bb3); - masm.xorpd(temp1, temp1); - masm.addsd(temp1, dest); - masm.movdl(gpr3, temp1); - masm.psrlq(temp1, 32); - masm.movdl(gpr2, temp1); - masm.orl(gpr3, gpr2); - masm.cmpl(gpr3, 0); - masm.jcc(ConditionFlag.Equal, bb7); - - masm.xorpd(temp1, temp1); - masm.movl(gpr1, 18416); - masm.pinsrw(temp1, gpr1, 3); - masm.mulsd(dest, temp1); - masm.movdqu(temp1, dest); - masm.pextrw(gpr1, dest, 3); - masm.por(dest, temp2); - masm.psrlq(dest, 27); - masm.movl(gpr2, 18416); - masm.psrld(dest, 2); - masm.rcpps(dest, dest); - masm.psllq(temp1, 12); - masm.pshufd(temp6, temp5, 0xE4); - masm.psrlq(temp1, 12); - masm.jmp(bb1); - - masm.bind(bb2); - masm.movdl(gpr3, temp1); - masm.psrlq(temp1, 32); - masm.movdl(gpr2, temp1); - masm.addl(gpr2, gpr2); - masm.cmpl(gpr2, -2097152); - masm.jcc(ConditionFlag.AboveEqual, bb5); - - masm.orl(gpr3, gpr2); - masm.cmpl(gpr3, 0); - masm.jcc(ConditionFlag.Equal, bb7); - - masm.bind(bb6); - masm.xorpd(temp1, temp1); - masm.xorpd(dest, dest); - masm.movl(gpr1, 32752); - masm.pinsrw(temp1, gpr1, 3); - masm.mulsd(dest, temp1); - masm.jmp(bb8); - - masm.bind(bb7); - masm.xorpd(temp1, temp1); - masm.xorpd(dest, dest); - masm.movl(gpr1, 49136); - masm.pinsrw(dest, gpr1, 3); - masm.divsd(dest, temp1); - - masm.bind(bb8); - } - - private static int[] highmaskLogTen = { - 0xf8000000, 0xffffffff, 0x00000000, 0xffffe000 - }; - - private static int[] logTenE = { - 0x00000000, 0x3fdbc000, 0xbf2e4108, 0x3f5a7a6c - }; - - private static int[] logTenTable = { - 0x509f7800, 0x3fd34413, 0x1f12b358, 0x3d1fef31, 0x80333400, - 0x3fd32418, 0xc671d9d0, 0xbcf542bf, 0x51195000, 0x3fd30442, - 0x78a4b0c3, 0x3d18216a, 0x6fc79400, 0x3fd2e490, 0x80fa389d, - 0xbc902869, 0x89d04000, 0x3fd2c502, 0x75c2f564, 0x3d040754, - 0x4ddd1c00, 0x3fd2a598, 0xd219b2c3, 0xbcfa1d84, 0x6baa7c00, - 0x3fd28651, 0xfd9abec1, 0x3d1be6d3, 0x94028800, 0x3fd2672d, - 0xe289a455, 0xbd1ede5e, 0x78b86400, 0x3fd2482c, 0x6734d179, - 0x3d1fe79b, 0xcca3c800, 0x3fd2294d, 0x981a40b8, 0xbced34ea, - 0x439c5000, 0x3fd20a91, 0xcc392737, 0xbd1a9cc3, 0x92752c00, - 0x3fd1ebf6, 0x03c9afe7, 0x3d1e98f8, 0x6ef8dc00, 0x3fd1cd7d, - 0x71dae7f4, 0x3d08a86c, 0x8fe4dc00, 0x3fd1af25, 0xee9185a1, - 0xbcff3412, 0xace59400, 0x3fd190ee, 0xc2cab353, 0x3cf17ed9, - 0x7e925000, 0x3fd172d8, 0x6952c1b2, 0x3cf1521c, 0xbe694400, - 0x3fd154e2, 0xcacb79ca, 0xbd0bdc78, 0x26cbac00, 0x3fd1370d, - 0xf71f4de1, 0xbd01f8be, 0x72fa0800, 0x3fd11957, 0x55bf910b, - 0x3c946e2b, 0x5f106000, 0x3fd0fbc1, 0x39e639c1, 0x3d14a84b, - 0xa802a800, 0x3fd0de4a, 0xd3f31d5d, 0xbd178385, 0x0b992000, - 0x3fd0c0f3, 0x3843106f, 0xbd1f602f, 0x486ce800, 0x3fd0a3ba, - 0x8819497c, 0x3cef987a, 0x1de49400, 0x3fd086a0, 0x1caa0467, - 0x3d0faec7, 0x4c30cc00, 0x3fd069a4, 0xa4424372, 0xbd1618fc, - 0x94490000, 0x3fd04cc6, 0x946517d2, 0xbd18384b, 0xb7e84000, - 0x3fd03006, 0xe0109c37, 0xbd19a6ac, 0x798a0c00, 0x3fd01364, - 0x5121e864, 0xbd164cf7, 0x38ce8000, 0x3fcfedbf, 0x46214d1a, - 0xbcbbc402, 0xc8e62000, 0x3fcfb4ef, 0xdab93203, 0x3d1e0176, - 0x2cb02800, 0x3fcf7c5a, 0x2a2ea8e4, 0xbcfec86a, 0xeeeaa000, - 0x3fcf43fd, 0xc18e49a4, 0x3cf110a8, 0x9bb6e800, 0x3fcf0bda, - 0x923cc9c0, 0xbd15ce99, 0xc093f000, 0x3fced3ef, 0x4d4b51e9, - 0x3d1a04c7, 0xec58f800, 0x3fce9c3c, 0x163cad59, 0x3cac8260, - 0x9a907000, 0x3fce2d7d, 0x3fa93646, 0x3ce4a1c0, 0x37311000, - 0x3fcdbf99, 0x32abd1fd, 0x3d07ea9d, 0x6744b800, 0x3fcd528c, - 0x4dcbdfd4, 0xbd1b08e2, 0xe36de800, 0x3fcce653, 0x0b7b7f7f, - 0xbd1b8f03, 0x77506800, 0x3fcc7aec, 0xa821c9fb, 0x3d13c163, - 0x00ff8800, 0x3fcc1053, 0x536bca76, 0xbd074ee5, 0x70719800, - 0x3fcba684, 0xd7da9b6b, 0xbd1fbf16, 0xc6f8d800, 0x3fcb3d7d, - 0xe2220bb3, 0x3d1a295d, 0x16c15800, 0x3fcad53c, 0xe724911e, - 0xbcf55822, 0x82533800, 0x3fca6dbc, 0x6d982371, 0x3cac567c, - 0x3c19e800, 0x3fca06fc, 0x84d17d80, 0x3d1da204, 0x85ef8000, - 0x3fc9a0f8, 0x54466a6a, 0xbd002204, 0xb0ac2000, 0x3fc93bae, - 0xd601fd65, 0x3d18840c, 0x1bb9b000, 0x3fc8d71c, 0x7bf58766, - 0xbd14f897, 0x34aae800, 0x3fc8733e, 0x3af6ac24, 0xbd0f5c45, - 0x76d68000, 0x3fc81012, 0x4303e1a1, 0xbd1f9a80, 0x6af57800, - 0x3fc7ad96, 0x43fbcb46, 0x3cf4c33e, 0xa6c51000, 0x3fc74bc7, - 0x70f0eac5, 0xbd192e3b, 0xccab9800, 0x3fc6eaa3, 0xc0093dfe, - 0xbd0faf15, 0x8b60b800, 0x3fc68a28, 0xde78d5fd, 0xbc9ea4ee, - 0x9d987000, 0x3fc62a53, 0x962bea6e, 0xbd194084, 0xc9b0e800, - 0x3fc5cb22, 0x888dd999, 0x3d1fe201, 0xe1634800, 0x3fc56c93, - 0x16ada7ad, 0x3d1b1188, 0xc176c000, 0x3fc50ea4, 0x4159b5b5, - 0xbcf09c08, 0x51766000, 0x3fc4b153, 0x84393d23, 0xbcf6a89c, - 0x83695000, 0x3fc4549d, 0x9f0b8bbb, 0x3d1c4b8c, 0x538d5800, - 0x3fc3f881, 0xf49df747, 0x3cf89b99, 0xc8138000, 0x3fc39cfc, - 0xd503b834, 0xbd13b99f, 0xf0df0800, 0x3fc3420d, 0xf011b386, - 0xbd05d8be, 0xe7466800, 0x3fc2e7b2, 0xf39c7bc2, 0xbd1bb94e, - 0xcdd62800, 0x3fc28de9, 0x05e6d69b, 0xbd10ed05, 0xd015d800, - 0x3fc234b0, 0xe29b6c9d, 0xbd1ff967, 0x224ea800, 0x3fc1dc06, - 0x727711fc, 0xbcffb30d, 0x01540000, 0x3fc183e8, 0x39786c5a, - 0x3cc23f57, 0xb24d9800, 0x3fc12c54, 0xc905a342, 0x3d003a1d, - 0x82835800, 0x3fc0d54a, 0x9b9920c0, 0x3d03b25a, 0xc72ac000, - 0x3fc07ec7, 0x46f26a24, 0x3cf0fa41, 0xdd35d800, 0x3fc028ca, - 0x41d9d6dc, 0x3d034a65, 0x52474000, 0x3fbfa6a4, 0x44f66449, - 0x3d19cad3, 0x2da3d000, 0x3fbefcb8, 0x67832999, 0x3d18400f, - 0x32a10000, 0x3fbe53ce, 0x9c0e3b1a, 0xbcff62fd, 0x556b7000, - 0x3fbdabe3, 0x02976913, 0xbcf8243b, 0x97e88000, 0x3fbd04f4, - 0xec793797, 0x3d1c0578, 0x09647000, 0x3fbc5eff, 0x05fc0565, - 0xbd1d799e, 0xc6426000, 0x3fbbb9ff, 0x4625f5ed, 0x3d1f5723, - 0xf7afd000, 0x3fbb15f3, 0xdd5aae61, 0xbd1a7e1e, 0xd358b000, - 0x3fba72d8, 0x3314e4d3, 0x3d17bc91, 0x9b1f5000, 0x3fb9d0ab, - 0x9a4d514b, 0x3cf18c9b, 0x9cd4e000, 0x3fb92f69, 0x7e4496ab, - 0x3cf1f96d, 0x31f4f000, 0x3fb88f10, 0xf56479e7, 0x3d165818, - 0xbf628000, 0x3fb7ef9c, 0x26bf486d, 0xbd1113a6, 0xb526b000, - 0x3fb7510c, 0x1a1c3384, 0x3ca9898d, 0x8e31e000, 0x3fb6b35d, - 0xb3875361, 0xbd0661ac, 0xd01de000, 0x3fb6168c, 0x2a7cacfa, - 0xbd1bdf10, 0x0af23000, 0x3fb57a98, 0xff868816, 0x3cf046d0, - 0xd8ea0000, 0x3fb4df7c, 0x1515fbe7, 0xbd1fd529, 0xde3b2000, - 0x3fb44538, 0x6e59a132, 0x3d1faeee, 0xc8df9000, 0x3fb3abc9, - 0xf1322361, 0xbd198807, 0x505f1000, 0x3fb3132d, 0x0888e6ab, - 0x3d1e5380, 0x359bd000, 0x3fb27b61, 0xdfbcbb22, 0xbcfe2724, - 0x429ee000, 0x3fb1e463, 0x6eb4c58c, 0xbcfe4dd6, 0x4a673000, - 0x3fb14e31, 0x4ce1ac9b, 0x3d1ba691, 0x28b96000, 0x3fb0b8c9, - 0x8c7813b8, 0xbd0b3872, 0xc1f08000, 0x3fb02428, 0xc2bc8c2c, - 0x3cb5ea6b, 0x05a1a000, 0x3faf209c, 0x72e8f18e, 0xbce8df84, - 0xc0b5e000, 0x3fadfa6d, 0x9fdef436, 0x3d087364, 0xaf416000, - 0x3facd5c2, 0x1068c3a9, 0x3d0827e7, 0xdb356000, 0x3fabb296, - 0x120a34d3, 0x3d101a9f, 0x5dfea000, 0x3faa90e6, 0xdaded264, - 0xbd14c392, 0x6034c000, 0x3fa970ad, 0x1c9d06a9, 0xbd1b705e, - 0x194c6000, 0x3fa851e8, 0x83996ad9, 0xbd0117bc, 0xcf4ac000, - 0x3fa73492, 0xb1a94a62, 0xbca5ea42, 0xd67b4000, 0x3fa618a9, - 0x75aed8ca, 0xbd07119b, 0x9126c000, 0x3fa4fe29, 0x5291d533, - 0x3d12658f, 0x6f4d4000, 0x3fa3e50e, 0xcd2c5cd9, 0x3d1d5c70, - 0xee608000, 0x3fa2cd54, 0xd1008489, 0x3d1a4802, 0x9900e000, - 0x3fa1b6f9, 0x54fb5598, 0xbd16593f, 0x06bb6000, 0x3fa0a1f9, - 0x64ef57b4, 0xbd17636b, 0xb7940000, 0x3f9f1c9f, 0xee6a4737, - 0x3cb5d479, 0x91aa0000, 0x3f9cf7f5, 0x3a16373c, 0x3d087114, - 0x156b8000, 0x3f9ad5ed, 0x836c554a, 0x3c6900b0, 0xd4764000, - 0x3f98b67f, 0xed12f17b, 0xbcffc974, 0x77dec000, 0x3f9699a7, - 0x232ce7ea, 0x3d1e35bb, 0xbfbf4000, 0x3f947f5d, 0xd84ffa6e, - 0x3d0e0a49, 0x82c7c000, 0x3f92679c, 0x8d170e90, 0xbd14d9f2, - 0xadd20000, 0x3f90525d, 0x86d9f88e, 0x3cdeb986, 0x86f10000, - 0x3f8c7f36, 0xb9e0a517, 0x3ce29faa, 0xb75c8000, 0x3f885e9e, - 0x542568cb, 0xbd1f7bdb, 0x46b30000, 0x3f8442e8, 0xb954e7d9, - 0x3d1e5287, 0xb7e60000, 0x3f802c07, 0x22da0b17, 0xbd19fb27, - 0x6c8b0000, 0x3f7833e3, 0x821271ef, 0xbd190f96, 0x29910000, - 0x3f701936, 0xbc3491a5, 0xbd1bcf45, 0x354a0000, 0x3f600fe3, - 0xc0ff520a, 0xbd19d71c, 0x00000000, 0x00000000, 0x00000000, - 0x00000000 - }; - - private static int[] logTwoLogTenData = { - 0x509f7800, 0x3f934413, 0x1f12b358, 0x3cdfef31 - }; - - private static int[] coeffLogTenData = { - 0xc1a5f12e, 0x40358874, 0x64d4ef0d, 0xc0089309, 0x385593b1, - 0xc025c917, 0xdc963467, 0x3ffc6a02, 0x7f9d3aa1, 0x4016ab9f, - 0xdc77b115, 0xbff27af2 - }; - - /* - * Copyright (c) 2014, 2016, Intel Corporation. All rights reserved. Intel Math Library (LIBM) - * Source Code - * - * ALGORITHM DESCRIPTION - LOG10() --------------------- - * - * Let x=2^k * mx, mx in [1,2) - * - * Get B~1/mx based on the output of rcpss instruction (B0) B = int((B0*LH*2^7+0.5))/2^7 LH is a - * short approximation for log10(e) - * - * Reduced argument: r=B*mx-LH (computed accurately in high and low parts) - * - * Result: k*log10(2) - log(B) + p(r) p(r) is a degree 7 polynomial -log(B) read from data table - * (high, low parts) Result is formed from high and low parts - * - * Special cases: log10(0) = -INF with divide-by-zero exception raised log10(1) = +0 log10(x) = - * NaN with invalid exception raised if x < -0, including -INF log10(+INF) = +INF - * - */ - - public void log10Intrinsic(Register dest, Register value, CompilationResultBuilder crb, AMD64MacroAssembler masm) { - ArrayDataPointerConstant highmaskLogTenPtr = new ArrayDataPointerConstant(highmaskLogTen, 16); - ArrayDataPointerConstant logTenEPtr = new ArrayDataPointerConstant(logTenE, 16); - ArrayDataPointerConstant logTenTablePtr = new ArrayDataPointerConstant(logTenTable, 16); - ArrayDataPointerConstant logTwoLogTenDataPtr = new ArrayDataPointerConstant(logTwoLogTenData, 16); - ArrayDataPointerConstant coeffLogTenDataPtr = new ArrayDataPointerConstant(coeffLogTenData, 16); - - Label bb0 = new Label(); - Label bb1 = new Label(); - Label bb2 = new Label(); - Label bb3 = new Label(); - Label bb4 = new Label(); - Label bb5 = new Label(); - Label bb6 = new Label(); - Label bb7 = new Label(); - Label bb8 = new Label(); - - Register gpr1 = asRegister(gpr1Temp, AMD64Kind.QWORD); - Register gpr2 = asRegister(gpr2Temp, AMD64Kind.QWORD); - Register gpr3 = asRegister(rcxTemp, AMD64Kind.QWORD); - Register gpr4 = asRegister(gpr4Temp, AMD64Kind.QWORD); - - Register temp1 = asRegister(xmm1Temp, AMD64Kind.DOUBLE); - Register temp2 = asRegister(xmm2Temp, AMD64Kind.DOUBLE); - Register temp3 = asRegister(xmm3Temp, AMD64Kind.DOUBLE); - Register temp4 = asRegister(xmm4Temp, AMD64Kind.DOUBLE); - Register temp5 = asRegister(xmm5Temp, AMD64Kind.DOUBLE); - Register temp6 = asRegister(xmm6Temp, AMD64Kind.DOUBLE); - Register temp7 = asRegister(xmm7Temp, AMD64Kind.DOUBLE); - - AMD64Address stackSlot = (AMD64Address) crb.asAddress(stackTemp); - - setCrb(crb); - masm.movdq(stackSlot, value); - if (dest.encoding != value.encoding) { - masm.movdqu(dest, value); - } - masm.movdqu(temp5, externalAddress(highmaskLogTenPtr)); // 0xf8000000, - // 0xffffffff, - // 0x00000000, - // 0xffffe000 - masm.xorpd(temp2, temp2); - masm.movl(gpr1, 16368); - masm.pinsrw(temp2, gpr1, 3); - masm.movl(gpr2, 1054736384); - masm.movdl(temp7, gpr2); - masm.xorpd(temp3, temp3); - masm.movl(gpr3, 30704); - masm.pinsrw(temp3, gpr3, 3); - masm.movl(gpr3, 32768); - masm.movdl(temp4, gpr3); - masm.movdqu(temp1, value); - masm.pextrw(gpr1, dest, 3); - masm.por(dest, temp2); - masm.movl(gpr2, 16352); - masm.psrlq(dest, 27); - masm.movdqu(temp2, externalAddress(logTenEPtr)); // 0x00000000, - // 0x3fdbc000, - // 0xbf2e4108, - // 0x3f5a7a6c - masm.psrld(dest, 2); - masm.rcpps(dest, dest); - masm.psllq(temp1, 12); - masm.pshufd(temp6, temp5, 0x4E); - masm.psrlq(temp1, 12); - masm.subl(gpr1, 16); - masm.cmpl(gpr1, 32736); - masm.jcc(ConditionFlag.AboveEqual, bb0); - - masm.bind(bb1); - masm.mulss(dest, temp7); - masm.por(temp1, temp3); - masm.andpd(temp5, temp1); - masm.paddd(dest, temp4); - masm.movdqu(temp3, externalAddress(coeffLogTenDataPtr)); // 0xc1a5f12e, - // 0x40358874, - // 0x64d4ef0d, - // 0xc0089309 - masm.leaq(gpr4, externalAddress(coeffLogTenDataPtr)); - masm.movdqu(temp4, new AMD64Address(gpr4, 16)); // 0x385593b1, - // 0xc025c917, - // 0xdc963467, - // 0x3ffc6a02 - masm.subsd(temp1, temp5); - masm.movdl(gpr3, dest); - masm.psllq(dest, 29); - masm.andpd(dest, temp6); - masm.movdq(temp6, externalAddress(logTwoLogTenDataPtr)); // 0x509f7800, - // 0x3f934413 - masm.andl(gpr1, 32752); - masm.subl(gpr1, gpr2); - masm.cvtsi2sdl(temp7, gpr1); - masm.mulpd(temp5, dest); - masm.mulsd(temp1, dest); - masm.subsd(temp5, temp2); - masm.movdqu(temp2, new AMD64Address(gpr4, 32)); // 0x7f9d3aa1, - // 0x4016ab9f, - // 0xdc77b115, - // 0xbff27af2 - masm.leaq(gpr4, externalAddress(logTenTablePtr)); - masm.andl(gpr3, 16711680); - masm.shrl(gpr3, 12); - masm.movdqu(dest, new AMD64Address(gpr4, gpr3, Scale.Times1, -1504)); - masm.addsd(temp1, temp5); - masm.mulsd(temp6, temp7); - masm.pshufd(temp5, temp1, 0x44); - masm.leaq(gpr4, externalAddress(logTwoLogTenDataPtr)); - masm.mulsd(temp7, new AMD64Address(gpr4, 8)); // 0x1f12b358, - // 0x3cdfef31 - masm.mulsd(temp3, temp1); - masm.addsd(dest, temp6); - masm.mulpd(temp4, temp5); - masm.leaq(gpr4, externalAddress(logTenEPtr)); - masm.movdq(temp6, new AMD64Address(gpr4, 8)); // 0xbf2e4108, - // 0x3f5a7a6c - masm.mulpd(temp5, temp5); - masm.addpd(temp4, temp2); - masm.mulpd(temp3, temp5); - masm.pshufd(temp2, dest, 0xE4); - masm.addsd(dest, temp1); - masm.mulsd(temp4, temp1); - masm.subsd(temp2, dest); - masm.mulsd(temp6, temp1); - masm.addsd(temp1, temp2); - masm.pshufd(temp2, dest, 0xEE); - masm.mulsd(temp5, temp5); - masm.addsd(temp7, temp2); - masm.addsd(temp1, temp6); - masm.addpd(temp4, temp3); - masm.addsd(temp1, temp7); - masm.mulpd(temp4, temp5); - masm.addsd(temp1, temp4); - masm.pshufd(temp5, temp4, 0xEE); - masm.addsd(temp1, temp5); - masm.addsd(dest, temp1); - masm.jmp(bb8); - - masm.bind(bb0); - masm.movdq(dest, stackSlot); - masm.movdq(temp1, stackSlot); - masm.addl(gpr1, 16); - masm.cmpl(gpr1, 32768); - masm.jcc(ConditionFlag.AboveEqual, bb2); - - masm.cmpl(gpr1, 16); - masm.jcc(ConditionFlag.Below, bb3); - - masm.bind(bb4); - masm.addsd(dest, dest); - masm.jmp(bb8); - - masm.bind(bb5); - masm.jcc(ConditionFlag.Above, bb4); - - masm.cmpl(gpr3, 0); - masm.jcc(ConditionFlag.Above, bb4); - - masm.jmp(bb6); - - masm.bind(bb3); - masm.xorpd(temp1, temp1); - masm.addsd(temp1, dest); - masm.movdl(gpr3, temp1); - masm.psrlq(temp1, 32); - masm.movdl(gpr2, temp1); - masm.orl(gpr3, gpr2); - masm.cmpl(gpr3, 0); - masm.jcc(ConditionFlag.Equal, bb7); - - masm.xorpd(temp1, temp1); - masm.xorpd(temp2, temp2); - masm.movl(gpr1, 18416); - masm.pinsrw(temp1, gpr1, 3); - masm.mulsd(dest, temp1); - masm.movl(gpr1, 16368); - masm.pinsrw(temp2, gpr1, 3); - masm.movdqu(temp1, dest); - masm.pextrw(gpr1, dest, 3); - masm.por(dest, temp2); - masm.movl(gpr2, 18416); - masm.psrlq(dest, 27); - masm.movdqu(temp2, externalAddress(logTenEPtr)); // 0x00000000, - // 0x3fdbc000, - // 0xbf2e4108, - // 0x3f5a7a6c - masm.psrld(dest, 2); - masm.rcpps(dest, dest); - masm.psllq(temp1, 12); - masm.pshufd(temp6, temp5, 0x4E); - masm.psrlq(temp1, 12); - masm.jmp(bb1); - - masm.bind(bb2); - masm.movdl(gpr3, temp1); - masm.psrlq(temp1, 32); - masm.movdl(gpr2, temp1); - masm.addl(gpr2, gpr2); - masm.cmpl(gpr2, -2097152); - masm.jcc(ConditionFlag.AboveEqual, bb5); - - masm.orl(gpr3, gpr2); - masm.cmpl(gpr3, 0); - masm.jcc(ConditionFlag.Equal, bb7); - - masm.bind(bb6); - masm.xorpd(temp1, temp1); - masm.xorpd(dest, dest); - masm.movl(gpr1, 32752); - masm.pinsrw(temp1, gpr1, 3); - masm.mulsd(dest, temp1); - masm.jmp(bb8); - - masm.bind(bb7); - masm.xorpd(temp1, temp1); - masm.xorpd(dest, dest); - masm.movl(gpr1, 49136); - masm.pinsrw(dest, gpr1, 3); - masm.divsd(dest, temp1); - - masm.bind(bb8); - } - - /* - * Copyright (c) 2014, 2016, Intel Corporation. All rights reserved. Intel Math Library (LIBM) - * Source Code - * - * ALGORITHM DESCRIPTION - SIN() --------------------- - * - * 1. RANGE REDUCTION - * - * We perform an initial range reduction from X to r with - * - * X =~= N * pi/32 + r - * - * so that |r| <= pi/64 + epsilon. We restrict inputs to those where |N| <= 932560. Beyond this, - * the range reduction is insufficiently accurate. For extremely small inputs, denormalization - * can occur internally, impacting performance. This means that the main path is actually only - * taken for 2^-252 <= |X| < 90112. - * - * To avoid branches, we perform the range reduction to full accuracy each time. - * - * X - N * (P_1 + P_2 + P_3) - * - * where P_1 and P_2 are 32-bit numbers (so multiplication by N is exact) and P_3 is a 53-bit - * number. Together, these approximate pi well enough for all cases in the restricted range. - * - * The main reduction sequence is: - * - * y = 32/pi * x N = integer(y) (computed by adding and subtracting off SHIFTER) - * - * m_1 = N * P_1 m_2 = N * P_2 r_1 = x - m_1 r = r_1 - m_2 (this r can be used for most of the - * calculation) - * - * c_1 = r_1 - r m_3 = N * P_3 c_2 = c_1 - m_2 c = c_2 - m_3 - * - * 2. MAIN ALGORITHM - * - * The algorithm uses a table lookup based on B = M * pi / 32 where M = N mod 64. The stored - * values are: sigma closest power of 2 to cos(B) C_hl 53-bit cos(B) - sigma S_hi + S_lo 2 * - * 53-bit sin(B) - * - * The computation is organized as follows: - * - * sin(B + r + c) = [sin(B) + sigma * r] + r * (cos(B) - sigma) + sin(B) * [cos(r + c) - 1] + - * cos(B) * [sin(r + c) - r] - * - * which is approximately: - * - * [S_hi + sigma * r] + C_hl * r + S_lo + S_hi * [(cos(r) - 1) - r * c] + (C_hl + sigma) * - * [(sin(r) - r) + c] - * - * and this is what is actually computed. We separate this sum into four parts: - * - * hi + med + pols + corr - * - * where - * - * hi = S_hi + sigma r med = C_hl * r pols = S_hi * (cos(r) - 1) + (C_hl + sigma) * (sin(r) - r) - * corr = S_lo + c * ((C_hl + sigma) - S_hi * r) - * - * 3. POLYNOMIAL - * - * The polynomial S_hi * (cos(r) - 1) + (C_hl + sigma) * (sin(r) - r) can be rearranged freely, - * since it is quite small, so we exploit parallelism to the fullest. - * - * psc4 = SC_4 * r_1 msc4 = psc4 * r r2 = r * r msc2 = SC_2 * r2 r4 = r2 * r2 psc3 = SC_3 + msc4 - * psc1 = SC_1 + msc2 msc3 = r4 * psc3 sincospols = psc1 + msc3 pols = sincospols * - * - * 4. CORRECTION TERM - * - * This is where the "c" component of the range reduction is taken into account; recall that - * just "r" is used for most of the calculation. - * - * -c = m_3 - c_2 -d = S_hi * r - (C_hl + sigma) corr = -c * -d + S_lo - * - * 5. COMPENSATED SUMMATIONS - * - * The two successive compensated summations add up the high and medium parts, leaving just the - * low parts to add up at the end. - * - * rs = sigma * r res_int = S_hi + rs k_0 = S_hi - res_int k_2 = k_0 + rs med = C_hl * r res_hi - * = res_int + med k_1 = res_int - res_hi k_3 = k_1 + med - * - * 6. FINAL SUMMATION - * - * We now add up all the small parts: - * - * res_lo = pols(hi) + pols(lo) + corr + k_1 + k_3 - * - * Now the overall result is just: - * - * res_hi + res_lo - * - * 7. SMALL ARGUMENTS - * - * If |x| < SNN (SNN meaning the smallest normal number), we simply perform 0.1111111 cdots 1111 - * * x. For SNN <= |x|, we do 2^-55 * (2^55 * x - x). - * - * Special cases: sin(NaN) = quiet NaN, and raise invalid exception sin(INF) = NaN and raise - * invalid exception sin(+/-0) = +/-0 - * - */ - - public int[] oneHalf = { - 0x00000000, 0x3fe00000, 0x00000000, 0x3fe00000 - }; - - public int[] pTwo = { - 0x1a600000, 0x3d90b461, 0x1a600000, 0x3d90b461 - }; - - public int[] scFour = { - 0xa556c734, 0x3ec71de3, 0x1a01a01a, 0x3efa01a0 - }; - - public int[] cTable = { - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x3ff00000, 0x176d6d31, 0xbf73b92e, - 0xbc29b42c, 0x3fb917a6, 0xe0000000, 0xbc3e2718, 0x00000000, - 0x3ff00000, 0x011469fb, 0xbf93ad06, 0x3c69a60b, 0x3fc8f8b8, - 0xc0000000, 0xbc626d19, 0x00000000, 0x3ff00000, 0x939d225a, - 0xbfa60bea, 0x2ed59f06, 0x3fd29406, 0xa0000000, 0xbc75d28d, - 0x00000000, 0x3ff00000, 0x866b95cf, 0xbfb37ca1, 0xa6aea963, - 0x3fd87de2, 0xe0000000, 0xbc672ced, 0x00000000, 0x3ff00000, - 0x73fa1279, 0xbfbe3a68, 0x3806f63b, 0x3fde2b5d, 0x20000000, - 0x3c5e0d89, 0x00000000, 0x3ff00000, 0x5bc57974, 0xbfc59267, - 0x39ae68c8, 0x3fe1c73b, 0x20000000, 0x3c8b25dd, 0x00000000, - 0x3ff00000, 0x53aba2fd, 0xbfcd0dfe, 0x25091dd6, 0x3fe44cf3, - 0x20000000, 0x3c68076a, 0x00000000, 0x3ff00000, 0x99fcef32, - 0x3fca8279, 0x667f3bcd, 0x3fe6a09e, 0x20000000, 0xbc8bdd34, - 0x00000000, 0x3fe00000, 0x94247758, 0x3fc133cc, 0x6b151741, - 0x3fe8bc80, 0x20000000, 0xbc82c5e1, 0x00000000, 0x3fe00000, - 0x9ae68c87, 0x3fac73b3, 0x290ea1a3, 0x3fea9b66, 0xe0000000, - 0x3c39f630, 0x00000000, 0x3fe00000, 0x7f909c4e, 0xbf9d4a2c, - 0xf180bdb1, 0x3fec38b2, 0x80000000, 0xbc76e0b1, 0x00000000, - 0x3fe00000, 0x65455a75, 0xbfbe0875, 0xcf328d46, 0x3fed906b, - 0x20000000, 0x3c7457e6, 0x00000000, 0x3fe00000, 0x76acf82d, - 0x3fa4a031, 0x56c62dda, 0x3fee9f41, 0xe0000000, 0x3c8760b1, - 0x00000000, 0x3fd00000, 0x0e5967d5, 0xbfac1d1f, 0xcff75cb0, - 0x3fef6297, 0x20000000, 0x3c756217, 0x00000000, 0x3fd00000, - 0x0f592f50, 0xbf9ba165, 0xa3d12526, 0x3fefd88d, 0x40000000, - 0xbc887df6, 0x00000000, 0x3fc00000, 0x00000000, 0x00000000, - 0x00000000, 0x3ff00000, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0x0f592f50, 0x3f9ba165, 0xa3d12526, 0x3fefd88d, - 0x40000000, 0xbc887df6, 0x00000000, 0xbfc00000, 0x0e5967d5, - 0x3fac1d1f, 0xcff75cb0, 0x3fef6297, 0x20000000, 0x3c756217, - 0x00000000, 0xbfd00000, 0x76acf82d, 0xbfa4a031, 0x56c62dda, - 0x3fee9f41, 0xe0000000, 0x3c8760b1, 0x00000000, 0xbfd00000, - 0x65455a75, 0x3fbe0875, 0xcf328d46, 0x3fed906b, 0x20000000, - 0x3c7457e6, 0x00000000, 0xbfe00000, 0x7f909c4e, 0x3f9d4a2c, - 0xf180bdb1, 0x3fec38b2, 0x80000000, 0xbc76e0b1, 0x00000000, - 0xbfe00000, 0x9ae68c87, 0xbfac73b3, 0x290ea1a3, 0x3fea9b66, - 0xe0000000, 0x3c39f630, 0x00000000, 0xbfe00000, 0x94247758, - 0xbfc133cc, 0x6b151741, 0x3fe8bc80, 0x20000000, 0xbc82c5e1, - 0x00000000, 0xbfe00000, 0x99fcef32, 0xbfca8279, 0x667f3bcd, - 0x3fe6a09e, 0x20000000, 0xbc8bdd34, 0x00000000, 0xbfe00000, - 0x53aba2fd, 0x3fcd0dfe, 0x25091dd6, 0x3fe44cf3, 0x20000000, - 0x3c68076a, 0x00000000, 0xbff00000, 0x5bc57974, 0x3fc59267, - 0x39ae68c8, 0x3fe1c73b, 0x20000000, 0x3c8b25dd, 0x00000000, - 0xbff00000, 0x73fa1279, 0x3fbe3a68, 0x3806f63b, 0x3fde2b5d, - 0x20000000, 0x3c5e0d89, 0x00000000, 0xbff00000, 0x866b95cf, - 0x3fb37ca1, 0xa6aea963, 0x3fd87de2, 0xe0000000, 0xbc672ced, - 0x00000000, 0xbff00000, 0x939d225a, 0x3fa60bea, 0x2ed59f06, - 0x3fd29406, 0xa0000000, 0xbc75d28d, 0x00000000, 0xbff00000, - 0x011469fb, 0x3f93ad06, 0x3c69a60b, 0x3fc8f8b8, 0xc0000000, - 0xbc626d19, 0x00000000, 0xbff00000, 0x176d6d31, 0x3f73b92e, - 0xbc29b42c, 0x3fb917a6, 0xe0000000, 0xbc3e2718, 0x00000000, - 0xbff00000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x00000000, 0xbff00000, 0x176d6d31, - 0x3f73b92e, 0xbc29b42c, 0xbfb917a6, 0xe0000000, 0x3c3e2718, - 0x00000000, 0xbff00000, 0x011469fb, 0x3f93ad06, 0x3c69a60b, - 0xbfc8f8b8, 0xc0000000, 0x3c626d19, 0x00000000, 0xbff00000, - 0x939d225a, 0x3fa60bea, 0x2ed59f06, 0xbfd29406, 0xa0000000, - 0x3c75d28d, 0x00000000, 0xbff00000, 0x866b95cf, 0x3fb37ca1, - 0xa6aea963, 0xbfd87de2, 0xe0000000, 0x3c672ced, 0x00000000, - 0xbff00000, 0x73fa1279, 0x3fbe3a68, 0x3806f63b, 0xbfde2b5d, - 0x20000000, 0xbc5e0d89, 0x00000000, 0xbff00000, 0x5bc57974, - 0x3fc59267, 0x39ae68c8, 0xbfe1c73b, 0x20000000, 0xbc8b25dd, - 0x00000000, 0xbff00000, 0x53aba2fd, 0x3fcd0dfe, 0x25091dd6, - 0xbfe44cf3, 0x20000000, 0xbc68076a, 0x00000000, 0xbff00000, - 0x99fcef32, 0xbfca8279, 0x667f3bcd, 0xbfe6a09e, 0x20000000, - 0x3c8bdd34, 0x00000000, 0xbfe00000, 0x94247758, 0xbfc133cc, - 0x6b151741, 0xbfe8bc80, 0x20000000, 0x3c82c5e1, 0x00000000, - 0xbfe00000, 0x9ae68c87, 0xbfac73b3, 0x290ea1a3, 0xbfea9b66, - 0xe0000000, 0xbc39f630, 0x00000000, 0xbfe00000, 0x7f909c4e, - 0x3f9d4a2c, 0xf180bdb1, 0xbfec38b2, 0x80000000, 0x3c76e0b1, - 0x00000000, 0xbfe00000, 0x65455a75, 0x3fbe0875, 0xcf328d46, - 0xbfed906b, 0x20000000, 0xbc7457e6, 0x00000000, 0xbfe00000, - 0x76acf82d, 0xbfa4a031, 0x56c62dda, 0xbfee9f41, 0xe0000000, - 0xbc8760b1, 0x00000000, 0xbfd00000, 0x0e5967d5, 0x3fac1d1f, - 0xcff75cb0, 0xbfef6297, 0x20000000, 0xbc756217, 0x00000000, - 0xbfd00000, 0x0f592f50, 0x3f9ba165, 0xa3d12526, 0xbfefd88d, - 0x40000000, 0x3c887df6, 0x00000000, 0xbfc00000, 0x00000000, - 0x00000000, 0x00000000, 0xbff00000, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x0f592f50, 0xbf9ba165, 0xa3d12526, - 0xbfefd88d, 0x40000000, 0x3c887df6, 0x00000000, 0x3fc00000, - 0x0e5967d5, 0xbfac1d1f, 0xcff75cb0, 0xbfef6297, 0x20000000, - 0xbc756217, 0x00000000, 0x3fd00000, 0x76acf82d, 0x3fa4a031, - 0x56c62dda, 0xbfee9f41, 0xe0000000, 0xbc8760b1, 0x00000000, - 0x3fd00000, 0x65455a75, 0xbfbe0875, 0xcf328d46, 0xbfed906b, - 0x20000000, 0xbc7457e6, 0x00000000, 0x3fe00000, 0x7f909c4e, - 0xbf9d4a2c, 0xf180bdb1, 0xbfec38b2, 0x80000000, 0x3c76e0b1, - 0x00000000, 0x3fe00000, 0x9ae68c87, 0x3fac73b3, 0x290ea1a3, - 0xbfea9b66, 0xe0000000, 0xbc39f630, 0x00000000, 0x3fe00000, - 0x94247758, 0x3fc133cc, 0x6b151741, 0xbfe8bc80, 0x20000000, - 0x3c82c5e1, 0x00000000, 0x3fe00000, 0x99fcef32, 0x3fca8279, - 0x667f3bcd, 0xbfe6a09e, 0x20000000, 0x3c8bdd34, 0x00000000, - 0x3fe00000, 0x53aba2fd, 0xbfcd0dfe, 0x25091dd6, 0xbfe44cf3, - 0x20000000, 0xbc68076a, 0x00000000, 0x3ff00000, 0x5bc57974, - 0xbfc59267, 0x39ae68c8, 0xbfe1c73b, 0x20000000, 0xbc8b25dd, - 0x00000000, 0x3ff00000, 0x73fa1279, 0xbfbe3a68, 0x3806f63b, - 0xbfde2b5d, 0x20000000, 0xbc5e0d89, 0x00000000, 0x3ff00000, - 0x866b95cf, 0xbfb37ca1, 0xa6aea963, 0xbfd87de2, 0xe0000000, - 0x3c672ced, 0x00000000, 0x3ff00000, 0x939d225a, 0xbfa60bea, - 0x2ed59f06, 0xbfd29406, 0xa0000000, 0x3c75d28d, 0x00000000, - 0x3ff00000, 0x011469fb, 0xbf93ad06, 0x3c69a60b, 0xbfc8f8b8, - 0xc0000000, 0x3c626d19, 0x00000000, 0x3ff00000, 0x176d6d31, - 0xbf73b92e, 0xbc29b42c, 0xbfb917a6, 0xe0000000, 0x3c3e2718, - 0x00000000, 0x3ff00000 - }; - - public int[] scTwo = { - 0x11111111, 0x3f811111, 0x55555555, 0x3fa55555 - }; - - public int[] scThree = { - 0x1a01a01a, 0xbf2a01a0, 0x16c16c17, 0xbf56c16c - }; - - public int[] scOne = { - 0x55555555, 0xbfc55555, 0x00000000, 0xbfe00000 - }; - - public int[] piInvTable = { - 0x00000000, 0x00000000, 0xa2f9836e, 0x4e441529, 0xfc2757d1, - 0xf534ddc0, 0xdb629599, 0x3c439041, 0xfe5163ab, 0xdebbc561, - 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c, 0xfe1deb1c, - 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41, - 0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, - 0xde05980f, 0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, - 0x4f463f66, 0x9e5fea2d, 0x7527bac7, 0xebe5f17b, 0x3d0739f7, - 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08, 0x56033046, 0xfc7b6bab, - 0xf0cfbc21 - }; - - public int[] piFour = { - 0x40000000, 0x3fe921fb, 0x18469899, 0x3e64442d - }; - - public int[] piThirtyTwoInv = { - 0x6dc9c883, 0x40245f30 - }; - - public int[] shifter = { - 0x00000000, 0x43380000 - }; - - public int[] signMask = { - 0x00000000, 0x80000000 - }; - - public int[] pThree = { - 0x2e037073, 0x3b63198a - }; - - public int[] allOnes = { - 0xffffffff, 0x3fefffff - }; - - public int[] twoPowFiftyFive = { - 0x00000000, 0x43600000 - }; - - public int[] twoPowFiftyFiveM = { - 0x00000000, 0x3c800000 - }; - - public int[] pOne = { - 0x54400000, 0x3fb921fb - }; - - public void sinIntrinsic(Register dest, Register value, CompilationResultBuilder crb, AMD64MacroAssembler masm) { - ArrayDataPointerConstant oneHalfPtr = new ArrayDataPointerConstant(oneHalf, 16); - ArrayDataPointerConstant pTwoPtr = new ArrayDataPointerConstant(pTwo, 16); - ArrayDataPointerConstant scFourPtr = new ArrayDataPointerConstant(scFour, 16); - ArrayDataPointerConstant cTablePtr = new ArrayDataPointerConstant(cTable, 16); - ArrayDataPointerConstant scTwoPtr = new ArrayDataPointerConstant(scTwo, 16); - ArrayDataPointerConstant scThreePtr = new ArrayDataPointerConstant(scThree, 16); - ArrayDataPointerConstant scOnePtr = new ArrayDataPointerConstant(scOne, 16); - ArrayDataPointerConstant piInvTablePtr = new ArrayDataPointerConstant(piInvTable, 16); - ArrayDataPointerConstant piFourPtr = new ArrayDataPointerConstant(piFour, 16); - ArrayDataPointerConstant piThirtyTwoInvPtr = new ArrayDataPointerConstant(piThirtyTwoInv, 8); - ArrayDataPointerConstant shifterPtr = new ArrayDataPointerConstant(shifter, 8); - ArrayDataPointerConstant signMaskPtr = new ArrayDataPointerConstant(signMask, 8); - ArrayDataPointerConstant pThreePtr = new ArrayDataPointerConstant(pThree, 8); - ArrayDataPointerConstant allOnesPtr = new ArrayDataPointerConstant(allOnes, 8); - ArrayDataPointerConstant twoPowFiftyFivePtr = new ArrayDataPointerConstant(twoPowFiftyFive, 8); - ArrayDataPointerConstant twoPowFiftyFiveMPtr = new ArrayDataPointerConstant(twoPowFiftyFiveM, 8); - ArrayDataPointerConstant pOnePtr = new ArrayDataPointerConstant(pOne, 8); - - Label bb0 = new Label(); - Label bb1 = new Label(); - Label bb2 = new Label(); - Label bb4 = new Label(); - Label bb5 = new Label(); - Label bb6 = new Label(); - Label bb8 = new Label(); - Label bb9 = new Label(); - Label bb10 = new Label(); - Label bb11 = new Label(); - Label bb12 = new Label(); - Label bb13 = new Label(); - Label bb14 = new Label(); - Label bb15 = new Label(); - - Register gpr1 = asRegister(gpr1Temp, AMD64Kind.QWORD); - Register gpr2 = asRegister(gpr2Temp, AMD64Kind.QWORD); - Register gpr3 = asRegister(rcxTemp, AMD64Kind.QWORD); - Register gpr4 = asRegister(gpr4Temp, AMD64Kind.QWORD); - Register gpr5 = asRegister(gpr5Temp, AMD64Kind.QWORD); - Register gpr6 = asRegister(gpr6Temp, AMD64Kind.QWORD); - Register gpr7 = asRegister(gpr7Temp, AMD64Kind.QWORD); - Register gpr8 = asRegister(gpr8Temp, AMD64Kind.QWORD); - Register gpr9 = asRegister(gpr9Temp, AMD64Kind.QWORD); - Register gpr10 = asRegister(gpr10Temp, AMD64Kind.QWORD); - - Register temp1 = asRegister(xmm1Temp, AMD64Kind.DOUBLE); - Register temp2 = asRegister(xmm2Temp, AMD64Kind.DOUBLE); - Register temp3 = asRegister(xmm3Temp, AMD64Kind.DOUBLE); - Register temp4 = asRegister(xmm4Temp, AMD64Kind.DOUBLE); - Register temp5 = asRegister(xmm5Temp, AMD64Kind.DOUBLE); - Register temp6 = asRegister(xmm6Temp, AMD64Kind.DOUBLE); - Register temp7 = asRegister(xmm7Temp, AMD64Kind.DOUBLE); - Register temp8 = asRegister(xmm8Temp, AMD64Kind.DOUBLE); - Register temp9 = asRegister(xmm9Temp, AMD64Kind.DOUBLE); - - AMD64Address stackSlot = (AMD64Address) crb.asAddress(stackTemp); - - setCrb(crb); - masm.movsd(stackSlot, value); - if (dest.encoding != value.encoding) { - masm.movdqu(dest, value); - } - - masm.leaq(gpr1, stackSlot); - masm.movl(gpr1, new AMD64Address(gpr1, 4)); - masm.movdq(temp1, externalAddress(piThirtyTwoInvPtr)); // 0x6dc9c883, - // 0x40245f30 - masm.movdq(temp2, externalAddress(shifterPtr)); // 0x00000000, - // 0x43380000 - - masm.andl(gpr1, 2147418112); - masm.subl(gpr1, 808452096); - masm.cmpl(gpr1, 281346048); - masm.jcc(ConditionFlag.Above, bb0); - - masm.mulsd(temp1, dest); - masm.movdqu(temp5, externalAddress(oneHalfPtr)); // 0x00000000, - // 0x3fe00000, - // 0x00000000, - // 0x3fe00000 - masm.movdq(temp4, externalAddress(signMaskPtr)); // 0x00000000, - // 0x80000000 - masm.pand(temp4, dest); - masm.por(temp5, temp4); - masm.addpd(temp1, temp5); - masm.cvttsd2sil(gpr4, temp1); - masm.cvtsi2sdl(temp1, gpr4); - masm.movdqu(temp6, externalAddress(pTwoPtr)); // 0x1a600000, - // 0x3d90b461, - // 0x1a600000, - // 0x3d90b461 - masm.movq(gpr7, 0x3fb921fb54400000L); - masm.movdq(temp3, gpr7); - masm.movdqu(temp5, externalAddress(scFourPtr)); // 0xa556c734, - // 0x3ec71de3, - // 0x1a01a01a, - // 0x3efa01a0 - masm.pshufd(temp4, dest, 0x44); - masm.mulsd(temp3, temp1); - if (masm.supports(CPUFeature.SSE3)) { - masm.movddup(temp1, temp1); - } else { - masm.movlhps(temp1, temp1); - } - masm.andl(gpr4, 63); - masm.shll(gpr4, 5); - masm.leaq(gpr1, externalAddress(cTablePtr)); - masm.addq(gpr1, gpr4); - masm.movdqu(temp8, new AMD64Address(gpr1, 0)); - masm.mulpd(temp6, temp1); - masm.mulsd(temp1, externalAddress(pThreePtr)); // 0x2e037073, - // 0x3b63198a - masm.subsd(temp4, temp3); - masm.subsd(dest, temp3); - if (masm.supports(CPUFeature.SSE3)) { - masm.movddup(temp3, temp4); - } else { - masm.movdqu(temp3, temp4); - masm.movlhps(temp3, temp3); - } - masm.subsd(temp4, temp6); - masm.pshufd(dest, dest, 0x44); - masm.pshufd(temp7, temp8, 0xE); - masm.movdqu(temp2, temp8); - masm.movdqu(temp9, temp7); - masm.mulpd(temp5, dest); - masm.subpd(dest, temp6); - masm.mulsd(temp7, temp4); - masm.subsd(temp3, temp4); - masm.mulpd(temp5, dest); - masm.mulpd(dest, dest); - masm.subsd(temp3, temp6); - masm.movdqu(temp6, externalAddress(scTwoPtr)); // 0x11111111, - // 0x3f811111, - // 0x55555555, - // 0x3fa55555 - masm.subsd(temp1, temp3); - masm.movdq(temp3, new AMD64Address(gpr1, 24)); - masm.addsd(temp2, temp3); - masm.subsd(temp7, temp2); - masm.mulsd(temp2, temp4); - masm.mulpd(temp6, dest); - masm.mulsd(temp3, temp4); - masm.mulpd(temp2, dest); - masm.mulpd(dest, dest); - masm.addpd(temp5, externalAddress(scThreePtr)); // 0x1a01a01a, - // 0xbf2a01a0, - // 0x16c16c17, - // 0xbf56c16c - masm.mulsd(temp4, temp8); - masm.addpd(temp6, externalAddress(scOnePtr)); // 0x55555555, - // 0xbfc55555, - // 0x00000000, - // 0xbfe00000 - masm.mulpd(temp5, dest); - masm.movdqu(dest, temp3); - masm.addsd(temp3, temp9); - masm.mulpd(temp1, temp7); - masm.movdqu(temp7, temp4); - masm.addsd(temp4, temp3); - masm.addpd(temp6, temp5); - masm.subsd(temp9, temp3); - masm.subsd(temp3, temp4); - masm.addsd(temp1, new AMD64Address(gpr1, 16)); - masm.mulpd(temp6, temp2); - masm.addsd(temp9, dest); - masm.addsd(temp3, temp7); - masm.addsd(temp1, temp9); - masm.addsd(temp1, temp3); - masm.addsd(temp1, temp6); - masm.unpckhpd(temp6, temp6); - masm.movdqu(dest, temp4); - masm.addsd(temp1, temp6); - masm.addsd(dest, temp1); - masm.jmp(bb15); - - masm.bind(bb14); - masm.xorpd(temp1, temp1); - masm.xorpd(dest, dest); - masm.divsd(dest, temp1); - masm.jmp(bb15); - - masm.bind(bb0); - masm.jcc(ConditionFlag.Greater, bb1); - - masm.shrl(gpr1, 20); - masm.cmpl(gpr1, 3325); - masm.jcc(ConditionFlag.NotEqual, bb2); - - masm.mulsd(dest, externalAddress(allOnesPtr)); // 0xffffffff, - // 0x3fefffff - masm.jmp(bb15); - - masm.bind(bb2); - masm.movdq(temp3, externalAddress(twoPowFiftyFivePtr)); // 0x00000000, - // 0x43600000 - masm.mulsd(temp3, dest); - masm.subsd(temp3, dest); - masm.mulsd(temp3, externalAddress(twoPowFiftyFiveMPtr)); // 0x00000000, - // 0x3c800000 - masm.jmp(bb15); - - masm.bind(bb1); - masm.pextrw(gpr3, dest, 3); - masm.andl(gpr3, 32752); - masm.cmpl(gpr3, 32752); - masm.jcc(ConditionFlag.Equal, bb14); - - masm.subl(gpr3, 16224); - masm.shrl(gpr3, 7); - masm.andl(gpr3, 65532); - masm.leaq(gpr10, externalAddress(piInvTablePtr)); - masm.addq(gpr3, gpr10); - masm.movdq(gpr1, dest); - masm.movl(gpr9, new AMD64Address(gpr3, 20)); - masm.movl(gpr7, new AMD64Address(gpr3, 24)); - masm.movl(gpr4, gpr1); - masm.shrq(gpr1, 21); - masm.orl(gpr1, Integer.MIN_VALUE); - masm.shrl(gpr1, 11); - masm.movl(gpr8, gpr9); - masm.imulq(gpr9, gpr4); - masm.imulq(gpr8, gpr1); - masm.imulq(gpr7, gpr1); - masm.movl(gpr5, new AMD64Address(gpr3, 16)); - masm.movl(gpr6, new AMD64Address(gpr3, 12)); - masm.movl(gpr10, gpr9); - masm.shrq(gpr9, 32); - masm.addq(gpr8, gpr9); - masm.addq(gpr10, gpr7); - masm.movl(gpr7, gpr10); - masm.shrq(gpr10, 32); - masm.addq(gpr8, gpr10); - masm.movl(gpr9, gpr5); - masm.imulq(gpr5, gpr4); - masm.imulq(gpr9, gpr1); - masm.movl(gpr10, gpr6); - masm.imulq(gpr6, gpr4); - masm.movl(gpr2, gpr5); - masm.shrq(gpr5, 32); - masm.addq(gpr8, gpr2); - masm.movl(gpr2, gpr8); - masm.shrq(gpr8, 32); - masm.addq(gpr9, gpr5); - masm.addq(gpr9, gpr8); - masm.shlq(gpr2, 32); - masm.orq(gpr7, gpr2); - masm.imulq(gpr10, gpr1); - masm.movl(gpr8, new AMD64Address(gpr3, 8)); - masm.movl(gpr5, new AMD64Address(gpr3, 4)); - masm.movl(gpr2, gpr6); - masm.shrq(gpr6, 32); - masm.addq(gpr9, gpr2); - masm.movl(gpr2, gpr9); - masm.shrq(gpr9, 32); - masm.addq(gpr10, gpr6); - masm.addq(gpr10, gpr9); - masm.movq(gpr6, gpr8); - masm.imulq(gpr8, gpr4); - masm.imulq(gpr6, gpr1); - masm.movl(gpr9, gpr8); - masm.shrq(gpr8, 32); - masm.addq(gpr10, gpr9); - masm.movl(gpr9, gpr10); - masm.shrq(gpr10, 32); - masm.addq(gpr6, gpr8); - masm.addq(gpr6, gpr10); - masm.movq(gpr8, gpr5); - masm.imulq(gpr5, gpr4); - masm.imulq(gpr8, gpr1); - masm.shlq(gpr9, 32); - masm.orq(gpr9, gpr2); - masm.movl(gpr1, new AMD64Address(gpr3, 0)); - masm.movl(gpr10, gpr5); - masm.shrq(gpr5, 32); - masm.addq(gpr6, gpr10); - masm.movl(gpr10, gpr6); - masm.shrq(gpr6, 32); - masm.addq(gpr8, gpr5); - masm.addq(gpr8, gpr6); - masm.imulq(gpr4, gpr1); - masm.pextrw(gpr2, dest, 3); - masm.leaq(gpr6, externalAddress(piInvTablePtr)); - masm.subq(gpr3, gpr6); - masm.addl(gpr3, gpr3); - masm.addl(gpr3, gpr3); - masm.addl(gpr3, gpr3); - masm.addl(gpr3, 19); - masm.movl(gpr5, 32768); - masm.andl(gpr5, gpr2); - masm.shrl(gpr2, 4); - masm.andl(gpr2, 2047); - masm.subl(gpr2, 1023); - masm.subl(gpr3, gpr2); - masm.addq(gpr8, gpr4); - masm.movl(gpr4, gpr3); - masm.addl(gpr4, 32); - masm.cmpl(gpr3, 1); - masm.jcc(ConditionFlag.Less, bb4); - - masm.negl(gpr3); - masm.addl(gpr3, 29); - masm.shll(gpr8); - masm.movl(gpr6, gpr8); - masm.andl(gpr8, 536870911); - masm.testl(gpr8, 268435456); - masm.jcc(ConditionFlag.NotEqual, bb5); - - masm.shrl(gpr8); - masm.movl(gpr2, 0); - masm.shlq(gpr8, 32); - masm.orq(gpr8, gpr10); - - masm.bind(bb6); - - masm.cmpq(gpr8, 0); - masm.jcc(ConditionFlag.Equal, bb8); - - masm.bind(bb9); - masm.bsrq(gpr10, gpr8); - masm.movl(gpr3, 29); - masm.subl(gpr3, gpr10); - masm.jcc(ConditionFlag.LessEqual, bb10); - - masm.shlq(gpr8); - masm.movq(gpr1, gpr9); - masm.shlq(gpr9); - masm.addl(gpr4, gpr3); - masm.negl(gpr3); - masm.addl(gpr3, 64); - masm.shrq(gpr1); - masm.shrq(gpr7); - masm.orq(gpr8, gpr1); - masm.orq(gpr9, gpr7); - - masm.bind(bb11); - masm.cvtsi2sdq(dest, gpr8); - masm.shrq(gpr9, 1); - masm.cvtsi2sdq(temp3, gpr9); - masm.xorpd(temp4, temp4); - masm.shll(gpr4, 4); - masm.negl(gpr4); - masm.addl(gpr4, 16368); - masm.orl(gpr4, gpr5); - masm.xorl(gpr4, gpr2); - masm.pinsrw(temp4, gpr4, 3); - masm.leaq(gpr1, externalAddress(piFourPtr)); - masm.movdqu(temp2, new AMD64Address(gpr1, 0)); // 0x40000000, - // 0x3fe921fb, - // 0x18469899, - // 0x3e64442d - masm.xorpd(temp5, temp5); - masm.subl(gpr4, 1008); - masm.pinsrw(temp5, gpr4, 3); - masm.mulsd(dest, temp4); - masm.shll(gpr5, 16); - masm.sarl(gpr5, 31); - masm.mulsd(temp3, temp5); - masm.movdqu(temp1, dest); - masm.pshufd(temp6, temp2, 0xE); - masm.mulsd(dest, temp2); - masm.shrl(gpr6, 29); - masm.addsd(temp1, temp3); - masm.mulsd(temp3, temp2); - masm.addl(gpr6, gpr5); - masm.xorl(gpr6, gpr5); - masm.mulsd(temp6, temp1); - masm.movl(gpr1, gpr6); - masm.addsd(temp6, temp3); - masm.movdqu(temp2, dest); - masm.addsd(dest, temp6); - masm.subsd(temp2, dest); - masm.addsd(temp6, temp2); - - masm.bind(bb12); - masm.movdq(temp1, externalAddress(piThirtyTwoInvPtr)); // 0x6dc9c883, - // 0x40245f30 - masm.mulsd(temp1, dest); - masm.movdq(temp5, externalAddress(oneHalfPtr)); // 0x00000000, - // 0x3fe00000, - // 0x00000000, - // 0x3fe00000 - masm.movdq(temp4, externalAddress(signMaskPtr)); // 0x00000000, - // 0x80000000 - masm.pand(temp4, dest); - masm.por(temp5, temp4); - masm.addpd(temp1, temp5); - masm.cvttsd2sil(gpr4, temp1); - masm.cvtsi2sdl(temp1, gpr4); - masm.movdq(temp3, externalAddress(pOnePtr)); // 0x54400000, - // 0x3fb921fb - masm.movdqu(temp2, externalAddress(pTwoPtr)); // 0x1a600000, - // 0x3d90b461, - // 0x1a600000, - // 0x3d90b461 - masm.mulsd(temp3, temp1); - masm.unpcklpd(temp1, temp1); - masm.shll(gpr1, 3); - masm.addl(gpr4, 1865216); - masm.movdqu(temp4, dest); - masm.addl(gpr4, gpr1); - masm.andl(gpr4, 63); - masm.movdqu(temp5, externalAddress(scFourPtr)); // 0x54400000, - // 0x3fb921fb - masm.leaq(gpr1, externalAddress(cTablePtr)); - masm.shll(gpr4, 5); - masm.addq(gpr1, gpr4); - masm.movdqu(temp8, new AMD64Address(gpr1, 0)); - masm.mulpd(temp2, temp1); - masm.subsd(dest, temp3); - masm.mulsd(temp1, externalAddress(pThreePtr)); // 0x2e037073, - // 0x3b63198a - masm.subsd(temp4, temp3); - masm.unpcklpd(dest, dest); - masm.movdqu(temp3, temp4); - masm.subsd(temp4, temp2); - masm.mulpd(temp5, dest); - masm.subpd(dest, temp2); - masm.pshufd(temp7, temp8, 0xE); - masm.movdqu(temp9, temp7); - masm.mulsd(temp7, temp4); - masm.subsd(temp3, temp4); - masm.mulpd(temp5, dest); - masm.mulpd(dest, dest); - masm.subsd(temp3, temp2); - masm.movdqu(temp2, temp8); - masm.subsd(temp1, temp3); - masm.movdq(temp3, new AMD64Address(gpr1, 24)); - masm.addsd(temp2, temp3); - masm.subsd(temp7, temp2); - masm.subsd(temp1, temp6); - masm.movdqu(temp6, externalAddress(scTwoPtr)); // 0x11111111, - // 0x3f811111, - // 0x55555555, - // 0x3fa55555 - masm.mulsd(temp2, temp4); - masm.mulpd(temp6, dest); - masm.mulsd(temp3, temp4); - masm.mulpd(temp2, dest); - masm.mulpd(dest, dest); - masm.addpd(temp5, externalAddress(scThreePtr)); // 0x1a01a01a, - // 0xbf2a01a0, - // 0x16c16c17, - // 0xbf56c16c - masm.mulsd(temp4, temp8); - masm.addpd(temp6, externalAddress(scOnePtr)); // 0x55555555, - // 0xbfc55555, - // 0x00000000, - // 0xbfe00000 - masm.mulpd(temp5, dest); - masm.movdqu(dest, temp3); - masm.addsd(temp3, temp9); - masm.mulpd(temp1, temp7); - masm.movdqu(temp7, temp4); - masm.addsd(temp4, temp3); - masm.addpd(temp6, temp5); - masm.subsd(temp9, temp3); - masm.subsd(temp3, temp4); - masm.addsd(temp1, new AMD64Address(gpr1, 16)); - masm.mulpd(temp6, temp2); - masm.addsd(temp9, dest); - masm.addsd(temp3, temp7); - masm.addsd(temp1, temp9); - masm.addsd(temp1, temp3); - masm.addsd(temp1, temp6); - masm.unpckhpd(temp6, temp6); - masm.movdqu(dest, temp4); - masm.addsd(temp1, temp6); - masm.addsd(dest, temp1); - masm.jmp(bb15); - - masm.bind(bb8); - masm.addl(gpr4, 64); - masm.movq(gpr8, gpr9); - masm.movq(gpr9, gpr7); - masm.movl(gpr7, 0); - masm.cmpq(gpr8, 0); - masm.jcc(ConditionFlag.NotEqual, bb9); - - masm.addl(gpr4, 64); - masm.movq(gpr8, gpr9); - masm.movq(gpr9, gpr7); - masm.cmpq(gpr8, 0); - masm.jcc(ConditionFlag.NotEqual, bb9); - - masm.xorpd(dest, dest); - masm.xorpd(temp6, temp6); - masm.jmp(bb12); - - masm.bind(bb10); - masm.jcc(ConditionFlag.Equal, bb11); - - masm.negl(gpr3); - masm.shrq(gpr9); - masm.movq(gpr1, gpr8); - masm.shrq(gpr8); - masm.subl(gpr4, gpr3); - masm.negl(gpr3); - masm.addl(gpr3, 64); - masm.shlq(gpr1); - masm.orq(gpr9, gpr1); - masm.jmp(bb11); - - masm.bind(bb4); - masm.negl(gpr3); - masm.shlq(gpr8, 32); - masm.orq(gpr8, gpr10); - masm.shlq(gpr8); - masm.movq(gpr6, gpr8); - masm.testl(gpr8, Integer.MIN_VALUE); - masm.jcc(ConditionFlag.NotEqual, bb13); - - masm.shrl(gpr8); - masm.movl(gpr2, 0); - masm.shrq(gpr6, 3); - masm.jmp(bb6); - - masm.bind(bb5); - masm.shrl(gpr8); - masm.movl(gpr2, 536870912); - masm.shrl(gpr2); - masm.shlq(gpr8, 32); - masm.orq(gpr8, gpr10); - masm.shlq(gpr2, 32); - masm.addl(gpr6, 536870912); - masm.movl(gpr3, 0); - masm.movl(gpr10, 0); - masm.subq(gpr3, gpr7); - masm.sbbq(gpr10, gpr9); - masm.sbbq(gpr2, gpr8); - masm.movq(gpr7, gpr3); - masm.movq(gpr9, gpr10); - masm.movq(gpr8, gpr2); - masm.movl(gpr2, 32768); - masm.jmp(bb6); - - masm.bind(bb13); - masm.shrl(gpr8); - masm.movq(gpr2, 0x100000000L); - masm.shrq(gpr2); - masm.movl(gpr3, 0); - masm.movl(gpr10, 0); - masm.subq(gpr3, gpr7); - masm.sbbq(gpr10, gpr9); - masm.sbbq(gpr2, gpr8); - masm.movq(gpr7, gpr3); - masm.movq(gpr9, gpr10); - masm.movq(gpr8, gpr2); - masm.movl(gpr2, 32768); - masm.shrq(gpr6, 3); - masm.addl(gpr6, 536870912); - masm.jmp(bb6); - - masm.bind(bb15); - } - - /* - * Copyright (c) 2014, 2016, Intel Corporation. All rights reserved. Intel Math Library (LIBM) - * Source Code - * - * ALGORITHM DESCRIPTION - COS() --------------------- - * - * 1. RANGE REDUCTION - * - * We perform an initial range reduction from X to r with - * - * X =~= N * pi/32 + r - * - * so that |r| <= pi/64 + epsilon. We restrict inputs to those where |N| <= 932560. Beyond this, - * the range reduction is insufficiently accurate. For extremely small inputs, denormalization - * can occur internally, impacting performance. This means that the main path is actually only - * taken for 2^-252 <= |X| < 90112. - * - * To avoid branches, we perform the range reduction to full accuracy each time. - * - * X - N * (P_1 + P_2 + P_3) - * - * where P_1 and P_2 are 32-bit numbers (so multiplication by N is exact) and P_3 is a 53-bit - * number. Together, these approximate pi well enough for all cases in the restricted range. - * - * The main reduction sequence is: - * - * y = 32/pi * x N = integer(y) (computed by adding and subtracting off SHIFTER) - * - * m_1 = N * P_1 m_2 = N * P_2 r_1 = x - m_1 r = r_1 - m_2 (this r can be used for most of the - * calculation) - * - * c_1 = r_1 - r m_3 = N * P_3 c_2 = c_1 - m_2 c = c_2 - m_3 - * - * 2. MAIN ALGORITHM - * - * The algorithm uses a table lookup based on B = M * pi / 32 where M = N mod 64. The stored - * values are: sigma closest power of 2 to cos(B) C_hl 53-bit cos(B) - sigma S_hi + S_lo 2 * - * 53-bit sin(B) - * - * The computation is organized as follows: - * - * sin(B + r + c) = [sin(B) + sigma * r] + r * (cos(B) - sigma) + sin(B) * [cos(r + c) - 1] + - * cos(B) * [sin(r + c) - r] - * - * which is approximately: - * - * [S_hi + sigma * r] + C_hl * r + S_lo + S_hi * [(cos(r) - 1) - r * c] + (C_hl + sigma) * - * [(sin(r) - r) + c] - * - * and this is what is actually computed. We separate this sum into four parts: - * - * hi + med + pols + corr - * - * where - * - * hi = S_hi + sigma r med = C_hl * r pols = S_hi * (cos(r) - 1) + (C_hl + sigma) * (sin(r) - r) - * corr = S_lo + c * ((C_hl + sigma) - S_hi * r) - * - * 3. POLYNOMIAL - * - * The polynomial S_hi * (cos(r) - 1) + (C_hl + sigma) * (sin(r) - r) can be rearranged freely, - * since it is quite small, so we exploit parallelism to the fullest. - * - * psc4 = SC_4 * r_1 msc4 = psc4 * r r2 = r * r msc2 = SC_2 * r2 r4 = r2 * r2 psc3 = SC_3 + msc4 - * psc1 = SC_1 + msc2 msc3 = r4 * psc3 sincospols = psc1 + msc3 pols = sincospols * - * - * 4. CORRECTION TERM - * - * This is where the "c" component of the range reduction is taken into account; recall that - * just "r" is used for most of the calculation. - * - * -c = m_3 - c_2 -d = S_hi * r - (C_hl + sigma) corr = -c * -d + S_lo - * - * 5. COMPENSATED SUMMATIONS - * - * The two successive compensated summations add up the high and medium parts, leaving just the - * low parts to add up at the end. - * - * rs = sigma * r res_int = S_hi + rs k_0 = S_hi - res_int k_2 = k_0 + rs med = C_hl * r res_hi - * = res_int + med k_1 = res_int - res_hi k_3 = k_1 + med - * - * 6. FINAL SUMMATION - * - * We now add up all the small parts: - * - * res_lo = pols(hi) + pols(lo) + corr + k_1 + k_3 - * - * Now the overall result is just: - * - * res_hi + res_lo - * - * 7. SMALL ARGUMENTS - * - * Inputs with |X| < 2^-252 are treated specially as 1 - |x|. - * - * Special cases: cos(NaN) = quiet NaN, and raise invalid exception cos(INF) = NaN and raise - * invalid exception cos(0) = 1 - * - */ - - public int[] one = { - 0x00000000, 0x3ff00000 - }; - - public void cosIntrinsic(Register dest, Register value, CompilationResultBuilder crb, AMD64MacroAssembler masm) { - ArrayDataPointerConstant oneHalfPtr = new ArrayDataPointerConstant(oneHalf, 16); - ArrayDataPointerConstant pTwoPtr = new ArrayDataPointerConstant(pTwo, 16); - ArrayDataPointerConstant scFourPtr = new ArrayDataPointerConstant(scFour, 16); - ArrayDataPointerConstant cTablePtr = new ArrayDataPointerConstant(cTable, 16); - ArrayDataPointerConstant scTwoPtr = new ArrayDataPointerConstant(scTwo, 16); - ArrayDataPointerConstant scThreePtr = new ArrayDataPointerConstant(scThree, 16); - ArrayDataPointerConstant scOnePtr = new ArrayDataPointerConstant(scOne, 16); - ArrayDataPointerConstant piInvTablePtr = new ArrayDataPointerConstant(piInvTable, 16); - ArrayDataPointerConstant piFourPtr = new ArrayDataPointerConstant(piFour, 16); - ArrayDataPointerConstant piThirtyTwoInvPtr = new ArrayDataPointerConstant(piThirtyTwoInv, 8); - ArrayDataPointerConstant signMaskPtr = new ArrayDataPointerConstant(signMask, 8); - ArrayDataPointerConstant pThreePtr = new ArrayDataPointerConstant(pThree, 8); - ArrayDataPointerConstant pOnePtr = new ArrayDataPointerConstant(pOne, 8); - ArrayDataPointerConstant onePtr = new ArrayDataPointerConstant(one, 8); - - Label bb0 = new Label(); - Label bb1 = new Label(); - Label bb3 = new Label(); - Label bb4 = new Label(); - Label bb5 = new Label(); - Label bb6 = new Label(); - Label bb7 = new Label(); - Label bb8 = new Label(); - Label bb9 = new Label(); - Label bb10 = new Label(); - Label bb11 = new Label(); - Label bb12 = new Label(); - Label bb13 = new Label(); - Label bb14 = new Label(); - - Register gpr1 = asRegister(gpr1Temp, AMD64Kind.QWORD); - Register gpr2 = asRegister(gpr2Temp, AMD64Kind.QWORD); - Register gpr3 = asRegister(rcxTemp, AMD64Kind.QWORD); - Register gpr4 = asRegister(gpr4Temp, AMD64Kind.QWORD); - Register gpr5 = asRegister(gpr5Temp, AMD64Kind.QWORD); - Register gpr6 = asRegister(gpr6Temp, AMD64Kind.QWORD); - Register gpr7 = asRegister(gpr7Temp, AMD64Kind.QWORD); - Register gpr8 = asRegister(gpr8Temp, AMD64Kind.QWORD); - Register gpr9 = asRegister(gpr9Temp, AMD64Kind.QWORD); - Register gpr10 = asRegister(gpr10Temp, AMD64Kind.QWORD); - - Register temp1 = asRegister(xmm1Temp, AMD64Kind.DOUBLE); - Register temp2 = asRegister(xmm2Temp, AMD64Kind.DOUBLE); - Register temp3 = asRegister(xmm3Temp, AMD64Kind.DOUBLE); - Register temp4 = asRegister(xmm4Temp, AMD64Kind.DOUBLE); - Register temp5 = asRegister(xmm5Temp, AMD64Kind.DOUBLE); - Register temp6 = asRegister(xmm6Temp, AMD64Kind.DOUBLE); - Register temp7 = asRegister(xmm7Temp, AMD64Kind.DOUBLE); - Register temp8 = asRegister(xmm8Temp, AMD64Kind.DOUBLE); - Register temp9 = asRegister(xmm9Temp, AMD64Kind.DOUBLE); - - AMD64Address stackSlot = (AMD64Address) crb.asAddress(stackTemp); - - setCrb(crb); - masm.movdq(stackSlot, value); - if (dest.encoding != value.encoding) { - masm.movdqu(dest, value); - } - - masm.leaq(gpr1, stackSlot); - masm.movl(gpr1, new AMD64Address(gpr1, 4)); - masm.movdq(temp1, externalAddress(piThirtyTwoInvPtr)); // 0x6dc9c883, - // 0x40245f30 - - masm.andl(gpr1, 2147418112); - masm.subl(gpr1, 808452096); - masm.cmpl(gpr1, 281346048); - masm.jcc(ConditionFlag.Above, bb0); - - masm.mulsd(temp1, dest); - masm.movdqu(temp5, externalAddress(oneHalfPtr)); // 0x00000000, - // 0x3fe00000, - // 0x00000000, - // 0x3fe00000 - masm.movdq(temp4, externalAddress(signMaskPtr)); // 0x00000000, - // 0x80000000 - masm.pand(temp4, dest); - masm.por(temp5, temp4); - masm.addpd(temp1, temp5); - masm.cvttsd2sil(gpr4, temp1); - masm.cvtsi2sdl(temp1, gpr4); - masm.movdqu(temp2, externalAddress(pTwoPtr)); // 0x1a600000, - // 0x3d90b461, - // 0x1a600000, - // 0x3d90b461 - masm.movdq(temp3, externalAddress(pOnePtr)); // 0x54400000, - // 0x3fb921fb - masm.mulsd(temp3, temp1); - masm.unpcklpd(temp1, temp1); - masm.addq(gpr4, 1865232); - masm.movdqu(temp4, dest); - masm.andq(gpr4, 63); - masm.movdqu(temp5, externalAddress(scFourPtr)); // 0xa556c734, - // 0x3ec71de3, - // 0x1a01a01a, - // 0x3efa01a0 - masm.leaq(gpr1, externalAddress(cTablePtr)); - masm.shlq(gpr4, 5); - masm.addq(gpr1, gpr4); - masm.movdqu(temp8, new AMD64Address(gpr1, 0)); - masm.mulpd(temp2, temp1); - masm.subsd(dest, temp3); - masm.mulsd(temp1, externalAddress(pThreePtr)); // 0x2e037073, - // 0x3b63198a - masm.subsd(temp4, temp3); - masm.unpcklpd(dest, dest); - masm.movdqu(temp3, temp4); - masm.subsd(temp4, temp2); - masm.mulpd(temp5, dest); - masm.subpd(dest, temp2); - masm.pshufd(temp7, temp8, 0xE); - masm.movdqu(temp6, externalAddress(scTwoPtr)); // 0x11111111, - // 0x3f811111, - // 0x55555555, - // 0x3fa55555 - masm.mulsd(temp7, temp4); - masm.subsd(temp3, temp4); - masm.mulpd(temp5, dest); - masm.mulpd(dest, dest); - masm.subsd(temp3, temp2); - masm.movdqu(temp2, temp8); - masm.subsd(temp1, temp3); - masm.movdq(temp3, new AMD64Address(gpr1, 24)); - masm.addsd(temp2, temp3); - masm.subsd(temp7, temp2); - masm.mulsd(temp2, temp4); - masm.mulpd(temp6, dest); - masm.mulsd(temp3, temp4); - masm.mulpd(temp2, dest); - masm.mulpd(dest, dest); - masm.addpd(temp5, externalAddress(scThreePtr)); // 0x1a01a01a, - // 0xbf2a01a0, - // 0x16c16c17, - // 0xbf56c16c - masm.mulsd(temp4, temp8); - masm.pshufd(temp9, temp8, 0xE); - masm.addpd(temp6, externalAddress(scOnePtr)); // 0x55555555, - // 0xbfc55555, - // 0x00000000, - // 0xbfe00000 - masm.mulpd(temp5, dest); - masm.movdqu(dest, temp3); - masm.addsd(temp3, temp9); - masm.mulpd(temp1, temp7); - masm.movdqu(temp7, temp4); - masm.addsd(temp4, temp3); - masm.addpd(temp6, temp5); - masm.subsd(temp9, temp3); - masm.subsd(temp3, temp4); - masm.addsd(temp1, new AMD64Address(gpr1, 16)); - masm.mulpd(temp6, temp2); - masm.addsd(dest, temp9); - masm.addsd(temp3, temp7); - masm.addsd(dest, temp1); - masm.addsd(dest, temp3); - masm.addsd(dest, temp6); - masm.unpckhpd(temp6, temp6); - masm.addsd(dest, temp6); - masm.addsd(dest, temp4); - masm.jmp(bb13); - - masm.bind(bb14); - masm.xorpd(temp1, temp1); - masm.xorpd(dest, dest); - masm.divsd(dest, temp1); - masm.jmp(bb13); - - masm.bind(bb0); - masm.jcc(ConditionFlag.Greater, bb1); - - masm.pextrw(gpr1, dest, 3); - masm.andl(gpr1, 32767); - masm.pinsrw(dest, gpr1, 3); - masm.movdq(temp1, externalAddress(onePtr)); // 0x00000000, - // 0x3ff00000 - masm.subsd(temp1, dest); - masm.movdqu(dest, temp1); - masm.jmp(bb13); - - masm.bind(bb1); - masm.pextrw(gpr3, dest, 3); - masm.andl(gpr3, 32752); - masm.cmpl(gpr3, 32752); - masm.jcc(ConditionFlag.Equal, bb14); - - masm.subl(gpr3, 16224); - masm.shrl(gpr3, 7); - masm.andl(gpr3, 65532); - masm.leaq(gpr10, externalAddress(piInvTablePtr)); - masm.addq(gpr3, gpr10); - masm.movdq(gpr1, dest); - masm.movl(gpr9, new AMD64Address(gpr3, 20)); - masm.movl(gpr7, new AMD64Address(gpr3, 24)); - masm.movl(gpr4, gpr1); - masm.shrq(gpr1, 21); - masm.orl(gpr1, Integer.MIN_VALUE); - masm.shrl(gpr1, 11); - masm.movl(gpr8, gpr9); - masm.imulq(gpr9, gpr4); - masm.imulq(gpr8, gpr1); - masm.imulq(gpr7, gpr1); - masm.movl(gpr5, new AMD64Address(gpr3, 16)); - masm.movl(gpr6, new AMD64Address(gpr3, 12)); - masm.movl(gpr10, gpr9); - masm.shrq(gpr9, 32); - masm.addq(gpr8, gpr9); - masm.addq(gpr10, gpr7); - masm.movl(gpr7, gpr10); - masm.shrq(gpr10, 32); - masm.addq(gpr8, gpr10); - masm.movl(gpr9, gpr5); - masm.imulq(gpr5, gpr4); - masm.imulq(gpr9, gpr1); - masm.movl(gpr10, gpr6); - masm.imulq(gpr6, gpr4); - masm.movl(gpr2, gpr5); - masm.shrq(gpr5, 32); - masm.addq(gpr8, gpr2); - masm.movl(gpr2, gpr8); - masm.shrq(gpr8, 32); - masm.addq(gpr9, gpr5); - masm.addq(gpr9, gpr8); - masm.shlq(gpr2, 32); - masm.orq(gpr7, gpr2); - masm.imulq(gpr10, gpr1); - masm.movl(gpr8, new AMD64Address(gpr3, 8)); - masm.movl(gpr5, new AMD64Address(gpr3, 4)); - masm.movl(gpr2, gpr6); - masm.shrq(gpr6, 32); - masm.addq(gpr9, gpr2); - masm.movl(gpr2, gpr9); - masm.shrq(gpr9, 32); - masm.addq(gpr10, gpr6); - masm.addq(gpr10, gpr9); - masm.movq(gpr6, gpr8); - masm.imulq(gpr8, gpr4); - masm.imulq(gpr6, gpr1); - masm.movl(gpr9, gpr8); - masm.shrq(gpr8, 32); - masm.addq(gpr10, gpr9); - masm.movl(gpr9, gpr10); - masm.shrq(gpr10, 32); - masm.addq(gpr6, gpr8); - masm.addq(gpr6, gpr10); - masm.movq(gpr8, gpr5); - masm.imulq(gpr5, gpr4); - masm.imulq(gpr8, gpr1); - masm.shlq(gpr9, 32); - masm.orq(gpr9, gpr2); - masm.movl(gpr1, new AMD64Address(gpr3, 0)); - masm.movl(gpr10, gpr5); - masm.shrq(gpr5, 32); - masm.addq(gpr6, gpr10); - masm.movl(gpr10, gpr6); - masm.shrq(gpr6, 32); - masm.addq(gpr8, gpr5); - masm.addq(gpr8, gpr6); - masm.imulq(gpr4, gpr1); - masm.pextrw(gpr2, dest, 3); - masm.leaq(gpr6, externalAddress(piInvTablePtr)); - masm.subq(gpr3, gpr6); - masm.addl(gpr3, gpr3); - masm.addl(gpr3, gpr3); - masm.addl(gpr3, gpr3); - masm.addl(gpr3, 19); - masm.movl(gpr5, 32768); - masm.andl(gpr5, gpr2); - masm.shrl(gpr2, 4); - masm.andl(gpr2, 2047); - masm.subl(gpr2, 1023); - masm.subl(gpr3, gpr2); - masm.addq(gpr8, gpr4); - masm.movl(gpr4, gpr3); - masm.addl(gpr4, 32); - masm.cmpl(gpr3, 1); - masm.jcc(ConditionFlag.Less, bb3); - - masm.negl(gpr3); - masm.addl(gpr3, 29); - masm.shll(gpr8); - masm.movl(gpr6, gpr8); - masm.andl(gpr8, 536870911); - masm.testl(gpr8, 268435456); - masm.jcc(ConditionFlag.NotEqual, bb4); - - masm.shrl(gpr8); - masm.movl(gpr2, 0); - masm.shlq(gpr8, 32); - masm.orq(gpr8, gpr10); - - masm.bind(bb5); - - masm.bind(bb6); - masm.cmpq(gpr8, 0); - masm.jcc(ConditionFlag.Equal, bb7); - - masm.bind(bb8); - masm.bsrq(gpr10, gpr8); - masm.movl(gpr3, 29); - masm.subl(gpr3, gpr10); - masm.jcc(ConditionFlag.LessEqual, bb9); - - masm.shlq(gpr8); - masm.movq(gpr1, gpr9); - masm.shlq(gpr9); - masm.addl(gpr4, gpr3); - masm.negl(gpr3); - masm.addl(gpr3, 64); - masm.shrq(gpr1); - masm.shrq(gpr7); - masm.orq(gpr8, gpr1); - masm.orq(gpr9, gpr7); - - masm.bind(bb10); - masm.cvtsi2sdq(dest, gpr8); - masm.shrq(gpr9, 1); - masm.cvtsi2sdq(temp3, gpr9); - masm.xorpd(temp4, temp4); - masm.shll(gpr4, 4); - masm.negl(gpr4); - masm.addl(gpr4, 16368); - masm.orl(gpr4, gpr5); - masm.xorl(gpr4, gpr2); - masm.pinsrw(temp4, gpr4, 3); - masm.leaq(gpr2, externalAddress(piFourPtr)); - masm.movdqu(temp2, new AMD64Address(gpr2, 0)); // 0x40000000, - // 0x3fe921fb, - // 0x18469899, - // 0x3e64442d - masm.xorpd(temp5, temp5); - masm.subl(gpr4, 1008); - masm.pinsrw(temp5, gpr4, 3); - masm.mulsd(dest, temp4); - masm.shll(gpr5, 16); - masm.sarl(gpr5, 31); - masm.mulsd(temp3, temp5); - masm.movdqu(temp1, dest); - masm.mulsd(dest, temp2); - masm.pshufd(temp6, temp2, 0xE); - masm.shrl(gpr6, 29); - masm.addsd(temp1, temp3); - masm.mulsd(temp3, temp2); - masm.addl(gpr6, gpr5); - masm.xorl(gpr6, gpr5); - masm.mulsd(temp6, temp1); - masm.movl(gpr1, gpr6); - masm.addsd(temp6, temp3); - masm.movdqu(temp2, dest); - masm.addsd(dest, temp6); - masm.subsd(temp2, dest); - masm.addsd(temp6, temp2); - - masm.bind(bb11); - masm.movq(temp1, externalAddress(piThirtyTwoInvPtr)); // 0x6dc9c883, - // 0x40245f30 - masm.mulsd(temp1, dest); - masm.movdq(temp5, externalAddress(oneHalfPtr)); // 0x00000000, - // 0x3fe00000, - // 0x00000000, - // 0x3fe00000 - masm.movdq(temp4, externalAddress(signMaskPtr)); // 0x00000000, - // 0x80000000 - masm.pand(temp4, dest); - masm.por(temp5, temp4); - masm.addpd(temp1, temp5); - masm.cvttsd2siq(gpr4, temp1); - masm.cvtsi2sdq(temp1, gpr4); - masm.movdq(temp3, externalAddress(pOnePtr)); // 0x54400000, - // 0x3fb921fb - masm.movdqu(temp2, externalAddress(pTwoPtr)); // 0x1a600000, - // 0x3d90b461, - // 0x1a600000, - // 0x3d90b461 - masm.mulsd(temp3, temp1); - masm.unpcklpd(temp1, temp1); - masm.shll(gpr1, 3); - masm.addl(gpr4, 1865232); - masm.movdqu(temp4, dest); - masm.addl(gpr4, gpr1); - masm.andl(gpr4, 63); - masm.movdqu(temp5, externalAddress(scFourPtr)); // 0xa556c734, - // 0x3ec71de3, - // 0x1a01a01a, - // 0x3efa01a0 - masm.leaq(gpr1, externalAddress(cTablePtr)); - masm.shll(gpr4, 5); - masm.addq(gpr1, gpr4); - masm.movdqu(temp8, new AMD64Address(gpr1, 0)); - masm.mulpd(temp2, temp1); - masm.subsd(dest, temp3); - masm.mulsd(temp1, externalAddress(pThreePtr)); // 0x2e037073, - // 0x3b63198a - masm.subsd(temp4, temp3); - masm.unpcklpd(dest, dest); - masm.movdqu(temp3, temp4); - masm.subsd(temp4, temp2); - masm.mulpd(temp5, dest); - masm.pshufd(temp7, temp8, 0xE); - masm.movdqu(temp9, temp7); - masm.subpd(dest, temp2); - masm.mulsd(temp7, temp4); - masm.subsd(temp3, temp4); - masm.mulpd(temp5, dest); - masm.mulpd(dest, dest); - masm.subsd(temp3, temp2); - masm.movdqu(temp2, temp8); - masm.subsd(temp1, temp3); - masm.movdq(temp3, new AMD64Address(gpr1, 24)); - masm.addsd(temp2, temp3); - masm.subsd(temp7, temp2); - masm.subsd(temp1, temp6); - masm.movdqu(temp6, externalAddress(scTwoPtr)); // 0x11111111, - // 0x3f811111, - // 0x55555555, - // 0x3fa55555 - masm.mulsd(temp2, temp4); - masm.mulpd(temp6, dest); - masm.mulsd(temp3, temp4); - masm.mulpd(temp2, dest); - masm.mulpd(dest, dest); - masm.addpd(temp5, externalAddress(scThreePtr)); // 0x1a01a01a, - // 0xbf2a01a0, - // 0x16c16c17, - // 0xbf56c16c - masm.mulsd(temp4, temp8); - masm.addpd(temp6, externalAddress(scOnePtr)); // 0x55555555, - // 0xbfc55555, - // 0x00000000, - // 0xbfe00000 - masm.mulpd(temp5, dest); - masm.movdqu(dest, temp3); - masm.addsd(temp3, temp9); - masm.mulpd(temp1, temp7); - masm.movdqu(temp7, temp4); - masm.addsd(temp4, temp3); - masm.addpd(temp6, temp5); - masm.subsd(temp9, temp3); - masm.subsd(temp3, temp4); - masm.addsd(temp1, new AMD64Address(gpr1, 16)); - masm.mulpd(temp6, temp2); - masm.addsd(temp9, dest); - masm.addsd(temp3, temp7); - masm.addsd(temp1, temp9); - masm.addsd(temp1, temp3); - masm.addsd(temp1, temp6); - masm.unpckhpd(temp6, temp6); - masm.movdqu(dest, temp4); - masm.addsd(temp1, temp6); - masm.addsd(dest, temp1); - masm.jmp(bb13); - - masm.bind(bb7); - masm.addl(gpr4, 64); - masm.movq(gpr8, gpr9); - masm.movq(gpr9, gpr7); - masm.movl(gpr7, 0); - masm.cmpq(gpr8, 0); - masm.jcc(ConditionFlag.NotEqual, bb8); - - masm.addl(gpr4, 64); - masm.movq(gpr8, gpr9); - masm.movq(gpr9, gpr7); - masm.cmpq(gpr8, 0); - masm.jcc(ConditionFlag.NotEqual, bb8); - - masm.xorpd(dest, dest); - masm.xorpd(temp6, temp6); - masm.jmp(bb11); - - masm.bind(bb9); - masm.jcc(ConditionFlag.Equal, bb10); - - masm.negl(gpr3); - masm.shrq(gpr9); - masm.movq(gpr1, gpr8); - masm.shrq(gpr8); - masm.subl(gpr4, gpr3); - masm.negl(gpr3); - masm.addl(gpr3, 64); - masm.shlq(gpr1); - masm.orq(gpr9, gpr1); - masm.jmp(bb10); - - masm.bind(bb3); - masm.negl(gpr3); - masm.shlq(gpr8, 32); - masm.orq(gpr8, gpr10); - masm.shlq(gpr8); - masm.movq(gpr6, gpr8); - masm.testl(gpr8, Integer.MIN_VALUE); - masm.jcc(ConditionFlag.NotEqual, bb12); - - masm.shrl(gpr8); - masm.movl(gpr2, 0); - masm.shrq(gpr6, 3); - masm.jmp(bb6); - - masm.bind(bb4); - masm.shrl(gpr8); - masm.movl(gpr2, 536870912); - masm.shrl(gpr2); - masm.shlq(gpr8, 32); - masm.orq(gpr8, gpr10); - masm.shlq(gpr2, 32); - masm.addl(gpr6, 536870912); - masm.movl(gpr3, 0); - masm.movl(gpr10, 0); - masm.subq(gpr3, gpr7); - masm.sbbq(gpr10, gpr9); - masm.sbbq(gpr2, gpr8); - masm.movq(gpr7, gpr3); - masm.movq(gpr9, gpr10); - masm.movq(gpr8, gpr2); - masm.movl(gpr2, 32768); - masm.jmp(bb5); - - masm.bind(bb12); - masm.shrl(gpr8); - masm.movq(gpr2, 0x100000000L); - masm.shrq(gpr2); - masm.movl(gpr3, 0); - masm.movl(gpr10, 0); - masm.subq(gpr3, gpr7); - masm.sbbq(gpr10, gpr9); - masm.sbbq(gpr2, gpr8); - masm.movq(gpr7, gpr3); - masm.movq(gpr9, gpr10); - masm.movq(gpr8, gpr2); - masm.movl(gpr2, 32768); - masm.shrq(gpr6, 3); - masm.addl(gpr6, 536870912); - masm.jmp(bb6); - - masm.bind(bb13); - } - - /* - * Copyright (c) 2014, 2016, Intel Corporation. All rights reserved. Intel Math Library (LIBM) - * Source Code - * - * ALGORITHM DESCRIPTION - TAN() --------------------- - * - * Polynomials coefficients and other constants. - * - * Note that in this algorithm, there is a different polynomial for each breakpoint, so there - * are 32 sets of polynomial coefficients as well as 32 instances of the other constants. - * - * The polynomial coefficients and constants are offset from the start of the main block as - * follows: - * - * 0: c8 | c0 16: c9 | c1 32: c10 | c2 48: c11 | c3 64: c12 | c4 80: c13 | c5 96: c14 | c6 112: - * c15 | c7 128: T_hi 136: T_lo 144: Sigma 152: T_hl 160: Tau 168: Mask 176: (end of block) - * - * The total table size is therefore 5632 bytes. - * - * Note that c0 and c1 are always zero. We could try storing other constants here, and just - * loading the low part of the SIMD register in these cases, after ensuring the high part is - * zero. - * - * The higher terms of the polynomial are computed in the *low* part of the SIMD register. This - * is so we can overlap the multiplication by r^8 and the unpacking of the other part. - * - * The constants are: T_hi + T_lo = accurate constant term in power series Sigma + T_hl = - * accurate coefficient of r in power series (Sigma=1 bit) Tau = multiplier for the reciprocal, - * always -1 or 0 - * - * The basic reconstruction formula using these constants is: - * - * High = tau * recip_hi + t_hi Med = (sgn * r + t_hl * r)_hi Low = (sgn * r + t_hl * r)_lo + - * tau * recip_lo + T_lo + (T_hl + sigma) * c + pol - * - * where pol = c0 + c1 * r + c2 * r^2 + ... + c15 * r^15 - * - * (c0 = c1 = 0, but using them keeps SIMD regularity) - * - * We then do a compensated sum High + Med, add the low parts together and then do the final - * sum. - * - * Here recip_hi + recip_lo is an accurate reciprocal of the remainder modulo pi/2 - * - * Special cases: tan(NaN) = quiet NaN, and raise invalid exception tan(INF) = NaN and raise - * invalid exception tan(+/-0) = +/-0 - * - */ - - private static int[] oneHalfTan = { - 0x00000000, 0x3fe00000, 0x00000000, 0x3fe00000 - }; - - private static int[] mulSixteen = { - 0x00000000, 0x40300000, 0x00000000, 0x3ff00000 - }; - - private static int[] signMaskTan = { - 0x00000000, 0x80000000, 0x00000000, 0x80000000 - }; - - private static int[] piThirtyTwoInvTan = { - 0x6dc9c883, 0x3fe45f30, 0x6dc9c883, 0x40245f30 - }; - - private static int[] pOneTan = { - 0x54444000, 0x3fb921fb, 0x54440000, 0x3fb921fb - }; - - private static int[] pTwoTan = { - 0x67674000, 0xbd32e7b9, 0x4c4c0000, 0x3d468c23 - }; - - private static int[] pThreeTan = { - 0x3707344a, 0x3aa8a2e0, 0x03707345, 0x3ae98a2e - }; - - private static int[] cTableTan = { - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x882c10fa, - 0x3f9664f4, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x55e6c23d, 0x3f8226e3, 0x55555555, - 0x3fd55555, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0x0e157de0, 0x3f6d6d3d, 0x11111111, 0x3fc11111, 0x00000000, - 0x00000000, 0x00000000, 0x00000000, 0x452b75e3, 0x3f57da36, - 0x1ba1ba1c, 0x3faba1ba, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x3ff00000, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x4e435f9b, - 0x3f953f83, 0x00000000, 0x00000000, 0x3c6e8e46, 0x3f9b74ea, - 0x00000000, 0x00000000, 0xda5b7511, 0x3f85ad63, 0xdc230b9b, - 0x3fb97558, 0x26cb3788, 0x3f881308, 0x76fc4985, 0x3fd62ac9, - 0x77bb08ba, 0x3f757c85, 0xb6247521, 0x3fb1381e, 0x5922170c, - 0x3f754e95, 0x8746482d, 0x3fc27f83, 0x11055b30, 0x3f64e391, - 0x3e666320, 0x3fa3e609, 0x0de9dae3, 0x3f6301df, 0x1f1dca06, - 0x3fafa8ae, 0x8c5b2da2, 0x3fb936bb, 0x4e88f7a5, 0x3c587d05, - 0x00000000, 0x3ff00000, 0xa8935dd9, 0x3f83dde2, 0x00000000, - 0x00000000, 0x00000000, 0x00000000, 0x5a279ea3, 0x3faa3407, - 0x00000000, 0x00000000, 0x432d65fa, 0x3fa70153, 0x00000000, - 0x00000000, 0x891a4602, 0x3f9d03ef, 0xd62ca5f8, 0x3fca77d9, - 0xb35f4628, 0x3f97a265, 0x433258fa, 0x3fd8cf51, 0xb58fd909, - 0x3f8f88e3, 0x01771cea, 0x3fc2b154, 0xf3562f8e, 0x3f888f57, - 0xc028a723, 0x3fc7370f, 0x20b7f9f0, 0x3f80f44c, 0x214368e9, - 0x3fb6dfaa, 0x28891863, 0x3f79b4b6, 0x172dbbf0, 0x3fb6cb8e, - 0xe0553158, 0x3fc975f5, 0x593fe814, 0x3c2ef5d3, 0x00000000, - 0x3ff00000, 0x03dec550, 0x3fa44203, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x9314533e, 0x3fbb8ec5, 0x00000000, - 0x00000000, 0x09aa36d0, 0x3fb6d3f4, 0x00000000, 0x00000000, - 0xdcb427fd, 0x3fb13950, 0xd87ab0bb, 0x3fd5335e, 0xce0ae8a5, - 0x3fabb382, 0x79143126, 0x3fddba41, 0x5f2b28d4, 0x3fa552f1, - 0x59f21a6d, 0x3fd015ab, 0x22c27d95, 0x3fa0e984, 0xe19fc6aa, - 0x3fd0576c, 0x8f2c2950, 0x3f9a4898, 0xc0b3f22c, 0x3fc59462, - 0x1883a4b8, 0x3f94b61c, 0x3f838640, 0x3fc30eb8, 0x355c63dc, - 0x3fd36a08, 0x1dce993d, 0xbc6d704d, 0x00000000, 0x3ff00000, - 0x2b82ab63, 0x3fb78e92, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0x56f37042, 0x3fccfc56, 0x00000000, 0x00000000, - 0xaa563951, 0x3fc90125, 0x00000000, 0x00000000, 0x3d0e7c5d, - 0x3fc50533, 0x9bed9b2e, 0x3fdf0ed9, 0x5fe7c47c, 0x3fc1f250, - 0x96c125e5, 0x3fe2edd9, 0x5a02bbd8, 0x3fbe5c71, 0x86362c20, - 0x3fda08b7, 0x4b4435ed, 0x3fb9d342, 0x4b494091, 0x3fd911bd, - 0xb56658be, 0x3fb5e4c7, 0x93a2fd76, 0x3fd3c092, 0xda271794, - 0x3fb29910, 0x3303df2b, 0x3fd189be, 0x99fcef32, 0x3fda8279, - 0xb68c1467, 0x3c708b2f, 0x00000000, 0x3ff00000, 0x980c4337, - 0x3fc5f619, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0xcc03e501, 0x3fdff10f, 0x00000000, 0x00000000, 0x44a4e845, - 0x3fddb63b, 0x00000000, 0x00000000, 0x3768ad9f, 0x3fdb72a4, - 0x3dd01cca, 0x3fe5fdb9, 0xa61d2811, 0x3fd972b2, 0x5645ad0b, - 0x3fe977f9, 0xd013b3ab, 0x3fd78ca3, 0xbf0bf914, 0x3fe4f192, - 0x4d53e730, 0x3fd5d060, 0x3f8b9000, 0x3fe49933, 0xe2b82f08, - 0x3fd4322a, 0x5936a835, 0x3fe27ae1, 0xb1c61c9b, 0x3fd2b3fb, - 0xef478605, 0x3fe1659e, 0x190834ec, 0x3fe11ab7, 0xcdb625ea, - 0xbc8e564b, 0x00000000, 0x3ff00000, 0xb07217e3, 0x3fd248f1, - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x2b2c49d0, - 0x3ff2de9c, 0x00000000, 0x00000000, 0x2655bc98, 0x3ff33e58, - 0x00000000, 0x00000000, 0xff691fa2, 0x3ff3972e, 0xe93463bd, - 0x3feeed87, 0x070e10a0, 0x3ff3f5b2, 0xf4d790a4, 0x3ff20c10, - 0xa04e8ea3, 0x3ff4541a, 0x386accd3, 0x3ff1369e, 0x222a66dd, - 0x3ff4b521, 0x22a9777e, 0x3ff20817, 0x52a04a6e, 0x3ff5178f, - 0xddaa0031, 0x3ff22137, 0x4447d47c, 0x3ff57c01, 0x1e9c7f1d, - 0x3ff29311, 0x2ab7f990, 0x3fe561b8, 0x209c7df1, 0x3c87a8c5, - 0x00000000, 0x3ff00000, 0x4170bcc6, 0x3fdc92d8, 0x00000000, - 0x00000000, 0x00000000, 0x00000000, 0xc7ab4d5a, 0x40085e24, - 0x00000000, 0x00000000, 0xe93ea75d, 0x400b963d, 0x00000000, - 0x00000000, 0x94a7f25a, 0x400f37e2, 0x4b6261cb, 0x3ff5f984, - 0x5a9dd812, 0x4011aab0, 0x74c30018, 0x3ffaf5a5, 0x7f2ce8e3, - 0x4013fe8b, 0xfe8e54fa, 0x3ffd7334, 0x670d618d, 0x4016a10c, - 0x4db97058, 0x4000e012, 0x24df44dd, 0x40199c5f, 0x697d6ece, - 0x4003006e, 0x83298b82, 0x401cfc4d, 0x19d490d6, 0x40058c19, - 0x2ae42850, 0x3fea4300, 0x118e20e6, 0xbc7a6db8, 0x00000000, - 0x40000000, 0xe33345b8, 0xbfd4e526, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x65965966, 0x40219659, 0x00000000, - 0x00000000, 0x882c10fa, 0x402664f4, 0x00000000, 0x00000000, - 0x83cd3723, 0x402c8342, 0x00000000, 0x40000000, 0x55e6c23d, - 0x403226e3, 0x55555555, 0x40055555, 0x34451939, 0x40371c96, - 0xaaaaaaab, 0x400aaaaa, 0x0e157de0, 0x403d6d3d, 0x11111111, - 0x40111111, 0xa738201f, 0x4042bbce, 0x05b05b06, 0x4015b05b, - 0x452b75e3, 0x4047da36, 0x1ba1ba1c, 0x401ba1ba, 0x00000000, - 0x3ff00000, 0x00000000, 0x00000000, 0x00000000, 0x40000000, - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0x4f48b8d3, 0xbf33eaf9, 0x00000000, 0x00000000, - 0x0cf7586f, 0x3f20b8ea, 0x00000000, 0x00000000, 0xd0258911, - 0xbf0abaf3, 0x23e49fe9, 0xbfab5a8c, 0x2d53222e, 0x3ef60d15, - 0x21169451, 0x3fa172b2, 0xbb254dbc, 0xbee1d3b5, 0xdbf93b8e, - 0xbf84c7db, 0x05b4630b, 0x3ecd3364, 0xee9aada7, 0x3f743924, - 0x794a8297, 0xbeb7b7b9, 0xe015f797, 0xbf5d41f5, 0xe41a4a56, - 0x3ea35dfb, 0xe4c2a251, 0x3f49a2ab, 0x5af9e000, 0xbfce49ce, - 0x8c743719, 0x3d1eb860, 0x00000000, 0x00000000, 0x1b4863cf, - 0x3fd78294, 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, - 0x535ad890, 0xbf2b9320, 0x00000000, 0x00000000, 0x018fdf1f, - 0x3f16d61d, 0x00000000, 0x00000000, 0x0359f1be, 0xbf0139e4, - 0xa4317c6d, 0xbfa67e17, 0x82672d0f, 0x3eebb405, 0x2f1b621e, - 0x3f9f455b, 0x51ccf238, 0xbed55317, 0xf437b9ac, 0xbf804bee, - 0xc791a2b5, 0x3ec0e993, 0x919a1db2, 0x3f7080c2, 0x336a5b0e, - 0xbeaa48a2, 0x0a268358, 0xbf55a443, 0xdfd978e4, 0x3e94b61f, - 0xd7767a58, 0x3f431806, 0x2aea0000, 0xbfc9bbe8, 0x7723ea61, - 0xbd3a2369, 0x00000000, 0x00000000, 0xdf7796ff, 0x3fd6e642, - 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, 0xb9ff07ce, - 0xbf231c78, 0x00000000, 0x00000000, 0xa5517182, 0x3f0ff0e0, - 0x00000000, 0x00000000, 0x790b4cbc, 0xbef66191, 0x848a46c6, - 0xbfa21ac0, 0xb16435fa, 0x3ee1d3ec, 0x2a1aa832, 0x3f9c71ea, - 0xfdd299ef, 0xbec9dd1a, 0x3f8dbaaf, 0xbf793363, 0x309fc6ea, - 0x3eb415d6, 0xbee60471, 0x3f6b83ba, 0x94a0a697, 0xbe9dae11, - 0x3e5c67b3, 0xbf4fd07b, 0x9a8f3e3e, 0x3e86bd75, 0xa4beb7a4, - 0x3f3d1eb1, 0x29cfc000, 0xbfc549ce, 0xbf159358, 0xbd397b33, - 0x00000000, 0x00000000, 0x871fee6c, 0x3fd666f0, 0x00000000, - 0x3ff00000, 0x00000000, 0xfffffff8, 0x7d98a556, 0xbf1a3958, - 0x00000000, 0x00000000, 0x9d88dc01, 0x3f0704c2, 0x00000000, - 0x00000000, 0x73742a2b, 0xbeed054a, 0x58844587, 0xbf9c2a13, - 0x55688a79, 0x3ed7a326, 0xee33f1d6, 0x3f9a48f4, 0xa8dc9888, - 0xbebf8939, 0xaad4b5b8, 0xbf72f746, 0x9102efa1, 0x3ea88f82, - 0xdabc29cf, 0x3f678228, 0x9289afb8, 0xbe90f456, 0x741fb4ed, - 0xbf46f3a3, 0xa97f6663, 0x3e79b4bf, 0xca89ff3f, 0x3f36db70, - 0xa8a2a000, 0xbfc0ee13, 0x3da24be1, 0xbd338b9f, 0x00000000, - 0x00000000, 0x11cd6c69, 0x3fd601fd, 0x00000000, 0x3ff00000, - 0x00000000, 0xfffffff8, 0x1a154b97, 0xbf116b01, 0x00000000, - 0x00000000, 0x2d427630, 0x3f0147bf, 0x00000000, 0x00000000, - 0xb93820c8, 0xbee264d4, 0xbb6cbb18, 0xbf94ab8c, 0x888d4d92, - 0x3ed0568b, 0x60730f7c, 0x3f98b19b, 0xe4b1fb11, 0xbeb2f950, - 0x22cf9f74, 0xbf6b21cd, 0x4a3ff0a6, 0x3e9f499e, 0xfd2b83ce, - 0x3f64aad7, 0x637b73af, 0xbe83487c, 0xe522591a, 0xbf3fc092, - 0xa158e8bc, 0x3e6e3aae, 0xe5e82ffa, 0x3f329d2f, 0xd636a000, - 0xbfb9477f, 0xc2c2d2bc, 0xbd135ef9, 0x00000000, 0x00000000, - 0xf2fdb123, 0x3fd5b566, 0x00000000, 0x3ff00000, 0x00000000, - 0xfffffff8, 0xc41acb64, 0xbf05448d, 0x00000000, 0x00000000, - 0xdbb03d6f, 0x3efb7ad2, 0x00000000, 0x00000000, 0x9e42962d, - 0xbed5aea5, 0x2579f8ef, 0xbf8b2398, 0x288a1ed9, 0x3ec81441, - 0xb0198dc5, 0x3f979a3a, 0x2fdfe253, 0xbea57cd3, 0x5766336f, - 0xbf617caa, 0x600944c3, 0x3e954ed6, 0xa4e0aaf8, 0x3f62c646, - 0x6b8fb29c, 0xbe74e3a3, 0xdc4c0409, 0xbf33f952, 0x9bffe365, - 0x3e6301ec, 0xb8869e44, 0x3f2fc566, 0xe1e04000, 0xbfb0cc62, - 0x016b907f, 0xbd119cbc, 0x00000000, 0x00000000, 0xe6b9d8fa, - 0x3fd57fb3, 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, - 0x5daf22a6, 0xbef429d7, 0x00000000, 0x00000000, 0x06bca545, - 0x3ef7a27d, 0x00000000, 0x00000000, 0x7211c19a, 0xbec41c3e, - 0x956ed53e, 0xbf7ae3f4, 0xee750e72, 0x3ec3901b, 0x91d443f5, - 0x3f96f713, 0x36661e6c, 0xbe936e09, 0x506f9381, 0xbf5122e8, - 0xcb6dd43f, 0x3e9041b9, 0x6698b2ff, 0x3f61b0c7, 0x576bf12b, - 0xbe625a8a, 0xe5a0e9dc, 0xbf23499d, 0x110384dd, 0x3e5b1c2c, - 0x68d43db6, 0x3f2cb899, 0x6ecac000, 0xbfa0c414, 0xcd7dd58c, - 0x3d13500f, 0x00000000, 0x00000000, 0x85a2c8fb, 0x3fd55fe0, - 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, 0x00000000, - 0x00000000, 0x00000000, 0x00000000, 0x2bf70ebe, 0x3ef66a8f, - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0xd644267f, 0x3ec22805, 0x16c16c17, 0x3f96c16c, - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xc4e09162, - 0x3e8d6db2, 0xbc011567, 0x3f61566a, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x1f79955c, 0x3e57da4e, 0x9334ef0b, - 0x3f2bbd77, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x55555555, 0x3fd55555, 0x00000000, - 0x3ff00000, 0x00000000, 0xfffffff8, 0x5daf22a6, 0x3ef429d7, - 0x00000000, 0x00000000, 0x06bca545, 0x3ef7a27d, 0x00000000, - 0x00000000, 0x7211c19a, 0x3ec41c3e, 0x956ed53e, 0x3f7ae3f4, - 0xee750e72, 0x3ec3901b, 0x91d443f5, 0x3f96f713, 0x36661e6c, - 0x3e936e09, 0x506f9381, 0x3f5122e8, 0xcb6dd43f, 0x3e9041b9, - 0x6698b2ff, 0x3f61b0c7, 0x576bf12b, 0x3e625a8a, 0xe5a0e9dc, - 0x3f23499d, 0x110384dd, 0x3e5b1c2c, 0x68d43db6, 0x3f2cb899, - 0x6ecac000, 0x3fa0c414, 0xcd7dd58c, 0xbd13500f, 0x00000000, - 0x00000000, 0x85a2c8fb, 0x3fd55fe0, 0x00000000, 0x3ff00000, - 0x00000000, 0xfffffff8, 0xc41acb64, 0x3f05448d, 0x00000000, - 0x00000000, 0xdbb03d6f, 0x3efb7ad2, 0x00000000, 0x00000000, - 0x9e42962d, 0x3ed5aea5, 0x2579f8ef, 0x3f8b2398, 0x288a1ed9, - 0x3ec81441, 0xb0198dc5, 0x3f979a3a, 0x2fdfe253, 0x3ea57cd3, - 0x5766336f, 0x3f617caa, 0x600944c3, 0x3e954ed6, 0xa4e0aaf8, - 0x3f62c646, 0x6b8fb29c, 0x3e74e3a3, 0xdc4c0409, 0x3f33f952, - 0x9bffe365, 0x3e6301ec, 0xb8869e44, 0x3f2fc566, 0xe1e04000, - 0x3fb0cc62, 0x016b907f, 0x3d119cbc, 0x00000000, 0x00000000, - 0xe6b9d8fa, 0x3fd57fb3, 0x00000000, 0x3ff00000, 0x00000000, - 0xfffffff8, 0x1a154b97, 0x3f116b01, 0x00000000, 0x00000000, - 0x2d427630, 0x3f0147bf, 0x00000000, 0x00000000, 0xb93820c8, - 0x3ee264d4, 0xbb6cbb18, 0x3f94ab8c, 0x888d4d92, 0x3ed0568b, - 0x60730f7c, 0x3f98b19b, 0xe4b1fb11, 0x3eb2f950, 0x22cf9f74, - 0x3f6b21cd, 0x4a3ff0a6, 0x3e9f499e, 0xfd2b83ce, 0x3f64aad7, - 0x637b73af, 0x3e83487c, 0xe522591a, 0x3f3fc092, 0xa158e8bc, - 0x3e6e3aae, 0xe5e82ffa, 0x3f329d2f, 0xd636a000, 0x3fb9477f, - 0xc2c2d2bc, 0x3d135ef9, 0x00000000, 0x00000000, 0xf2fdb123, - 0x3fd5b566, 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, - 0x7d98a556, 0x3f1a3958, 0x00000000, 0x00000000, 0x9d88dc01, - 0x3f0704c2, 0x00000000, 0x00000000, 0x73742a2b, 0x3eed054a, - 0x58844587, 0x3f9c2a13, 0x55688a79, 0x3ed7a326, 0xee33f1d6, - 0x3f9a48f4, 0xa8dc9888, 0x3ebf8939, 0xaad4b5b8, 0x3f72f746, - 0x9102efa1, 0x3ea88f82, 0xdabc29cf, 0x3f678228, 0x9289afb8, - 0x3e90f456, 0x741fb4ed, 0x3f46f3a3, 0xa97f6663, 0x3e79b4bf, - 0xca89ff3f, 0x3f36db70, 0xa8a2a000, 0x3fc0ee13, 0x3da24be1, - 0x3d338b9f, 0x00000000, 0x00000000, 0x11cd6c69, 0x3fd601fd, - 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, 0xb9ff07ce, - 0x3f231c78, 0x00000000, 0x00000000, 0xa5517182, 0x3f0ff0e0, - 0x00000000, 0x00000000, 0x790b4cbc, 0x3ef66191, 0x848a46c6, - 0x3fa21ac0, 0xb16435fa, 0x3ee1d3ec, 0x2a1aa832, 0x3f9c71ea, - 0xfdd299ef, 0x3ec9dd1a, 0x3f8dbaaf, 0x3f793363, 0x309fc6ea, - 0x3eb415d6, 0xbee60471, 0x3f6b83ba, 0x94a0a697, 0x3e9dae11, - 0x3e5c67b3, 0x3f4fd07b, 0x9a8f3e3e, 0x3e86bd75, 0xa4beb7a4, - 0x3f3d1eb1, 0x29cfc000, 0x3fc549ce, 0xbf159358, 0x3d397b33, - 0x00000000, 0x00000000, 0x871fee6c, 0x3fd666f0, 0x00000000, - 0x3ff00000, 0x00000000, 0xfffffff8, 0x535ad890, 0x3f2b9320, - 0x00000000, 0x00000000, 0x018fdf1f, 0x3f16d61d, 0x00000000, - 0x00000000, 0x0359f1be, 0x3f0139e4, 0xa4317c6d, 0x3fa67e17, - 0x82672d0f, 0x3eebb405, 0x2f1b621e, 0x3f9f455b, 0x51ccf238, - 0x3ed55317, 0xf437b9ac, 0x3f804bee, 0xc791a2b5, 0x3ec0e993, - 0x919a1db2, 0x3f7080c2, 0x336a5b0e, 0x3eaa48a2, 0x0a268358, - 0x3f55a443, 0xdfd978e4, 0x3e94b61f, 0xd7767a58, 0x3f431806, - 0x2aea0000, 0x3fc9bbe8, 0x7723ea61, 0x3d3a2369, 0x00000000, - 0x00000000, 0xdf7796ff, 0x3fd6e642, 0x00000000, 0x3ff00000, - 0x00000000, 0xfffffff8, 0x4f48b8d3, 0x3f33eaf9, 0x00000000, - 0x00000000, 0x0cf7586f, 0x3f20b8ea, 0x00000000, 0x00000000, - 0xd0258911, 0x3f0abaf3, 0x23e49fe9, 0x3fab5a8c, 0x2d53222e, - 0x3ef60d15, 0x21169451, 0x3fa172b2, 0xbb254dbc, 0x3ee1d3b5, - 0xdbf93b8e, 0x3f84c7db, 0x05b4630b, 0x3ecd3364, 0xee9aada7, - 0x3f743924, 0x794a8297, 0x3eb7b7b9, 0xe015f797, 0x3f5d41f5, - 0xe41a4a56, 0x3ea35dfb, 0xe4c2a251, 0x3f49a2ab, 0x5af9e000, - 0x3fce49ce, 0x8c743719, 0xbd1eb860, 0x00000000, 0x00000000, - 0x1b4863cf, 0x3fd78294, 0x00000000, 0x3ff00000, 0x00000000, - 0xfffffff8, 0x65965966, 0xc0219659, 0x00000000, 0x00000000, - 0x882c10fa, 0x402664f4, 0x00000000, 0x00000000, 0x83cd3723, - 0xc02c8342, 0x00000000, 0xc0000000, 0x55e6c23d, 0x403226e3, - 0x55555555, 0x40055555, 0x34451939, 0xc0371c96, 0xaaaaaaab, - 0xc00aaaaa, 0x0e157de0, 0x403d6d3d, 0x11111111, 0x40111111, - 0xa738201f, 0xc042bbce, 0x05b05b06, 0xc015b05b, 0x452b75e3, - 0x4047da36, 0x1ba1ba1c, 0x401ba1ba, 0x00000000, 0xbff00000, - 0x00000000, 0x00000000, 0x00000000, 0x40000000, 0x00000000, - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0xc7ab4d5a, 0xc0085e24, 0x00000000, 0x00000000, 0xe93ea75d, - 0x400b963d, 0x00000000, 0x00000000, 0x94a7f25a, 0xc00f37e2, - 0x4b6261cb, 0xbff5f984, 0x5a9dd812, 0x4011aab0, 0x74c30018, - 0x3ffaf5a5, 0x7f2ce8e3, 0xc013fe8b, 0xfe8e54fa, 0xbffd7334, - 0x670d618d, 0x4016a10c, 0x4db97058, 0x4000e012, 0x24df44dd, - 0xc0199c5f, 0x697d6ece, 0xc003006e, 0x83298b82, 0x401cfc4d, - 0x19d490d6, 0x40058c19, 0x2ae42850, 0xbfea4300, 0x118e20e6, - 0x3c7a6db8, 0x00000000, 0x40000000, 0xe33345b8, 0xbfd4e526, - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x2b2c49d0, - 0xbff2de9c, 0x00000000, 0x00000000, 0x2655bc98, 0x3ff33e58, - 0x00000000, 0x00000000, 0xff691fa2, 0xbff3972e, 0xe93463bd, - 0xbfeeed87, 0x070e10a0, 0x3ff3f5b2, 0xf4d790a4, 0x3ff20c10, - 0xa04e8ea3, 0xbff4541a, 0x386accd3, 0xbff1369e, 0x222a66dd, - 0x3ff4b521, 0x22a9777e, 0x3ff20817, 0x52a04a6e, 0xbff5178f, - 0xddaa0031, 0xbff22137, 0x4447d47c, 0x3ff57c01, 0x1e9c7f1d, - 0x3ff29311, 0x2ab7f990, 0xbfe561b8, 0x209c7df1, 0xbc87a8c5, - 0x00000000, 0x3ff00000, 0x4170bcc6, 0x3fdc92d8, 0x00000000, - 0x00000000, 0x00000000, 0x00000000, 0xcc03e501, 0xbfdff10f, - 0x00000000, 0x00000000, 0x44a4e845, 0x3fddb63b, 0x00000000, - 0x00000000, 0x3768ad9f, 0xbfdb72a4, 0x3dd01cca, 0xbfe5fdb9, - 0xa61d2811, 0x3fd972b2, 0x5645ad0b, 0x3fe977f9, 0xd013b3ab, - 0xbfd78ca3, 0xbf0bf914, 0xbfe4f192, 0x4d53e730, 0x3fd5d060, - 0x3f8b9000, 0x3fe49933, 0xe2b82f08, 0xbfd4322a, 0x5936a835, - 0xbfe27ae1, 0xb1c61c9b, 0x3fd2b3fb, 0xef478605, 0x3fe1659e, - 0x190834ec, 0xbfe11ab7, 0xcdb625ea, 0x3c8e564b, 0x00000000, - 0x3ff00000, 0xb07217e3, 0x3fd248f1, 0x00000000, 0x00000000, - 0x00000000, 0x00000000, 0x56f37042, 0xbfccfc56, 0x00000000, - 0x00000000, 0xaa563951, 0x3fc90125, 0x00000000, 0x00000000, - 0x3d0e7c5d, 0xbfc50533, 0x9bed9b2e, 0xbfdf0ed9, 0x5fe7c47c, - 0x3fc1f250, 0x96c125e5, 0x3fe2edd9, 0x5a02bbd8, 0xbfbe5c71, - 0x86362c20, 0xbfda08b7, 0x4b4435ed, 0x3fb9d342, 0x4b494091, - 0x3fd911bd, 0xb56658be, 0xbfb5e4c7, 0x93a2fd76, 0xbfd3c092, - 0xda271794, 0x3fb29910, 0x3303df2b, 0x3fd189be, 0x99fcef32, - 0xbfda8279, 0xb68c1467, 0xbc708b2f, 0x00000000, 0x3ff00000, - 0x980c4337, 0x3fc5f619, 0x00000000, 0x00000000, 0x00000000, - 0x00000000, 0x9314533e, 0xbfbb8ec5, 0x00000000, 0x00000000, - 0x09aa36d0, 0x3fb6d3f4, 0x00000000, 0x00000000, 0xdcb427fd, - 0xbfb13950, 0xd87ab0bb, 0xbfd5335e, 0xce0ae8a5, 0x3fabb382, - 0x79143126, 0x3fddba41, 0x5f2b28d4, 0xbfa552f1, 0x59f21a6d, - 0xbfd015ab, 0x22c27d95, 0x3fa0e984, 0xe19fc6aa, 0x3fd0576c, - 0x8f2c2950, 0xbf9a4898, 0xc0b3f22c, 0xbfc59462, 0x1883a4b8, - 0x3f94b61c, 0x3f838640, 0x3fc30eb8, 0x355c63dc, 0xbfd36a08, - 0x1dce993d, 0x3c6d704d, 0x00000000, 0x3ff00000, 0x2b82ab63, - 0x3fb78e92, 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0x5a279ea3, 0xbfaa3407, 0x00000000, 0x00000000, 0x432d65fa, - 0x3fa70153, 0x00000000, 0x00000000, 0x891a4602, 0xbf9d03ef, - 0xd62ca5f8, 0xbfca77d9, 0xb35f4628, 0x3f97a265, 0x433258fa, - 0x3fd8cf51, 0xb58fd909, 0xbf8f88e3, 0x01771cea, 0xbfc2b154, - 0xf3562f8e, 0x3f888f57, 0xc028a723, 0x3fc7370f, 0x20b7f9f0, - 0xbf80f44c, 0x214368e9, 0xbfb6dfaa, 0x28891863, 0x3f79b4b6, - 0x172dbbf0, 0x3fb6cb8e, 0xe0553158, 0xbfc975f5, 0x593fe814, - 0xbc2ef5d3, 0x00000000, 0x3ff00000, 0x03dec550, 0x3fa44203, - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x4e435f9b, - 0xbf953f83, 0x00000000, 0x00000000, 0x3c6e8e46, 0x3f9b74ea, - 0x00000000, 0x00000000, 0xda5b7511, 0xbf85ad63, 0xdc230b9b, - 0xbfb97558, 0x26cb3788, 0x3f881308, 0x76fc4985, 0x3fd62ac9, - 0x77bb08ba, 0xbf757c85, 0xb6247521, 0xbfb1381e, 0x5922170c, - 0x3f754e95, 0x8746482d, 0x3fc27f83, 0x11055b30, 0xbf64e391, - 0x3e666320, 0xbfa3e609, 0x0de9dae3, 0x3f6301df, 0x1f1dca06, - 0x3fafa8ae, 0x8c5b2da2, 0xbfb936bb, 0x4e88f7a5, 0xbc587d05, - 0x00000000, 0x3ff00000, 0xa8935dd9, 0x3f83dde2, 0x00000000, - 0x00000000, 0x00000000, 0x00000000 - }; - - private static int[] maskThirtyFiveTan = { - 0xfffc0000, 0xffffffff, 0x00000000, 0x00000000 - }; - - private static int[] qElevenTan = { - 0xb8fe4d77, 0x3f82609a - }; - - private static int[] qNineTan = { - 0xbf847a43, 0x3f9664a0 - }; - - private static int[] qSevenTan = { - 0x52c4c8ab, 0x3faba1ba - }; - - private static int[] qFiveTan = { - 0x11092746, 0x3fc11111 - }; - - private static int[] qThreeTan = { - 0x55555612, 0x3fd55555 - }; - - private static int[] piInvTableTan = { - 0x00000000, 0x00000000, 0xa2f9836e, 0x4e441529, 0xfc2757d1, - 0xf534ddc0, 0xdb629599, 0x3c439041, 0xfe5163ab, 0xdebbc561, - 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c, 0xfe1deb1c, - 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41, - 0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, - 0xde05980f, 0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, - 0x4f463f66, 0x9e5fea2d, 0x7527bac7, 0xebe5f17b, 0x3d0739f7, - 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08, 0x56033046, 0xfc7b6bab, - 0xf0cfbc21 - }; - - private static int[] piFourTan = { - 0x00000000, 0x3fe921fb, 0x4611a626, 0x3e85110b - }; - - private static int[] qqTwoTan = { - 0x676733af, 0x3d32e7b9 - }; - - private static int[] twoPowFiftyFiveTan = { - 0x00000000, 0x43600000 - }; - - private static int[] twoPowMFiftyFiveTan = { - 0x00000000, 0x3c800000 - }; - - public void tanIntrinsic(Register dest, Register value, CompilationResultBuilder crb, AMD64MacroAssembler masm) { - ArrayDataPointerConstant oneHalfTanPtr = new ArrayDataPointerConstant(oneHalfTan, 16); - ArrayDataPointerConstant mulSixteenPtr = new ArrayDataPointerConstant(mulSixteen, 16); - ArrayDataPointerConstant signMaskTanPtr = new ArrayDataPointerConstant(signMaskTan, 16); - ArrayDataPointerConstant piThirtyTwoInvTanPtr = new ArrayDataPointerConstant(piThirtyTwoInvTan, 16); - ArrayDataPointerConstant pOneTanPtr = new ArrayDataPointerConstant(pOneTan, 16); - ArrayDataPointerConstant pTwoTanPtr = new ArrayDataPointerConstant(pTwoTan, 16); - ArrayDataPointerConstant pThreeTanPtr = new ArrayDataPointerConstant(pThreeTan, 16); - ArrayDataPointerConstant cTableTanPtr = new ArrayDataPointerConstant(cTableTan, 16); - ArrayDataPointerConstant maskThirtyFiveTanPtr = new ArrayDataPointerConstant(maskThirtyFiveTan, 16); - ArrayDataPointerConstant qElevenTanPtr = new ArrayDataPointerConstant(qElevenTan, 16); - ArrayDataPointerConstant qNineTanPtr = new ArrayDataPointerConstant(qNineTan, 16); - ArrayDataPointerConstant qSevenTanPtr = new ArrayDataPointerConstant(qSevenTan, 8); - ArrayDataPointerConstant qFiveTanPtr = new ArrayDataPointerConstant(qFiveTan, 16); - ArrayDataPointerConstant qThreeTanPtr = new ArrayDataPointerConstant(qThreeTan, 16); - ArrayDataPointerConstant piInvTableTanPtr = new ArrayDataPointerConstant(piInvTableTan, 16); - ArrayDataPointerConstant piFourTanPtr = new ArrayDataPointerConstant(piFourTan, 8); - ArrayDataPointerConstant qqTwoTanPtr = new ArrayDataPointerConstant(qqTwoTan, 8); - ArrayDataPointerConstant onePtr = new ArrayDataPointerConstant(one, 8); - ArrayDataPointerConstant twoPowFiftyFiveTanPtr = new ArrayDataPointerConstant(twoPowFiftyFiveTan, 8); - ArrayDataPointerConstant twoPowMFiftyFiveTanPtr = new ArrayDataPointerConstant(twoPowMFiftyFiveTan, 8); - - Label bb0 = new Label(); - Label bb1 = new Label(); - Label bb2 = new Label(); - Label bb3 = new Label(); - Label bb5 = new Label(); - Label bb6 = new Label(); - Label bb8 = new Label(); - Label bb9 = new Label(); - Label bb10 = new Label(); - Label bb11 = new Label(); - Label bb12 = new Label(); - Label bb13 = new Label(); - Label bb14 = new Label(); - Label bb15 = new Label(); - - Register gpr1 = asRegister(gpr1Temp, AMD64Kind.QWORD); - Register gpr2 = asRegister(gpr2Temp, AMD64Kind.QWORD); - Register gpr3 = asRegister(rcxTemp, AMD64Kind.QWORD); - Register gpr4 = asRegister(gpr4Temp, AMD64Kind.QWORD); - Register gpr5 = asRegister(gpr5Temp, AMD64Kind.QWORD); - Register gpr6 = asRegister(gpr6Temp, AMD64Kind.QWORD); - Register gpr7 = asRegister(gpr7Temp, AMD64Kind.QWORD); - Register gpr8 = asRegister(gpr8Temp, AMD64Kind.QWORD); - Register gpr9 = asRegister(gpr9Temp, AMD64Kind.QWORD); - Register gpr10 = asRegister(gpr10Temp, AMD64Kind.QWORD); - - Register temp1 = asRegister(xmm1Temp, AMD64Kind.DOUBLE); - Register temp2 = asRegister(xmm2Temp, AMD64Kind.DOUBLE); - Register temp3 = asRegister(xmm3Temp, AMD64Kind.DOUBLE); - Register temp4 = asRegister(xmm4Temp, AMD64Kind.DOUBLE); - Register temp5 = asRegister(xmm5Temp, AMD64Kind.DOUBLE); - Register temp6 = asRegister(xmm6Temp, AMD64Kind.DOUBLE); - Register temp7 = asRegister(xmm7Temp, AMD64Kind.DOUBLE); - - setCrb(crb); - if (dest.encoding != value.encoding) { - masm.movdqu(dest, value); - } - - masm.pextrw(gpr1, dest, 3); - masm.andl(gpr1, 32767); - masm.subl(gpr1, 16314); - masm.cmpl(gpr1, 270); - masm.jcc(ConditionFlag.Above, bb0); - - masm.movdqu(temp5, externalAddress(oneHalfTanPtr)); // 0x00000000, - // 0x3fe00000, - // 0x00000000, - // 0x3fe00000 - masm.movdqu(temp6, externalAddress(mulSixteenPtr)); // 0x00000000, - // 0x40300000, - // 0x00000000, - // 0x3ff00000 - masm.unpcklpd(dest, dest); - masm.movdqu(temp4, externalAddress(signMaskTanPtr)); // 0x00000000, - // 0x80000000, - // 0x00000000, - // 0x80000000 - masm.andpd(temp4, dest); - masm.movdqu(temp1, externalAddress(piThirtyTwoInvTanPtr)); // 0x6dc9c883, - // 0x3fe45f30, - // 0x6dc9c883, - // 0x40245f30 - masm.mulpd(temp1, dest); - masm.por(temp5, temp4); - masm.addpd(temp1, temp5); - masm.movdqu(temp7, temp1); - masm.unpckhpd(temp7, temp7); - masm.cvttsd2sil(gpr4, temp7); - masm.cvttpd2dq(temp1, temp1); - masm.cvtdq2pd(temp1, temp1); - masm.mulpd(temp1, temp6); - masm.movdqu(temp3, externalAddress(pOneTanPtr)); // 0x54444000, - // 0x3fb921fb, - // 0x54440000, - // 0x3fb921fb - masm.movdq(temp5, externalAddress(qqTwoTanPtr)); // 0x676733af, - // 0x3d32e7b9 - masm.addq(gpr4, 469248); - masm.movdqu(temp4, externalAddress(pTwoTanPtr)); // 0x67674000, - // 0xbd32e7b9, - // 0x4c4c0000, - // 0x3d468c23 - masm.mulpd(temp3, temp1); - masm.andq(gpr4, 31); - masm.mulsd(temp5, temp1); - masm.movq(gpr3, gpr4); - masm.mulpd(temp4, temp1); - masm.shlq(gpr3, 1); - masm.subpd(dest, temp3); - masm.mulpd(temp1, externalAddress(pThreeTanPtr)); // 0x3707344a, - // 0x3aa8a2e0, - // 0x03707345, - // 0x3ae98a2e - masm.addq(gpr4, gpr3); - masm.shlq(gpr3, 2); - masm.addq(gpr4, gpr3); - masm.addsd(temp5, dest); - masm.movdqu(temp2, dest); - masm.subpd(dest, temp4); - masm.movdq(temp6, externalAddress(onePtr)); // 0x00000000, - // 0x3ff00000 - masm.shlq(gpr4, 4); - masm.leaq(gpr1, externalAddress(cTableTanPtr)); - masm.andpd(temp5, externalAddress(maskThirtyFiveTanPtr)); // 0xfffc0000, - // 0xffffffff, - // 0x00000000, - // 0x00000000 - masm.movdqu(temp3, dest); - masm.addq(gpr1, gpr4); - masm.subpd(temp2, dest); - masm.unpckhpd(dest, dest); - masm.divsd(temp6, temp5); - masm.subpd(temp2, temp4); - masm.movdqu(temp7, new AMD64Address(gpr1, 16)); - masm.subsd(temp3, temp5); - masm.mulpd(temp7, dest); - masm.subpd(temp2, temp1); - masm.movdqu(temp1, new AMD64Address(gpr1, 48)); - masm.mulpd(temp1, dest); - masm.movdqu(temp4, new AMD64Address(gpr1, 96)); - masm.mulpd(temp4, dest); - masm.addsd(temp2, temp3); - masm.movdqu(temp3, dest); - masm.mulpd(dest, dest); - masm.addpd(temp7, new AMD64Address(gpr1, 0)); - masm.addpd(temp1, new AMD64Address(gpr1, 32)); - masm.mulpd(temp1, dest); - masm.addpd(temp4, new AMD64Address(gpr1, 80)); - masm.addpd(temp7, temp1); - masm.movdqu(temp1, new AMD64Address(gpr1, 112)); - masm.mulpd(temp1, dest); - masm.mulpd(dest, dest); - masm.addpd(temp4, temp1); - masm.movdqu(temp1, new AMD64Address(gpr1, 64)); - masm.mulpd(temp1, dest); - masm.addpd(temp7, temp1); - masm.movdqu(temp1, temp3); - masm.mulpd(temp3, dest); - masm.mulsd(dest, dest); - masm.mulpd(temp1, new AMD64Address(gpr1, 144)); - masm.mulpd(temp4, temp3); - masm.movdqu(temp3, temp1); - masm.addpd(temp7, temp4); - masm.movdqu(temp4, temp1); - masm.mulsd(dest, temp7); - masm.unpckhpd(temp7, temp7); - masm.addsd(dest, temp7); - masm.unpckhpd(temp1, temp1); - masm.addsd(temp3, temp1); - masm.subsd(temp4, temp3); - masm.addsd(temp1, temp4); - masm.movdqu(temp4, temp2); - masm.movdq(temp7, new AMD64Address(gpr1, 144)); - masm.unpckhpd(temp2, temp2); - masm.addsd(temp7, new AMD64Address(gpr1, 152)); - masm.mulsd(temp7, temp2); - masm.addsd(temp7, new AMD64Address(gpr1, 136)); - masm.addsd(temp7, temp1); - masm.addsd(dest, temp7); - masm.movdq(temp7, externalAddress(onePtr)); // 0x00000000, - // 0x3ff00000 - masm.mulsd(temp4, temp6); - masm.movdq(temp2, new AMD64Address(gpr1, 168)); - masm.andpd(temp2, temp6); - masm.mulsd(temp5, temp2); - masm.mulsd(temp6, new AMD64Address(gpr1, 160)); - masm.subsd(temp7, temp5); - masm.subsd(temp2, new AMD64Address(gpr1, 128)); - masm.subsd(temp7, temp4); - masm.mulsd(temp7, temp6); - masm.movdqu(temp4, temp3); - masm.subsd(temp3, temp2); - masm.addsd(temp2, temp3); - masm.subsd(temp4, temp2); - masm.addsd(dest, temp4); - masm.subsd(dest, temp7); - masm.addsd(dest, temp3); - masm.jmp(bb15); - - masm.bind(bb0); - masm.jcc(ConditionFlag.Greater, bb1); - - masm.pextrw(gpr1, dest, 3); - masm.movl(gpr4, gpr1); - masm.andl(gpr1, 32752); - masm.jcc(ConditionFlag.Equal, bb2); - - masm.andl(gpr4, 32767); - masm.cmpl(gpr4, 15904); - masm.jcc(ConditionFlag.Below, bb3); - - masm.movdqu(temp2, dest); - masm.movdqu(temp3, dest); - masm.movdq(temp1, externalAddress(qElevenTanPtr)); // 0xb8fe4d77, - // 0x3f82609a - masm.mulsd(temp2, dest); - masm.mulsd(temp3, temp2); - masm.mulsd(temp1, temp2); - masm.addsd(temp1, externalAddress(qNineTanPtr)); // 0xbf847a43, - // 0x3f9664a0 - masm.mulsd(temp1, temp2); - masm.addsd(temp1, externalAddress(qSevenTanPtr)); // 0x52c4c8ab, - // 0x3faba1ba - masm.mulsd(temp1, temp2); - masm.addsd(temp1, externalAddress(qFiveTanPtr)); // 0x11092746, - // 0x3fc11111 - masm.mulsd(temp1, temp2); - masm.addsd(temp1, externalAddress(qThreeTanPtr)); // 0x55555612, - // 0x3fd55555 - masm.mulsd(temp1, temp3); - masm.addsd(dest, temp1); - masm.jmp(bb15); - - masm.bind(bb3); - masm.movdq(temp3, externalAddress(twoPowFiftyFiveTanPtr)); // 0x00000000, - // 0x43600000 - masm.mulsd(temp3, dest); - masm.addsd(dest, temp3); - masm.mulsd(dest, externalAddress(twoPowMFiftyFiveTanPtr)); // 0x00000000, - // 0x3c800000 - masm.jmp(bb15); - - masm.bind(bb14); - masm.xorpd(temp1, temp1); - masm.xorpd(dest, dest); - masm.divsd(dest, temp1); - masm.jmp(bb15); - - masm.bind(bb2); - masm.movdqu(temp1, dest); - masm.mulsd(temp1, temp1); - masm.jmp(bb15); - - masm.bind(bb1); - masm.pextrw(gpr3, dest, 3); - masm.andl(gpr3, 32752); - masm.cmpl(gpr3, 32752); - masm.jcc(ConditionFlag.Equal, bb14); - - masm.subl(gpr3, 16224); - masm.shrl(gpr3, 7); - masm.andl(gpr3, 65532); - masm.leaq(gpr10, externalAddress(piInvTableTanPtr)); - masm.addq(gpr3, gpr10); - masm.movdq(gpr1, dest); - masm.movl(gpr9, new AMD64Address(gpr3, 20)); - masm.movl(gpr7, new AMD64Address(gpr3, 24)); - masm.movl(gpr4, gpr1); - masm.shrq(gpr1, 21); - masm.orl(gpr1, Integer.MIN_VALUE); - masm.shrl(gpr1, 11); - masm.movl(gpr8, gpr9); - masm.imulq(gpr9, gpr4); - masm.imulq(gpr8, gpr1); - masm.imulq(gpr7, gpr1); - masm.movl(gpr5, new AMD64Address(gpr3, 16)); - masm.movl(gpr6, new AMD64Address(gpr3, 12)); - masm.movl(gpr10, gpr9); - masm.shrq(gpr9, 32); - masm.addq(gpr8, gpr9); - masm.addq(gpr10, gpr7); - masm.movl(gpr7, gpr10); - masm.shrq(gpr10, 32); - masm.addq(gpr8, gpr10); - masm.movl(gpr9, gpr5); - masm.imulq(gpr5, gpr4); - masm.imulq(gpr9, gpr1); - masm.movl(gpr10, gpr6); - masm.imulq(gpr6, gpr4); - masm.movl(gpr2, gpr5); - masm.shrq(gpr5, 32); - masm.addq(gpr8, gpr2); - masm.movl(gpr2, gpr8); - masm.shrq(gpr8, 32); - masm.addq(gpr9, gpr5); - masm.addq(gpr9, gpr8); - masm.shlq(gpr2, 32); - masm.orq(gpr7, gpr2); - masm.imulq(gpr10, gpr1); - masm.movl(gpr8, new AMD64Address(gpr3, 8)); - masm.movl(gpr5, new AMD64Address(gpr3, 4)); - masm.movl(gpr2, gpr6); - masm.shrq(gpr6, 32); - masm.addq(gpr9, gpr2); - masm.movl(gpr2, gpr9); - masm.shrq(gpr9, 32); - masm.addq(gpr10, gpr6); - masm.addq(gpr10, gpr9); - masm.movq(gpr6, gpr8); - masm.imulq(gpr8, gpr4); - masm.imulq(gpr6, gpr1); - masm.movl(gpr9, gpr8); - masm.shrq(gpr8, 32); - masm.addq(gpr10, gpr9); - masm.movl(gpr9, gpr10); - masm.shrq(gpr10, 32); - masm.addq(gpr6, gpr8); - masm.addq(gpr6, gpr10); - masm.movq(gpr8, gpr5); - masm.imulq(gpr5, gpr4); - masm.imulq(gpr8, gpr1); - masm.shlq(gpr9, 32); - masm.orq(gpr9, gpr2); - masm.movl(gpr1, new AMD64Address(gpr3, 0)); - masm.movl(gpr10, gpr5); - masm.shrq(gpr5, 32); - masm.addq(gpr6, gpr10); - masm.movl(gpr10, gpr6); - masm.shrq(gpr6, 32); - masm.addq(gpr8, gpr5); - masm.addq(gpr8, gpr6); - masm.imulq(gpr4, gpr1); - masm.pextrw(gpr2, dest, 3); - masm.leaq(gpr6, externalAddress(piInvTableTanPtr)); - masm.subq(gpr3, gpr6); - masm.addl(gpr3, gpr3); - masm.addl(gpr3, gpr3); - masm.addl(gpr3, gpr3); - masm.addl(gpr3, 19); - masm.movl(gpr5, 32768); - masm.andl(gpr5, gpr2); - masm.shrl(gpr2, 4); - masm.andl(gpr2, 2047); - masm.subl(gpr2, 1023); - masm.subl(gpr3, gpr2); - masm.addq(gpr8, gpr4); - masm.movl(gpr4, gpr3); - masm.addl(gpr4, 32); - masm.cmpl(gpr3, 0); - masm.jcc(ConditionFlag.Less, bb5); - - masm.negl(gpr3); - masm.addl(gpr3, 29); - masm.shll(gpr8); - masm.movl(gpr6, gpr8); - masm.andl(gpr8, 1073741823); - masm.testl(gpr8, 536870912); - masm.jcc(ConditionFlag.NotEqual, bb6); - - masm.shrl(gpr8); - masm.movl(gpr2, 0); - masm.shlq(gpr8, 32); - masm.orq(gpr8, gpr10); - - masm.bind(bb8); - masm.cmpq(gpr8, 0); - masm.jcc(ConditionFlag.Equal, bb9); - - masm.bind(bb10); - masm.bsrq(gpr10, gpr8); - masm.movl(gpr3, 29); - masm.subl(gpr3, gpr10); - masm.jcc(ConditionFlag.LessEqual, bb11); - - masm.shlq(gpr8); - masm.movq(gpr1, gpr9); - masm.shlq(gpr9); - masm.addl(gpr4, gpr3); - masm.negl(gpr3); - masm.addl(gpr3, 64); - masm.shrq(gpr1); - masm.shrq(gpr7); - masm.orq(gpr8, gpr1); - masm.orq(gpr9, gpr7); - - masm.bind(bb12); - masm.cvtsi2sdq(dest, gpr8); - masm.shrq(gpr9, 1); - masm.cvtsi2sdq(temp3, gpr9); - masm.xorpd(temp4, temp4); - masm.shll(gpr4, 4); - masm.negl(gpr4); - masm.addl(gpr4, 16368); - masm.orl(gpr4, gpr5); - masm.xorl(gpr4, gpr2); - masm.pinsrw(temp4, gpr4, 3); - masm.leaq(gpr1, externalAddress(piFourTanPtr)); - masm.movdq(temp2, new AMD64Address(gpr1, 0)); // 0x00000000, - // 0x3fe921fb, - masm.movdq(temp7, new AMD64Address(gpr1, 8)); // 0x4611a626, - // 0x3e85110b - masm.xorpd(temp5, temp5); - masm.subl(gpr4, 1008); - masm.pinsrw(temp5, gpr4, 3); - masm.mulsd(dest, temp4); - masm.shll(gpr5, 16); - masm.sarl(gpr5, 31); - masm.mulsd(temp3, temp5); - masm.movdqu(temp1, dest); - masm.mulsd(dest, temp2); - masm.shrl(gpr6, 30); - masm.addsd(temp1, temp3); - masm.mulsd(temp3, temp2); - masm.addl(gpr6, gpr5); - masm.xorl(gpr6, gpr5); - masm.mulsd(temp7, temp1); - masm.movl(gpr1, gpr6); - masm.addsd(temp7, temp3); - masm.movdqu(temp2, dest); - masm.addsd(dest, temp7); - masm.subsd(temp2, dest); - masm.addsd(temp7, temp2); - masm.movdqu(temp1, externalAddress(piThirtyTwoInvTanPtr)); // 0x6dc9c883, - // 0x3fe45f30, - // 0x6dc9c883, - // 0x40245f30 - if (masm.supports(CPUFeature.SSE3)) { - masm.movddup(dest, dest); - } else { - masm.movlhps(dest, dest); - } - masm.movdqu(temp4, externalAddress(signMaskTanPtr)); // 0x00000000, - // 0x80000000, - // 0x00000000, - // 0x80000000 - masm.andpd(temp4, dest); - masm.mulpd(temp1, dest); - if (masm.supports(CPUFeature.SSE3)) { - masm.movddup(temp7, temp7); - } else { - masm.movlhps(temp7, temp7); - } - masm.movdqu(temp5, externalAddress(oneHalfTanPtr)); // 0x00000000, - // 0x3fe00000, - // 0x00000000, - // 0x3fe00000 - masm.movdqu(temp6, externalAddress(mulSixteenPtr)); // 0x00000000, - // 0x40300000, - // 0x00000000, - // 0x3ff00000 - masm.por(temp5, temp4); - masm.addpd(temp1, temp5); - masm.movdqu(temp5, temp1); - masm.unpckhpd(temp5, temp5); - masm.cvttsd2sil(gpr4, temp5); - masm.cvttpd2dq(temp1, temp1); - masm.cvtdq2pd(temp1, temp1); - masm.mulpd(temp1, temp6); - masm.movdqu(temp3, externalAddress(pOneTanPtr)); // 0x54444000, - // 0x3fb921fb, - // 0x54440000, - // 0x3fb921fb - masm.movdq(temp5, externalAddress(qqTwoTanPtr)); // 0x676733af, - // 0x3d32e7b9 - masm.shll(gpr1, 4); - masm.addl(gpr4, 469248); - masm.movdqu(temp4, externalAddress(pTwoTanPtr)); // 0x67674000, - // 0xbd32e7b9, - // 0x4c4c0000, - // 0x3d468c23 - masm.mulpd(temp3, temp1); - masm.addl(gpr4, gpr1); - masm.andl(gpr4, 31); - masm.mulsd(temp5, temp1); - masm.movl(gpr3, gpr4); - masm.mulpd(temp4, temp1); - masm.shll(gpr3, 1); - masm.subpd(dest, temp3); - masm.mulpd(temp1, externalAddress(pThreeTanPtr)); // 0x3707344a, - // 0x3aa8a2e0, - // 0x03707345, - // 0x3ae98a2e - masm.addl(gpr4, gpr3); - masm.shll(gpr3, 2); - masm.addl(gpr4, gpr3); - masm.addsd(temp5, dest); - masm.movdqu(temp2, dest); - masm.subpd(dest, temp4); - masm.movdq(temp6, externalAddress(onePtr)); // 0x00000000, - // 0x3ff00000 - masm.shll(gpr4, 4); - masm.leaq(gpr1, externalAddress(cTableTanPtr)); - masm.andpd(temp5, externalAddress(maskThirtyFiveTanPtr)); // 0xfffc0000, - // 0xffffffff, - // 0x00000000, - // 0x00000000 - masm.movdqu(temp3, dest); - masm.addq(gpr1, gpr4); - masm.subpd(temp2, dest); - masm.unpckhpd(dest, dest); - masm.divsd(temp6, temp5); - masm.subpd(temp2, temp4); - masm.subsd(temp3, temp5); - masm.subpd(temp2, temp1); - masm.movdqu(temp1, new AMD64Address(gpr1, 48)); - masm.addpd(temp2, temp7); - masm.movdqu(temp7, new AMD64Address(gpr1, 16)); - masm.mulpd(temp7, dest); - masm.movdqu(temp4, new AMD64Address(gpr1, 96)); - masm.mulpd(temp1, dest); - masm.mulpd(temp4, dest); - masm.addsd(temp2, temp3); - masm.movdqu(temp3, dest); - masm.mulpd(dest, dest); - masm.addpd(temp7, new AMD64Address(gpr1, 0)); - masm.addpd(temp1, new AMD64Address(gpr1, 32)); - masm.mulpd(temp1, dest); - masm.addpd(temp4, new AMD64Address(gpr1, 80)); - masm.addpd(temp7, temp1); - masm.movdqu(temp1, new AMD64Address(gpr1, 112)); - masm.mulpd(temp1, dest); - masm.mulpd(dest, dest); - masm.addpd(temp4, temp1); - masm.movdqu(temp1, new AMD64Address(gpr1, 64)); - masm.mulpd(temp1, dest); - masm.addpd(temp7, temp1); - masm.movdqu(temp1, temp3); - masm.mulpd(temp3, dest); - masm.mulsd(dest, dest); - masm.mulpd(temp1, new AMD64Address(gpr1, 144)); - masm.mulpd(temp4, temp3); - masm.movdqu(temp3, temp1); - masm.addpd(temp7, temp4); - masm.movdqu(temp4, temp1); - masm.mulsd(dest, temp7); - masm.unpckhpd(temp7, temp7); - masm.addsd(dest, temp7); - masm.unpckhpd(temp1, temp1); - masm.addsd(temp3, temp1); - masm.subsd(temp4, temp3); - masm.addsd(temp1, temp4); - masm.movdqu(temp4, temp2); - masm.movdq(temp7, new AMD64Address(gpr1, 144)); - masm.unpckhpd(temp2, temp2); - masm.addsd(temp7, new AMD64Address(gpr1, 152)); - masm.mulsd(temp7, temp2); - masm.addsd(temp7, new AMD64Address(gpr1, 136)); - masm.addsd(temp7, temp1); - masm.addsd(dest, temp7); - masm.movdq(temp7, externalAddress(onePtr)); // 0x00000000, - // 0x3ff00000 - masm.mulsd(temp4, temp6); - masm.movdq(temp2, new AMD64Address(gpr1, 168)); - masm.andpd(temp2, temp6); - masm.mulsd(temp5, temp2); - masm.mulsd(temp6, new AMD64Address(gpr1, 160)); - masm.subsd(temp7, temp5); - masm.subsd(temp2, new AMD64Address(gpr1, 128)); - masm.subsd(temp7, temp4); - masm.mulsd(temp7, temp6); - masm.movdqu(temp4, temp3); - masm.subsd(temp3, temp2); - masm.addsd(temp2, temp3); - masm.subsd(temp4, temp2); - masm.addsd(dest, temp4); - masm.subsd(dest, temp7); - masm.addsd(dest, temp3); - masm.jmp(bb15); - - masm.bind(bb9); - masm.addl(gpr4, 64); - masm.movq(gpr8, gpr9); - masm.movq(gpr9, gpr7); - masm.movl(gpr7, 0); - masm.cmpq(gpr8, 0); - masm.jcc(ConditionFlag.NotEqual, bb10); - - masm.addl(gpr4, 64); - masm.movq(gpr8, gpr9); - masm.movq(gpr9, gpr7); - masm.cmpq(gpr8, 0); - masm.jcc(ConditionFlag.NotEqual, bb10); - - masm.jmp(bb12); - - masm.bind(bb11); - masm.jcc(ConditionFlag.Equal, bb12); - - masm.negl(gpr3); - masm.shrq(gpr9); - masm.movq(gpr1, gpr8); - masm.shrq(gpr8); - masm.subl(gpr4, gpr3); - masm.negl(gpr3); - masm.addl(gpr3, 64); - masm.shlq(gpr1); - masm.orq(gpr9, gpr1); - masm.jmp(bb12); - - masm.bind(bb5); - masm.notl(gpr3); - masm.shlq(gpr8, 32); - masm.orq(gpr8, gpr10); - masm.shlq(gpr8); - masm.movq(gpr6, gpr8); - masm.testl(gpr8, Integer.MIN_VALUE); - masm.jcc(ConditionFlag.NotEqual, bb13); - - masm.shrl(gpr8); - masm.movl(gpr2, 0); - masm.shrq(gpr6, 2); - masm.jmp(bb8); - - masm.bind(bb6); - masm.shrl(gpr8); - masm.movl(gpr2, 1073741824); - masm.shrl(gpr2); - masm.shlq(gpr8, 32); - masm.orq(gpr8, gpr10); - masm.shlq(gpr2, 32); - masm.addl(gpr6, 1073741824); - masm.movl(gpr3, 0); - masm.movl(gpr10, 0); - masm.subq(gpr3, gpr7); - masm.sbbq(gpr10, gpr9); - masm.sbbq(gpr2, gpr8); - masm.movq(gpr7, gpr3); - masm.movq(gpr9, gpr10); - masm.movq(gpr8, gpr2); - masm.movl(gpr2, 32768); - masm.jmp(bb8); - - masm.bind(bb13); - masm.shrl(gpr8); - masm.movq(gpr2, 0x100000000L); - masm.shrq(gpr2); - masm.movl(gpr3, 0); - masm.movl(gpr10, 0); - masm.subq(gpr3, gpr7); - masm.sbbq(gpr10, gpr9); - masm.sbbq(gpr2, gpr8); - masm.movq(gpr7, gpr3); - masm.movq(gpr9, gpr10); - masm.movq(gpr8, gpr2); - masm.movl(gpr2, 32768); - masm.shrq(gpr6, 2); - masm.addl(gpr6, 1073741824); - masm.jmp(bb8); - - masm.bind(bb15); - } - - /* - * Copyright (c) 2014, 2016, Intel Corporation. All rights reserved. Intel Math Library (LIBM) - * Source Code - * - * ALGORITHM DESCRIPTION - EXP() --------------------- - * - * Description: Let K = 64 (table size). x x/log(2) n e = 2 = 2 * T[j] * (1 + P(y)) where x = - * m*log(2)/K + y, y in [-log(2)/K..log(2)/K] m = n*K + j, m,n,j - signed integer, j in - * [-K/2..K/2] j/K values of 2 are tabulated as T[j] = T_hi[j] ( 1 + T_lo[j]). - * - * P(y) is a minimax polynomial approximation of exp(x)-1 on small interval - * [-log(2)/K..log(2)/K] (were calculated by Maple V). - * - * To avoid problems with arithmetic overflow and underflow, n n1 n2 value of 2 is safely - * computed as 2 * 2 where n1 in [-BIAS/2..BIAS/2] where BIAS is a value of exponent bias. - * - * Special cases: exp(NaN) = NaN exp(+INF) = +INF exp(-INF) = 0 exp(x) = 1 for subnormals for - * finite argument, only exp(0)=1 is exact For IEEE double if x > 709.782712893383973096 then - * exp(x) overflow if x < -745.133219101941108420 then exp(x) underflow - * - */ - - private static int[] cvExp = { - 0x652b82fe, 0x40571547, 0x652b82fe, 0x40571547, 0xfefa0000, - 0x3f862e42, 0xfefa0000, 0x3f862e42, 0xbc9e3b3a, 0x3d1cf79a, - 0xbc9e3b3a, 0x3d1cf79a, 0xfffffffe, 0x3fdfffff, 0xfffffffe, - 0x3fdfffff, 0xe3289860, 0x3f56c15c, 0x555b9e25, 0x3fa55555, - 0xc090cf0f, 0x3f811115, 0x55548ba1, 0x3fc55555 - }; - - private static int[] shifterExp = { - 0x00000000, 0x43380000, 0x00000000, 0x43380000 - }; - - private static int[] mMaskExp = { - 0xffffffc0, 0x00000000, 0xffffffc0, 0x00000000 - }; - - private static int[] biasExp = { - 0x0000ffc0, 0x00000000, 0x0000ffc0, 0x00000000 - }; - - private static int[] tblAddrExp = { - 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x0e03754d, - 0x3cad7bbf, 0x3e778060, 0x00002c9a, 0x3567f613, 0x3c8cd252, - 0xd3158574, 0x000059b0, 0x61e6c861, 0x3c60f74e, 0x18759bc8, - 0x00008745, 0x5d837b6c, 0x3c979aa6, 0x6cf9890f, 0x0000b558, - 0x702f9cd1, 0x3c3ebe3d, 0x32d3d1a2, 0x0000e3ec, 0x1e63bcd8, - 0x3ca3516e, 0xd0125b50, 0x00011301, 0x26f0387b, 0x3ca4c554, - 0xaea92ddf, 0x0001429a, 0x62523fb6, 0x3ca95153, 0x3c7d517a, - 0x000172b8, 0x3f1353bf, 0x3c8b898c, 0xeb6fcb75, 0x0001a35b, - 0x3e3a2f5f, 0x3c9aecf7, 0x3168b9aa, 0x0001d487, 0x44a6c38d, - 0x3c8a6f41, 0x88628cd6, 0x0002063b, 0xe3a8a894, 0x3c968efd, - 0x6e756238, 0x0002387a, 0x981fe7f2, 0x3c80472b, 0x65e27cdd, - 0x00026b45, 0x6d09ab31, 0x3c82f7e1, 0xf51fdee1, 0x00029e9d, - 0x720c0ab3, 0x3c8b3782, 0xa6e4030b, 0x0002d285, 0x4db0abb6, - 0x3c834d75, 0x0a31b715, 0x000306fe, 0x5dd3f84a, 0x3c8fdd39, - 0xb26416ff, 0x00033c08, 0xcc187d29, 0x3ca12f8c, 0x373aa9ca, - 0x000371a7, 0x738b5e8b, 0x3ca7d229, 0x34e59ff6, 0x0003a7db, - 0xa72a4c6d, 0x3c859f48, 0x4c123422, 0x0003dea6, 0x259d9205, - 0x3ca8b846, 0x21f72e29, 0x0004160a, 0x60c2ac12, 0x3c4363ed, - 0x6061892d, 0x00044e08, 0xdaa10379, 0x3c6ecce1, 0xb5c13cd0, - 0x000486a2, 0xbb7aafb0, 0x3c7690ce, 0xd5362a27, 0x0004bfda, - 0x9b282a09, 0x3ca083cc, 0x769d2ca6, 0x0004f9b2, 0xc1aae707, - 0x3ca509b0, 0x569d4f81, 0x0005342b, 0x18fdd78e, 0x3c933505, - 0x36b527da, 0x00056f47, 0xe21c5409, 0x3c9063e1, 0xdd485429, - 0x0005ab07, 0x2b64c035, 0x3c9432e6, 0x15ad2148, 0x0005e76f, - 0x99f08c0a, 0x3ca01284, 0xb03a5584, 0x0006247e, 0x0073dc06, - 0x3c99f087, 0x82552224, 0x00066238, 0x0da05571, 0x3c998d4d, - 0x667f3bcc, 0x0006a09e, 0x86ce4786, 0x3ca52bb9, 0x3c651a2e, - 0x0006dfb2, 0x206f0dab, 0x3ca32092, 0xe8ec5f73, 0x00071f75, - 0x8e17a7a6, 0x3ca06122, 0x564267c8, 0x00075feb, 0x461e9f86, - 0x3ca244ac, 0x73eb0186, 0x0007a114, 0xabd66c55, 0x3c65ebe1, - 0x36cf4e62, 0x0007e2f3, 0xbbff67d0, 0x3c96fe9f, 0x994cce12, - 0x00082589, 0x14c801df, 0x3c951f14, 0x9b4492ec, 0x000868d9, - 0xc1f0eab4, 0x3c8db72f, 0x422aa0db, 0x0008ace5, 0x59f35f44, - 0x3c7bf683, 0x99157736, 0x0008f1ae, 0x9c06283c, 0x3ca360ba, - 0xb0cdc5e4, 0x00093737, 0x20f962aa, 0x3c95e8d1, 0x9fde4e4f, - 0x00097d82, 0x2b91ce27, 0x3c71affc, 0x82a3f090, 0x0009c491, - 0x589a2ebd, 0x3c9b6d34, 0x7b5de564, 0x000a0c66, 0x9ab89880, - 0x3c95277c, 0xb23e255c, 0x000a5503, 0x6e735ab3, 0x3c846984, - 0x5579fdbf, 0x000a9e6b, 0x92cb3387, 0x3c8c1a77, 0x995ad3ad, - 0x000ae89f, 0xdc2d1d96, 0x3ca22466, 0xb84f15fa, 0x000b33a2, - 0xb19505ae, 0x3ca1112e, 0xf2fb5e46, 0x000b7f76, 0x0a5fddcd, - 0x3c74ffd7, 0x904bc1d2, 0x000bcc1e, 0x30af0cb3, 0x3c736eae, - 0xdd85529c, 0x000c199b, 0xd10959ac, 0x3c84e08f, 0x2e57d14b, - 0x000c67f1, 0x6c921968, 0x3c676b2c, 0xdcef9069, 0x000cb720, - 0x36df99b3, 0x3c937009, 0x4a07897b, 0x000d072d, 0xa63d07a7, - 0x3c74a385, 0xdcfba487, 0x000d5818, 0xd5c192ac, 0x3c8e5a50, - 0x03db3285, 0x000da9e6, 0x1c4a9792, 0x3c98bb73, 0x337b9b5e, - 0x000dfc97, 0x603a88d3, 0x3c74b604, 0xe78b3ff6, 0x000e502e, - 0x92094926, 0x3c916f27, 0xa2a490d9, 0x000ea4af, 0x41aa2008, - 0x3c8ec3bc, 0xee615a27, 0x000efa1b, 0x31d185ee, 0x3c8a64a9, - 0x5b6e4540, 0x000f5076, 0x4d91cd9d, 0x3c77893b, 0x819e90d8, - 0x000fa7c1 - }; - - private static int[] allOnesExp = { - 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff - }; - - private static int[] expBias = { - 0x00000000, 0x3ff00000, 0x00000000, 0x3ff00000 - }; - - private static int[] xMaxExp = { - 0xffffffff, 0x7fefffff - }; - - private static int[] xMinExp = { - 0x00000000, 0x00100000 - }; - - private static int[] infExp = { - 0x00000000, 0x7ff00000 - }; - - private static int[] zeroExp = { - 0x00000000, 0x00000000 - }; - - public void expIntrinsic(Register dest, Register value, CompilationResultBuilder crb, AMD64MacroAssembler masm) { - ArrayDataPointerConstant onePtr = new ArrayDataPointerConstant(one, 16); - ArrayDataPointerConstant cvExpPtr = new ArrayDataPointerConstant(cvExp, 16); - ArrayDataPointerConstant shifterExpPtr = new ArrayDataPointerConstant(shifterExp, 8); - ArrayDataPointerConstant mMaskExpPtr = new ArrayDataPointerConstant(mMaskExp, 16); - ArrayDataPointerConstant biasExpPtr = new ArrayDataPointerConstant(biasExp, 16); - ArrayDataPointerConstant tblAddrExpPtr = new ArrayDataPointerConstant(tblAddrExp, 16); - ArrayDataPointerConstant expBiasPtr = new ArrayDataPointerConstant(expBias, 8); - ArrayDataPointerConstant xMaxExpPtr = new ArrayDataPointerConstant(xMaxExp, 8); - ArrayDataPointerConstant xMinExpPtr = new ArrayDataPointerConstant(xMinExp, 8); - ArrayDataPointerConstant infExpPtr = new ArrayDataPointerConstant(infExp, 8); - ArrayDataPointerConstant zeroExpPtr = new ArrayDataPointerConstant(zeroExp, 8); - ArrayDataPointerConstant allOnesExpPtr = new ArrayDataPointerConstant(allOnesExp, 8); - - Label bb0 = new Label(); - Label bb1 = new Label(); - Label bb2 = new Label(); - Label bb3 = new Label(); - Label bb4 = new Label(); - Label bb5 = new Label(); - Label bb7 = new Label(); - Label bb8 = new Label(); - Label bb9 = new Label(); - Label bb10 = new Label(); - Label bb11 = new Label(); - Label bb12 = new Label(); - Label bb14 = new Label(); - - Register gpr1 = asRegister(gpr1Temp, AMD64Kind.QWORD); - Register gpr2 = asRegister(gpr2Temp, AMD64Kind.QWORD); - Register gpr3 = asRegister(rcxTemp, AMD64Kind.QWORD); - Register gpr4 = asRegister(gpr4Temp, AMD64Kind.QWORD); - Register gpr5 = asRegister(gpr5Temp, AMD64Kind.QWORD); - - Register temp1 = asRegister(xmm1Temp, AMD64Kind.DOUBLE); - Register temp2 = asRegister(xmm2Temp, AMD64Kind.DOUBLE); - Register temp3 = asRegister(xmm3Temp, AMD64Kind.DOUBLE); - Register temp4 = asRegister(xmm4Temp, AMD64Kind.DOUBLE); - Register temp5 = asRegister(xmm5Temp, AMD64Kind.DOUBLE); - Register temp6 = asRegister(xmm6Temp, AMD64Kind.DOUBLE); - Register temp7 = asRegister(xmm7Temp, AMD64Kind.DOUBLE); - Register temp8 = asRegister(xmm8Temp, AMD64Kind.DOUBLE); - Register temp9 = asRegister(xmm9Temp, AMD64Kind.DOUBLE); - Register temp10 = asRegister(xmm10Temp, AMD64Kind.DOUBLE); - - AMD64Address stackSlot = (AMD64Address) crb.asAddress(stackTemp); - - setCrb(crb); - masm.movsd(stackSlot, value); - if (dest.encoding != value.encoding) { - masm.movdqu(dest, value); - } - - masm.movdqu(temp9, externalAddress(mMaskExpPtr)); // 0xffffffc0, - // 0x00000000, - // 0xffffffc0, - // 0x00000000 - masm.movdqu(temp10, externalAddress(biasExpPtr)); // 0x0000ffc0, - // 0x00000000, - // 0x0000ffc0, - // 0x00000000 - masm.unpcklpd(dest, dest); - masm.leaq(gpr5, stackSlot); - masm.leaq(gpr2, externalAddress(cvExpPtr)); - masm.movdqu(temp1, new AMD64Address(gpr2, 0)); // 0x652b82fe, - // 0x40571547, - // 0x652b82fe, - // 0x40571547 - masm.movdqu(temp6, externalAddress(shifterExpPtr)); // 0x00000000, - // 0x43380000, - // 0x00000000, - // 0x43380000 - masm.movdqu(temp2, new AMD64Address(gpr2, 16)); // 0xfefa0000, - // 0x3f862e42, - // 0xfefa0000, - // 0x3f862e42 - masm.movdqu(temp3, new AMD64Address(gpr2, 32)); // 0xbc9e3b3a, - // 0x3d1cf79a, - // 0xbc9e3b3a, - // 0x3d1cf79a - masm.pextrw(gpr1, dest, 3); - masm.andl(gpr1, 32767); - masm.movl(gpr4, 16527); - masm.subl(gpr4, gpr1); - masm.subl(gpr1, 15504); - masm.orl(gpr4, gpr1); - masm.cmpl(gpr4, Integer.MIN_VALUE); - masm.jcc(ConditionFlag.AboveEqual, bb0); - - masm.leaq(gpr4, externalAddress(tblAddrExpPtr)); - masm.movdqu(temp8, new AMD64Address(gpr2, 48)); // 0xfffffffe, - // 0x3fdfffff, - // 0xfffffffe, - // 0x3fdfffff - masm.movdqu(temp4, new AMD64Address(gpr2, 64)); // 0xe3289860, - // 0x3f56c15c, - // 0x555b9e25, - // 0x3fa55555 - masm.movdqu(temp5, new AMD64Address(gpr2, 80)); // 0xc090cf0f, - // 0x3f811115, - // 0x55548ba1, - // 0x3fc55555 - masm.mulpd(temp1, dest); - masm.addpd(temp1, temp6); - masm.movapd(temp7, temp1); - masm.movdl(gpr1, temp1); - masm.pand(temp7, temp9); - masm.subpd(temp1, temp6); - masm.mulpd(temp2, temp1); - masm.mulpd(temp3, temp1); - masm.paddq(temp7, temp10); - masm.subpd(dest, temp2); - masm.movl(gpr3, gpr1); - masm.andl(gpr3, 63); - masm.shll(gpr3, 4); - masm.movdqu(temp2, new AMD64Address(gpr3, gpr4, Scale.Times1, 0)); - masm.sarl(gpr1, 6); - masm.psllq(temp7, 46); - masm.subpd(dest, temp3); - masm.mulpd(temp4, dest); - masm.movl(gpr4, gpr1); - masm.movapd(temp6, dest); - masm.movapd(temp1, dest); - masm.mulpd(temp6, temp6); - masm.mulpd(dest, temp6); - masm.addpd(temp5, temp4); - masm.mulsd(dest, temp6); - masm.mulpd(temp6, temp8); - masm.addsd(temp1, temp2); - masm.unpckhpd(temp2, temp2); - masm.mulpd(dest, temp5); - masm.addsd(temp1, dest); - masm.por(temp2, temp7); - masm.unpckhpd(dest, dest); - masm.addsd(dest, temp1); - masm.addsd(dest, temp6); - masm.addl(gpr4, 894); - masm.cmpl(gpr4, 1916); - masm.jcc(ConditionFlag.Above, bb1); - - masm.mulsd(dest, temp2); - masm.addsd(dest, temp2); - masm.jmp(bb14); - - masm.bind(bb1); - masm.movdqu(temp6, externalAddress(expBiasPtr)); // 0x00000000, - // 0x3ff00000, - // 0x00000000, - // 0x3ff00000 - masm.xorpd(temp3, temp3); - masm.movdqu(temp4, externalAddress(allOnesExpPtr)); // 0xffffffff, - // 0xffffffff, - // 0xffffffff, - // 0xffffffff - masm.movl(gpr4, -1022); - masm.subl(gpr4, gpr1); - masm.movdl(temp5, gpr4); - masm.psllq(temp4, temp5); - masm.movl(gpr3, gpr1); - masm.sarl(gpr1, 1); - masm.pinsrw(temp3, gpr1, 3); - masm.psllq(temp3, 4); - masm.psubd(temp2, temp3); - masm.mulsd(dest, temp2); - masm.cmpl(gpr4, 52); - masm.jcc(ConditionFlag.Greater, bb2); - - masm.pand(temp4, temp2); - masm.paddd(temp3, temp6); - masm.subsd(temp2, temp4); - masm.addsd(dest, temp2); - masm.cmpl(gpr3, 1023); - masm.jcc(ConditionFlag.GreaterEqual, bb3); - - masm.pextrw(gpr3, dest, 3); - masm.andl(gpr3, 32768); - masm.orl(gpr4, gpr3); - masm.cmpl(gpr4, 0); - masm.jcc(ConditionFlag.Equal, bb4); - - masm.movapd(temp6, dest); - masm.addsd(dest, temp4); - masm.mulsd(dest, temp3); - masm.pextrw(gpr3, dest, 3); - masm.andl(gpr3, 32752); - masm.cmpl(gpr3, 0); - masm.jcc(ConditionFlag.Equal, bb5); - - masm.jmp(bb14); - - masm.bind(bb5); - masm.mulsd(temp6, temp3); - masm.mulsd(temp4, temp3); - masm.movdqu(dest, temp6); - masm.pxor(temp6, temp4); - masm.psrad(temp6, 31); - masm.pshufd(temp6, temp6, 85); - masm.psllq(dest, 1); - masm.psrlq(dest, 1); - masm.pxor(dest, temp6); - masm.psrlq(temp6, 63); - masm.paddq(dest, temp6); - masm.paddq(dest, temp4); - masm.jmp(bb14); - - masm.bind(bb4); - masm.addsd(dest, temp4); - masm.mulsd(dest, temp3); - masm.jmp(bb14); - - masm.bind(bb3); - masm.addsd(dest, temp4); - masm.mulsd(dest, temp3); - masm.pextrw(gpr3, dest, 3); - masm.andl(gpr3, 32752); - masm.cmpl(gpr3, 32752); - masm.jcc(ConditionFlag.AboveEqual, bb7); - - masm.jmp(bb14); - - masm.bind(bb2); - masm.paddd(temp3, temp6); - masm.addpd(dest, temp2); - masm.mulsd(dest, temp3); - masm.jmp(bb14); - - masm.bind(bb8); - masm.movsd(dest, externalAddress(xMaxExpPtr)); // 0xffffffff, - // 0x7fefffff - masm.movsd(temp8, externalAddress(xMinExpPtr)); // 0x00000000, - // 0x00100000 - masm.cmpl(gpr1, 2146435072); - masm.jcc(ConditionFlag.AboveEqual, bb9); - - masm.movl(gpr1, new AMD64Address(gpr5, 4)); - masm.cmpl(gpr1, Integer.MIN_VALUE); - masm.jcc(ConditionFlag.AboveEqual, bb10); - - masm.mulsd(dest, dest); - - masm.bind(bb7); - masm.jmp(bb14); - - masm.bind(bb10); - masm.mulsd(dest, temp8); - masm.jmp(bb14); - - masm.bind(bb9); - masm.movl(gpr4, stackSlot); - masm.cmpl(gpr1, 2146435072); - masm.jcc(ConditionFlag.Above, bb11); - - masm.cmpl(gpr4, 0); - masm.jcc(ConditionFlag.NotEqual, bb11); - - masm.movl(gpr1, new AMD64Address(gpr5, 4)); - masm.cmpl(gpr1, 2146435072); - masm.jcc(ConditionFlag.NotEqual, bb12); - - masm.movsd(dest, externalAddress(infExpPtr)); // 0x00000000, - // 0x7ff00000 - masm.jmp(bb14); - - masm.bind(bb12); - masm.movsd(dest, externalAddress(zeroExpPtr)); // 0x00000000, - // 0x00000000 - masm.jmp(bb14); - - masm.bind(bb11); - masm.movsd(dest, stackSlot); - masm.addsd(dest, dest); - masm.jmp(bb14); - - masm.bind(bb0); - masm.movl(gpr1, new AMD64Address(gpr5, 4)); - masm.andl(gpr1, 2147483647); - masm.cmpl(gpr1, 1083179008); - masm.jcc(ConditionFlag.AboveEqual, bb8); - - masm.addsd(dest, externalAddress(onePtr)); // 0x00000000, - // 0x3ff00000 - masm.bind(bb14); - } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathLog10Op.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathLog10Op.java new file mode 100644 index 00000000000..dec3d32a51f --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathLog10Op.java @@ -0,0 +1,424 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, Intel Corporation. All rights reserved. + * Intel Math Library (LIBM) Source Code + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.amd64; + +import static jdk.vm.ci.amd64.AMD64.r11; +import static jdk.vm.ci.amd64.AMD64.r8; +import static jdk.vm.ci.amd64.AMD64.rax; +import static jdk.vm.ci.amd64.AMD64.rcx; +import static jdk.vm.ci.amd64.AMD64.rdx; +import static jdk.vm.ci.amd64.AMD64.rsp; +import static jdk.vm.ci.amd64.AMD64.xmm0; +import static jdk.vm.ci.amd64.AMD64.xmm1; +import static jdk.vm.ci.amd64.AMD64.xmm2; +import static jdk.vm.ci.amd64.AMD64.xmm3; +import static jdk.vm.ci.amd64.AMD64.xmm4; +import static jdk.vm.ci.amd64.AMD64.xmm5; +import static jdk.vm.ci.amd64.AMD64.xmm6; +import static jdk.vm.ci.amd64.AMD64.xmm7; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.pointerConstant; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.recordExternalAddress; + +import org.graalvm.compiler.asm.Label; +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.asm.amd64.AMD64Assembler; +import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.lir.LIRInstructionClass; +import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; + +/** + * 
+ *                     ALGORITHM DESCRIPTION - LOG10()
+ *                     ---------------------
+ *
+ *    Let x=2^k * mx, mx in [1,2)
+ *
+ *    Get B~1/mx based on the output of rcpss instruction (B0)
+ *    B = int((B0*LH*2^7+0.5))/2^7
+ *    LH is a short approximation for log10(e)
+ *
+ *    Reduced argument: r=B*mx-LH (computed accurately in high and low parts)
+ *
+ *    Result:  k*log10(2) - log(B) + p(r)
+ *             p(r) is a degree 7 polynomial
+ *             -log(B) read from data table (high, low parts)
+ *             Result is formed from high and low parts.
+ *
+ * Special cases:
+ *  log10(0) = -INF with divide-by-zero exception raised
+ *  log10(1) = +0
+ *  log10(x) = NaN with invalid exception raised if x < -0, including -INF
+ *  log10(+INF) = +INF
+ *
+ */ +public final class AMD64MathLog10Op extends AMD64MathIntrinsicUnaryOp { + + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64MathLog10Op.class); + + public AMD64MathLog10Op() { + super(TYPE, /* GPR */ rax, rcx, rdx, r8, r11, + /* XMM */ xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7); + } + + private ArrayDataPointerConstant highsigmask = pointerConstant(16, new int[]{ + // @formatter:off + 0xf8000000, 0xffffffff, 0x00000000, 0xffffe000 + // @formatter:on + }); + + private ArrayDataPointerConstant log10E = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x3fdbc000, + }); + private ArrayDataPointerConstant log10E8 = pointerConstant(8, new int[]{ + 0xbf2e4108, 0x3f5a7a6c + // @formatter:on + }); + + private ArrayDataPointerConstant lTbl = pointerConstant(16, new int[]{ + // @formatter:off + 0x509f7800, 0x3fd34413, 0x1f12b358, 0x3d1fef31, 0x80333400, + 0x3fd32418, 0xc671d9d0, 0xbcf542bf, 0x51195000, 0x3fd30442, + 0x78a4b0c3, 0x3d18216a, 0x6fc79400, 0x3fd2e490, 0x80fa389d, + 0xbc902869, 0x89d04000, 0x3fd2c502, 0x75c2f564, 0x3d040754, + 0x4ddd1c00, 0x3fd2a598, 0xd219b2c3, 0xbcfa1d84, 0x6baa7c00, + 0x3fd28651, 0xfd9abec1, 0x3d1be6d3, 0x94028800, 0x3fd2672d, + 0xe289a455, 0xbd1ede5e, 0x78b86400, 0x3fd2482c, 0x6734d179, + 0x3d1fe79b, 0xcca3c800, 0x3fd2294d, 0x981a40b8, 0xbced34ea, + 0x439c5000, 0x3fd20a91, 0xcc392737, 0xbd1a9cc3, 0x92752c00, + 0x3fd1ebf6, 0x03c9afe7, 0x3d1e98f8, 0x6ef8dc00, 0x3fd1cd7d, + 0x71dae7f4, 0x3d08a86c, 0x8fe4dc00, 0x3fd1af25, 0xee9185a1, + 0xbcff3412, 0xace59400, 0x3fd190ee, 0xc2cab353, 0x3cf17ed9, + 0x7e925000, 0x3fd172d8, 0x6952c1b2, 0x3cf1521c, 0xbe694400, + 0x3fd154e2, 0xcacb79ca, 0xbd0bdc78, 0x26cbac00, 0x3fd1370d, + 0xf71f4de1, 0xbd01f8be, 0x72fa0800, 0x3fd11957, 0x55bf910b, + 0x3c946e2b, 0x5f106000, 0x3fd0fbc1, 0x39e639c1, 0x3d14a84b, + 0xa802a800, 0x3fd0de4a, 0xd3f31d5d, 0xbd178385, 0x0b992000, + 0x3fd0c0f3, 0x3843106f, 0xbd1f602f, 0x486ce800, 0x3fd0a3ba, + 0x8819497c, 0x3cef987a, 0x1de49400, 0x3fd086a0, 0x1caa0467, + 0x3d0faec7, 0x4c30cc00, 0x3fd069a4, 0xa4424372, 0xbd1618fc, + 0x94490000, 0x3fd04cc6, 0x946517d2, 0xbd18384b, 0xb7e84000, + 0x3fd03006, 0xe0109c37, 0xbd19a6ac, 0x798a0c00, 0x3fd01364, + 0x5121e864, 0xbd164cf7, 0x38ce8000, 0x3fcfedbf, 0x46214d1a, + 0xbcbbc402, 0xc8e62000, 0x3fcfb4ef, 0xdab93203, 0x3d1e0176, + 0x2cb02800, 0x3fcf7c5a, 0x2a2ea8e4, 0xbcfec86a, 0xeeeaa000, + 0x3fcf43fd, 0xc18e49a4, 0x3cf110a8, 0x9bb6e800, 0x3fcf0bda, + 0x923cc9c0, 0xbd15ce99, 0xc093f000, 0x3fced3ef, 0x4d4b51e9, + 0x3d1a04c7, 0xec58f800, 0x3fce9c3c, 0x163cad59, 0x3cac8260, + 0x9a907000, 0x3fce2d7d, 0x3fa93646, 0x3ce4a1c0, 0x37311000, + 0x3fcdbf99, 0x32abd1fd, 0x3d07ea9d, 0x6744b800, 0x3fcd528c, + 0x4dcbdfd4, 0xbd1b08e2, 0xe36de800, 0x3fcce653, 0x0b7b7f7f, + 0xbd1b8f03, 0x77506800, 0x3fcc7aec, 0xa821c9fb, 0x3d13c163, + 0x00ff8800, 0x3fcc1053, 0x536bca76, 0xbd074ee5, 0x70719800, + 0x3fcba684, 0xd7da9b6b, 0xbd1fbf16, 0xc6f8d800, 0x3fcb3d7d, + 0xe2220bb3, 0x3d1a295d, 0x16c15800, 0x3fcad53c, 0xe724911e, + 0xbcf55822, 0x82533800, 0x3fca6dbc, 0x6d982371, 0x3cac567c, + 0x3c19e800, 0x3fca06fc, 0x84d17d80, 0x3d1da204, 0x85ef8000, + 0x3fc9a0f8, 0x54466a6a, 0xbd002204, 0xb0ac2000, 0x3fc93bae, + 0xd601fd65, 0x3d18840c, 0x1bb9b000, 0x3fc8d71c, 0x7bf58766, + 0xbd14f897, 0x34aae800, 0x3fc8733e, 0x3af6ac24, 0xbd0f5c45, + 0x76d68000, 0x3fc81012, 0x4303e1a1, 0xbd1f9a80, 0x6af57800, + 0x3fc7ad96, 0x43fbcb46, 0x3cf4c33e, 0xa6c51000, 0x3fc74bc7, + 0x70f0eac5, 0xbd192e3b, 0xccab9800, 0x3fc6eaa3, 0xc0093dfe, + 0xbd0faf15, 0x8b60b800, 0x3fc68a28, 0xde78d5fd, 0xbc9ea4ee, + 0x9d987000, 0x3fc62a53, 0x962bea6e, 0xbd194084, 0xc9b0e800, + 0x3fc5cb22, 0x888dd999, 0x3d1fe201, 0xe1634800, 0x3fc56c93, + 0x16ada7ad, 0x3d1b1188, 0xc176c000, 0x3fc50ea4, 0x4159b5b5, + 0xbcf09c08, 0x51766000, 0x3fc4b153, 0x84393d23, 0xbcf6a89c, + 0x83695000, 0x3fc4549d, 0x9f0b8bbb, 0x3d1c4b8c, 0x538d5800, + 0x3fc3f881, 0xf49df747, 0x3cf89b99, 0xc8138000, 0x3fc39cfc, + 0xd503b834, 0xbd13b99f, 0xf0df0800, 0x3fc3420d, 0xf011b386, + 0xbd05d8be, 0xe7466800, 0x3fc2e7b2, 0xf39c7bc2, 0xbd1bb94e, + 0xcdd62800, 0x3fc28de9, 0x05e6d69b, 0xbd10ed05, 0xd015d800, + 0x3fc234b0, 0xe29b6c9d, 0xbd1ff967, 0x224ea800, 0x3fc1dc06, + 0x727711fc, 0xbcffb30d, 0x01540000, 0x3fc183e8, 0x39786c5a, + 0x3cc23f57, 0xb24d9800, 0x3fc12c54, 0xc905a342, 0x3d003a1d, + 0x82835800, 0x3fc0d54a, 0x9b9920c0, 0x3d03b25a, 0xc72ac000, + 0x3fc07ec7, 0x46f26a24, 0x3cf0fa41, 0xdd35d800, 0x3fc028ca, + 0x41d9d6dc, 0x3d034a65, 0x52474000, 0x3fbfa6a4, 0x44f66449, + 0x3d19cad3, 0x2da3d000, 0x3fbefcb8, 0x67832999, 0x3d18400f, + 0x32a10000, 0x3fbe53ce, 0x9c0e3b1a, 0xbcff62fd, 0x556b7000, + 0x3fbdabe3, 0x02976913, 0xbcf8243b, 0x97e88000, 0x3fbd04f4, + 0xec793797, 0x3d1c0578, 0x09647000, 0x3fbc5eff, 0x05fc0565, + 0xbd1d799e, 0xc6426000, 0x3fbbb9ff, 0x4625f5ed, 0x3d1f5723, + 0xf7afd000, 0x3fbb15f3, 0xdd5aae61, 0xbd1a7e1e, 0xd358b000, + 0x3fba72d8, 0x3314e4d3, 0x3d17bc91, 0x9b1f5000, 0x3fb9d0ab, + 0x9a4d514b, 0x3cf18c9b, 0x9cd4e000, 0x3fb92f69, 0x7e4496ab, + 0x3cf1f96d, 0x31f4f000, 0x3fb88f10, 0xf56479e7, 0x3d165818, + 0xbf628000, 0x3fb7ef9c, 0x26bf486d, 0xbd1113a6, 0xb526b000, + 0x3fb7510c, 0x1a1c3384, 0x3ca9898d, 0x8e31e000, 0x3fb6b35d, + 0xb3875361, 0xbd0661ac, 0xd01de000, 0x3fb6168c, 0x2a7cacfa, + 0xbd1bdf10, 0x0af23000, 0x3fb57a98, 0xff868816, 0x3cf046d0, + 0xd8ea0000, 0x3fb4df7c, 0x1515fbe7, 0xbd1fd529, 0xde3b2000, + 0x3fb44538, 0x6e59a132, 0x3d1faeee, 0xc8df9000, 0x3fb3abc9, + 0xf1322361, 0xbd198807, 0x505f1000, 0x3fb3132d, 0x0888e6ab, + 0x3d1e5380, 0x359bd000, 0x3fb27b61, 0xdfbcbb22, 0xbcfe2724, + 0x429ee000, 0x3fb1e463, 0x6eb4c58c, 0xbcfe4dd6, 0x4a673000, + 0x3fb14e31, 0x4ce1ac9b, 0x3d1ba691, 0x28b96000, 0x3fb0b8c9, + 0x8c7813b8, 0xbd0b3872, 0xc1f08000, 0x3fb02428, 0xc2bc8c2c, + 0x3cb5ea6b, 0x05a1a000, 0x3faf209c, 0x72e8f18e, 0xbce8df84, + 0xc0b5e000, 0x3fadfa6d, 0x9fdef436, 0x3d087364, 0xaf416000, + 0x3facd5c2, 0x1068c3a9, 0x3d0827e7, 0xdb356000, 0x3fabb296, + 0x120a34d3, 0x3d101a9f, 0x5dfea000, 0x3faa90e6, 0xdaded264, + 0xbd14c392, 0x6034c000, 0x3fa970ad, 0x1c9d06a9, 0xbd1b705e, + 0x194c6000, 0x3fa851e8, 0x83996ad9, 0xbd0117bc, 0xcf4ac000, + 0x3fa73492, 0xb1a94a62, 0xbca5ea42, 0xd67b4000, 0x3fa618a9, + 0x75aed8ca, 0xbd07119b, 0x9126c000, 0x3fa4fe29, 0x5291d533, + 0x3d12658f, 0x6f4d4000, 0x3fa3e50e, 0xcd2c5cd9, 0x3d1d5c70, + 0xee608000, 0x3fa2cd54, 0xd1008489, 0x3d1a4802, 0x9900e000, + 0x3fa1b6f9, 0x54fb5598, 0xbd16593f, 0x06bb6000, 0x3fa0a1f9, + 0x64ef57b4, 0xbd17636b, 0xb7940000, 0x3f9f1c9f, 0xee6a4737, + 0x3cb5d479, 0x91aa0000, 0x3f9cf7f5, 0x3a16373c, 0x3d087114, + 0x156b8000, 0x3f9ad5ed, 0x836c554a, 0x3c6900b0, 0xd4764000, + 0x3f98b67f, 0xed12f17b, 0xbcffc974, 0x77dec000, 0x3f9699a7, + 0x232ce7ea, 0x3d1e35bb, 0xbfbf4000, 0x3f947f5d, 0xd84ffa6e, + 0x3d0e0a49, 0x82c7c000, 0x3f92679c, 0x8d170e90, 0xbd14d9f2, + 0xadd20000, 0x3f90525d, 0x86d9f88e, 0x3cdeb986, 0x86f10000, + 0x3f8c7f36, 0xb9e0a517, 0x3ce29faa, 0xb75c8000, 0x3f885e9e, + 0x542568cb, 0xbd1f7bdb, 0x46b30000, 0x3f8442e8, 0xb954e7d9, + 0x3d1e5287, 0xb7e60000, 0x3f802c07, 0x22da0b17, 0xbd19fb27, + 0x6c8b0000, 0x3f7833e3, 0x821271ef, 0xbd190f96, 0x29910000, + 0x3f701936, 0xbc3491a5, 0xbd1bcf45, 0x354a0000, 0x3f600fe3, + 0xc0ff520a, 0xbd19d71c, 0x00000000, 0x00000000, 0x00000000, + 0x00000000 + // @formatter:on + }); + + private ArrayDataPointerConstant log2 = pointerConstant(8, new int[]{ + // @formatter:off + 0x509f7800, 0x3f934413, + }); + private ArrayDataPointerConstant log28 = pointerConstant(8, new int[]{ + 0x1f12b358, 0x3cdfef31 + // @formatter:on + }); + + private ArrayDataPointerConstant coeff = pointerConstant(16, new int[]{ + // @formatter:off + 0xc1a5f12e, 0x40358874, 0x64d4ef0d, 0xc0089309, + }); + private ArrayDataPointerConstant coeff16 = pointerConstant(16, new int[]{ + 0x385593b1, 0xc025c917, 0xdc963467, 0x3ffc6a02, + }); + private ArrayDataPointerConstant coeff32 = pointerConstant(16, new int[]{ + 0x7f9d3aa1, 0x4016ab9f, 0xdc77b115, 0xbff27af2 + // @formatter:on + }); + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + // Registers: + // input: xmm0 + // scratch: xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 + // rax, rdx, rcx, tmp - r11 + // Code generated by Intel C compiler for LIBM library + + Label block0 = new Label(); + Label block1 = new Label(); + Label block2 = new Label(); + Label block3 = new Label(); + Label block4 = new Label(); + Label block5 = new Label(); + Label block6 = new Label(); + Label block7 = new Label(); + Label block8 = new Label(); + Label block9 = new Label(); + + masm.subq(rsp, 24); + masm.movsd(new AMD64Address(rsp, 0), xmm0); + + masm.xorpd(xmm2, xmm2); + masm.movl(rax, 16368); + masm.pinsrw(xmm2, rax, 3); + masm.movl(rcx, 1054736384); + masm.movdl(xmm7, rcx); + masm.xorpd(xmm3, xmm3); + masm.movl(rdx, 30704); + masm.pinsrw(xmm3, rdx, 3); + masm.movdqu(xmm1, xmm0); + masm.movl(rdx, 32768); + masm.movdl(xmm4, rdx); + masm.movdqu(xmm5, recordExternalAddress(crb, highsigmask)); // 0xf8000000, 0xffffffff, + // 0x00000000, 0xffffe000 + masm.pextrw(rax, xmm0, 3); + masm.por(xmm0, xmm2); + masm.movl(rcx, 16352); + masm.psrlq(xmm0, 27); + masm.movdqu(xmm2, recordExternalAddress(crb, log10E)); // 0x00000000, 0x3fdbc000, + // 0xbf2e4108, 0x3f5a7a6c + masm.psrld(xmm0, 2); + masm.rcpps(xmm0, xmm0); + masm.psllq(xmm1, 12); + masm.pshufd(xmm6, xmm5, 78); + masm.psrlq(xmm1, 12); + masm.subl(rax, 16); + masm.cmpl(rax, 32736); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block0); + + masm.bind(block1); + masm.mulss(xmm0, xmm7); + masm.por(xmm1, xmm3); + masm.leaq(r11, recordExternalAddress(crb, lTbl)); + masm.andpd(xmm5, xmm1); + masm.paddd(xmm0, xmm4); + masm.subsd(xmm1, xmm5); + masm.movdl(rdx, xmm0); + masm.psllq(xmm0, 29); + masm.andpd(xmm0, xmm6); + masm.andl(rax, 32752); + masm.subl(rax, rcx); + masm.cvtsi2sdl(xmm7, rax); + masm.mulpd(xmm5, xmm0); + masm.mulsd(xmm1, xmm0); + masm.movq(xmm6, recordExternalAddress(crb, log2)); // 0x509f7800, 0x3f934413, + // 0x1f12b358, 0x3cdfef31 + masm.movdqu(xmm3, recordExternalAddress(crb, coeff)); // 0xc1a5f12e, 0x40358874, + // 0x64d4ef0d, 0xc0089309 + masm.subsd(xmm5, xmm2); + masm.andl(rdx, 16711680); + masm.shrl(rdx, 12); + masm.movdqu(xmm0, new AMD64Address(r11, rdx, AMD64Address.Scale.Times1, -1504)); + masm.movdqu(xmm4, recordExternalAddress(crb, coeff16)); // 0x385593b1, 0xc025c917, + // 0xdc963467, 0x3ffc6a02 + masm.addsd(xmm1, xmm5); + masm.movdqu(xmm2, recordExternalAddress(crb, coeff32)); // 0x7f9d3aa1, 0x4016ab9f, + // 0xdc77b115, 0xbff27af2 + masm.mulsd(xmm6, xmm7); + masm.pshufd(xmm5, xmm1, 68); + masm.mulsd(xmm7, recordExternalAddress(crb, log28)); // 0x1f12b358, 0x3cdfef31 + masm.mulsd(xmm3, xmm1); + masm.addsd(xmm0, xmm6); + masm.mulpd(xmm4, xmm5); + masm.movq(xmm6, recordExternalAddress(crb, log10E8)); // 0xbf2e4108, 0x3f5a7a6c + masm.mulpd(xmm5, xmm5); + masm.addpd(xmm4, xmm2); + masm.mulpd(xmm3, xmm5); + masm.pshufd(xmm2, xmm0, 228); + masm.addsd(xmm0, xmm1); + masm.mulsd(xmm4, xmm1); + masm.subsd(xmm2, xmm0); + masm.mulsd(xmm6, xmm1); + masm.addsd(xmm1, xmm2); + masm.pshufd(xmm2, xmm0, 238); + masm.mulsd(xmm5, xmm5); + masm.addsd(xmm7, xmm2); + masm.addsd(xmm1, xmm6); + masm.addpd(xmm4, xmm3); + masm.addsd(xmm1, xmm7); + masm.mulpd(xmm4, xmm5); + masm.addsd(xmm1, xmm4); + masm.pshufd(xmm5, xmm4, 238); + masm.addsd(xmm1, xmm5); + masm.addsd(xmm0, xmm1); + masm.jmp(block9); + + masm.bind(block0); + masm.movq(xmm0, new AMD64Address(rsp, 0)); + masm.movq(xmm1, new AMD64Address(rsp, 0)); + masm.addl(rax, 16); + masm.cmpl(rax, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block2); + masm.cmpl(rax, 16); + masm.jcc(AMD64Assembler.ConditionFlag.Below, block3); + + masm.bind(block4); + masm.addsd(xmm0, xmm0); + masm.jmp(block9); + + masm.bind(block5); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block4); + masm.cmpl(rdx, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block4); + masm.jmp(block6); + + masm.bind(block3); + masm.xorpd(xmm1, xmm1); + masm.addsd(xmm1, xmm0); + masm.movdl(rdx, xmm1); + masm.psrlq(xmm1, 32); + masm.movdl(rcx, xmm1); + masm.orl(rdx, rcx); + masm.cmpl(rdx, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block7); + masm.xorpd(xmm1, xmm1); + masm.movl(rax, 18416); + masm.pinsrw(xmm1, rax, 3); + masm.mulsd(xmm0, xmm1); + masm.xorpd(xmm2, xmm2); + masm.movl(rax, 16368); + masm.pinsrw(xmm2, rax, 3); + masm.movdqu(xmm1, xmm0); + masm.pextrw(rax, xmm0, 3); + masm.por(xmm0, xmm2); + masm.movl(rcx, 18416); + masm.psrlq(xmm0, 27); + masm.movdqu(xmm2, recordExternalAddress(crb, log10E)); // 0x00000000, 0x3fdbc000, + // 0xbf2e4108, 0x3f5a7a6c + masm.psrld(xmm0, 2); + masm.rcpps(xmm0, xmm0); + masm.psllq(xmm1, 12); + masm.pshufd(xmm6, xmm5, 78); + masm.psrlq(xmm1, 12); + masm.jmp(block1); + + masm.bind(block2); + masm.movdl(rdx, xmm1); + masm.psrlq(xmm1, 32); + masm.movdl(rcx, xmm1); + masm.addl(rcx, rcx); + masm.cmpl(rcx, -2097152); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block5); + masm.orl(rdx, rcx); + masm.cmpl(rdx, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block7); + + masm.bind(block6); + masm.xorpd(xmm1, xmm1); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 32752); + masm.pinsrw(xmm1, rax, 3); + masm.mulsd(xmm0, xmm1); + masm.movl(new AMD64Address(rsp, 16), 9); + masm.jmp(block8); + + masm.bind(block7); + masm.xorpd(xmm1, xmm1); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 49136); + masm.pinsrw(xmm0, rax, 3); + masm.divsd(xmm0, xmm1); + masm.movl(new AMD64Address(rsp, 16), 8); + + masm.bind(block8); + masm.movq(new AMD64Address(rsp, 8), xmm0); + masm.movq(xmm0, new AMD64Address(rsp, 8)); + + masm.bind(block9); + masm.addq(rsp, 24); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathLogOp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathLogOp.java new file mode 100644 index 00000000000..84a5f3b4b3b --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathLogOp.java @@ -0,0 +1,402 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, Intel Corporation. All rights reserved. + * Intel Math Library (LIBM) Source Code + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.amd64; + +import static jdk.vm.ci.amd64.AMD64.r11; +import static jdk.vm.ci.amd64.AMD64.r8; +import static jdk.vm.ci.amd64.AMD64.rax; +import static jdk.vm.ci.amd64.AMD64.rcx; +import static jdk.vm.ci.amd64.AMD64.rdx; +import static jdk.vm.ci.amd64.AMD64.rsp; +import static jdk.vm.ci.amd64.AMD64.xmm0; +import static jdk.vm.ci.amd64.AMD64.xmm1; +import static jdk.vm.ci.amd64.AMD64.xmm2; +import static jdk.vm.ci.amd64.AMD64.xmm3; +import static jdk.vm.ci.amd64.AMD64.xmm4; +import static jdk.vm.ci.amd64.AMD64.xmm5; +import static jdk.vm.ci.amd64.AMD64.xmm6; +import static jdk.vm.ci.amd64.AMD64.xmm7; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.pointerConstant; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.recordExternalAddress; + +import org.graalvm.compiler.asm.Label; +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.asm.amd64.AMD64Assembler; +import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.lir.LIRInstructionClass; +import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; + +import jdk.vm.ci.amd64.AMD64; + +/** + * 
+ *                     ALGORITHM DESCRIPTION - LOG()
+ *                     ---------------------
+ *
+ *    x=2^k * mx, mx in [1,2)
+ *
+ *    Get B~1/mx based on the output of rcpss instruction (B0)
+ *    B = int((B0*2^7+0.5))/2^7
+ *
+ *    Reduced argument: r=B*mx-1.0 (computed accurately in high and low parts)
+ *
+ *    Result:  k*log(2) - log(B) + p(r) if |x-1| >= small value (2^-6)  and
+ *             p(r) is a degree 7 polynomial
+ *             -log(B) read from data table (high, low parts)
+ *             Result is formed from high and low parts.
+ *
+ * Special cases:
+ *  log(NaN) = quiet NaN, and raise invalid exception
+ *  log(+INF) = that INF
+ *  log(0) = -INF with divide-by-zero exception raised
+ *  log(1) = +0
+ *  log(x) = NaN with invalid exception raised if x < -0, including -INF
+ *
+ */ +public final class AMD64MathLogOp extends AMD64MathIntrinsicUnaryOp { + + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64MathLogOp.class); + + public AMD64MathLogOp() { + super(TYPE, /* GPR */ rax, rcx, rdx, r8, r11, + /* XMM */ xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7); + } + + private ArrayDataPointerConstant lTbl = pointerConstant(16, new int[]{ + // @formatter:off + 0xfefa3800, 0x3fe62e42, 0x93c76730, 0x3d2ef357, 0xaa241800, + 0x3fe5ee82, 0x0cda46be, 0x3d220238, 0x5c364800, 0x3fe5af40, + 0xac10c9fb, 0x3d2dfa63, 0x26bb8c00, 0x3fe5707a, 0xff3303dd, + 0x3d09980b, 0x26867800, 0x3fe5322e, 0x5d257531, 0x3d05ccc4, + 0x835a5000, 0x3fe4f45a, 0x6d93b8fb, 0xbd2e6c51, 0x6f970c00, + 0x3fe4b6fd, 0xed4c541c, 0x3cef7115, 0x27e8a400, 0x3fe47a15, + 0xf94d60aa, 0xbd22cb6a, 0xf2f92400, 0x3fe43d9f, 0x481051f7, + 0xbcfd984f, 0x2125cc00, 0x3fe4019c, 0x30f0c74c, 0xbd26ce79, + 0x0c36c000, 0x3fe3c608, 0x7cfe13c2, 0xbd02b736, 0x17197800, + 0x3fe38ae2, 0xbb5569a4, 0xbd218b7a, 0xad9d8c00, 0x3fe35028, + 0x9527e6ac, 0x3d10b83f, 0x44340800, 0x3fe315da, 0xc5a0ed9c, + 0xbd274e93, 0x57b0e000, 0x3fe2dbf5, 0x07b9dc11, 0xbd17a6e5, + 0x6d0ec000, 0x3fe2a278, 0xe797882d, 0x3d206d2b, 0x1134dc00, + 0x3fe26962, 0x05226250, 0xbd0b61f1, 0xd8bebc00, 0x3fe230b0, + 0x6e48667b, 0x3d12fc06, 0x5fc61800, 0x3fe1f863, 0xc9fe81d3, + 0xbd2a7242, 0x49ae6000, 0x3fe1c078, 0xed70e667, 0x3cccacde, + 0x40f23c00, 0x3fe188ee, 0xf8ab4650, 0x3d14cc4e, 0xf6f29800, + 0x3fe151c3, 0xa293ae49, 0xbd2edd97, 0x23c75c00, 0x3fe11af8, + 0xbb9ddcb2, 0xbd258647, 0x8611cc00, 0x3fe0e489, 0x07801742, + 0x3d1c2998, 0xe2d05400, 0x3fe0ae76, 0x887e7e27, 0x3d1f486b, + 0x0533c400, 0x3fe078bf, 0x41edf5fd, 0x3d268122, 0xbe760400, + 0x3fe04360, 0xe79539e0, 0xbd04c45f, 0xe5b20800, 0x3fe00e5a, + 0xb1727b1c, 0xbd053ba3, 0xaf7a4800, 0x3fdfb358, 0x3c164935, + 0x3d0085fa, 0xee031800, 0x3fdf4aa7, 0x6f014a8b, 0x3d12cde5, + 0x56b41000, 0x3fdee2a1, 0x5a470251, 0x3d2f27f4, 0xc3ddb000, + 0x3fde7b42, 0x5372bd08, 0xbd246550, 0x1a272800, 0x3fde148a, + 0x07322938, 0xbd1326b2, 0x484c9800, 0x3fddae75, 0x60dc616a, + 0xbd1ea42d, 0x46def800, 0x3fdd4902, 0xe9a767a8, 0x3d235baf, + 0x18064800, 0x3fdce42f, 0x3ec7a6b0, 0xbd0797c3, 0xc7455800, + 0x3fdc7ff9, 0xc15249ae, 0xbd29b6dd, 0x693fa000, 0x3fdc1c60, + 0x7fe8e180, 0x3d2cec80, 0x1b80e000, 0x3fdbb961, 0xf40a666d, + 0x3d27d85b, 0x04462800, 0x3fdb56fa, 0x2d841995, 0x3d109525, + 0x5248d000, 0x3fdaf529, 0x52774458, 0xbd217cc5, 0x3c8ad800, + 0x3fda93ed, 0xbea77a5d, 0x3d1e36f2, 0x0224f800, 0x3fda3344, + 0x7f9d79f5, 0x3d23c645, 0xea15f000, 0x3fd9d32b, 0x10d0c0b0, + 0xbd26279e, 0x43135800, 0x3fd973a3, 0xa502d9f0, 0xbd152313, + 0x635bf800, 0x3fd914a8, 0x2ee6307d, 0xbd1766b5, 0xa88b3000, + 0x3fd8b639, 0xe5e70470, 0xbd205ae1, 0x776dc800, 0x3fd85855, + 0x3333778a, 0x3d2fd56f, 0x3bd81800, 0x3fd7fafa, 0xc812566a, + 0xbd272090, 0x687cf800, 0x3fd79e26, 0x2efd1778, 0x3d29ec7d, + 0x76c67800, 0x3fd741d8, 0x49dc60b3, 0x3d2d8b09, 0xe6af1800, + 0x3fd6e60e, 0x7c222d87, 0x3d172165, 0x3e9c6800, 0x3fd68ac8, + 0x2756eba0, 0x3d20a0d3, 0x0b3ab000, 0x3fd63003, 0xe731ae00, + 0xbd2db623, 0xdf596000, 0x3fd5d5bd, 0x08a465dc, 0xbd0a0b2a, + 0x53c8d000, 0x3fd57bf7, 0xee5d40ef, 0x3d1faded, 0x0738a000, + 0x3fd522ae, 0x8164c759, 0x3d2ebe70, 0x9e173000, 0x3fd4c9e0, + 0x1b0ad8a4, 0xbd2e2089, 0xc271c800, 0x3fd4718d, 0x0967d675, + 0xbd2f27ce, 0x23d5e800, 0x3fd419b4, 0xec90e09d, 0x3d08e436, + 0x77333000, 0x3fd3c252, 0xb606bd5c, 0x3d183b54, 0x76be1000, + 0x3fd36b67, 0xb0f177c8, 0x3d116ecd, 0xe1d36000, 0x3fd314f1, + 0xd3213cb8, 0xbd28e27a, 0x7cdc9000, 0x3fd2bef0, 0x4a5004f4, + 0x3d2a9cfa, 0x1134d800, 0x3fd26962, 0xdf5bb3b6, 0x3d2c93c1, + 0x6d0eb800, 0x3fd21445, 0xba46baea, 0x3d0a87de, 0x635a6800, + 0x3fd1bf99, 0x5147bdb7, 0x3d2ca6ed, 0xcbacf800, 0x3fd16b5c, + 0xf7a51681, 0x3d2b9acd, 0x8227e800, 0x3fd1178e, 0x63a5f01c, + 0xbd2c210e, 0x67616000, 0x3fd0c42d, 0x163ceae9, 0x3d27188b, + 0x604d5800, 0x3fd07138, 0x16ed4e91, 0x3cf89cdb, 0x5626c800, + 0x3fd01eae, 0x1485e94a, 0xbd16f08c, 0x6cb3b000, 0x3fcf991c, + 0xca0cdf30, 0x3d1bcbec, 0xe4dd0000, 0x3fcef5ad, 0x65bb8e11, + 0xbcca2115, 0xffe71000, 0x3fce530e, 0x6041f430, 0x3cc21227, + 0xb0d49000, 0x3fcdb13d, 0xf715b035, 0xbd2aff2a, 0xf2656000, + 0x3fcd1037, 0x75b6f6e4, 0xbd084a7e, 0xc6f01000, 0x3fcc6ffb, + 0xc5962bd2, 0xbcf1ec72, 0x383be000, 0x3fcbd087, 0x595412b6, + 0xbd2d4bc4, 0x575bd000, 0x3fcb31d8, 0x4eace1aa, 0xbd0c358d, + 0x3c8ae000, 0x3fca93ed, 0x50562169, 0xbd287243, 0x07089000, + 0x3fc9f6c4, 0x6865817a, 0x3d29904d, 0xdcf70000, 0x3fc95a5a, + 0x58a0ff6f, 0x3d07f228, 0xeb390000, 0x3fc8beaf, 0xaae92cd1, + 0xbd073d54, 0x6551a000, 0x3fc823c1, 0x9a631e83, 0x3d1e0ddb, + 0x85445000, 0x3fc7898d, 0x70914305, 0xbd1c6610, 0x8b757000, + 0x3fc6f012, 0xe59c21e1, 0xbd25118d, 0xbe8c1000, 0x3fc6574e, + 0x2c3c2e78, 0x3d19cf8b, 0x6b544000, 0x3fc5bf40, 0xeb68981c, + 0xbd127023, 0xe4a1b000, 0x3fc527e5, 0xe5697dc7, 0x3d2633e8, + 0x8333b000, 0x3fc4913d, 0x54fdb678, 0x3d258379, 0xa5993000, + 0x3fc3fb45, 0x7e6a354d, 0xbd2cd1d8, 0xb0159000, 0x3fc365fc, + 0x234b7289, 0x3cc62fa8, 0x0c868000, 0x3fc2d161, 0xcb81b4a1, + 0x3d039d6c, 0x2a49c000, 0x3fc23d71, 0x8fd3df5c, 0x3d100d23, + 0x7e23f000, 0x3fc1aa2b, 0x44389934, 0x3d2ca78e, 0x8227e000, + 0x3fc1178e, 0xce2d07f2, 0x3d21ef78, 0xb59e4000, 0x3fc08598, + 0x7009902c, 0xbd27e5dd, 0x39dbe000, 0x3fbfe891, 0x4fa10afd, + 0xbd2534d6, 0x830a2000, 0x3fbec739, 0xafe645e0, 0xbd2dc068, + 0x63844000, 0x3fbda727, 0x1fa71733, 0x3d1a8940, 0x01bc4000, + 0x3fbc8858, 0xc65aacd3, 0x3d2646d1, 0x8dad6000, 0x3fbb6ac8, + 0x2bf768e5, 0xbd139080, 0x40b1c000, 0x3fba4e76, 0xb94407c8, + 0xbd0e42b6, 0x5d594000, 0x3fb9335e, 0x3abd47da, 0x3d23115c, + 0x2f40e000, 0x3fb8197e, 0xf96ffdf7, 0x3d0f80dc, 0x0aeac000, + 0x3fb700d3, 0xa99ded32, 0x3cec1e8d, 0x4d97a000, 0x3fb5e95a, + 0x3c5d1d1e, 0xbd2c6906, 0x5d208000, 0x3fb4d311, 0x82f4e1ef, + 0xbcf53a25, 0xa7d1e000, 0x3fb3bdf5, 0xa5db4ed7, 0x3d2cc85e, + 0xa4472000, 0x3fb2aa04, 0xae9c697d, 0xbd20b6e8, 0xd1466000, + 0x3fb1973b, 0x560d9e9b, 0xbd25325d, 0xb59e4000, 0x3fb08598, + 0x7009902c, 0xbd17e5dd, 0xc006c000, 0x3faeea31, 0x4fc93b7b, + 0xbd0e113e, 0xcdddc000, 0x3faccb73, 0x47d82807, 0xbd1a68f2, + 0xd0fb0000, 0x3faaaef2, 0x353bb42e, 0x3d20fc1a, 0x149fc000, + 0x3fa894aa, 0xd05a267d, 0xbd197995, 0xf2d4c000, 0x3fa67c94, + 0xec19afa2, 0xbd029efb, 0xd42e0000, 0x3fa466ae, 0x75bdfd28, + 0xbd2c1673, 0x2f8d0000, 0x3fa252f3, 0xe021b67b, 0x3d283e9a, + 0x89e74000, 0x3fa0415d, 0x5cf1d753, 0x3d0111c0, 0xec148000, + 0x3f9c63d2, 0x3f9eb2f3, 0x3d2578c6, 0x28c90000, 0x3f984925, + 0x325a0c34, 0xbd2aa0ba, 0x25980000, 0x3f9432a9, 0x928637fe, + 0x3d098139, 0x58938000, 0x3f902056, 0x06e2f7d2, 0xbd23dc5b, + 0xa3890000, 0x3f882448, 0xda74f640, 0xbd275577, 0x75890000, + 0x3f801015, 0x999d2be8, 0xbd10c76b, 0x59580000, 0x3f700805, + 0xcb31c67b, 0x3d2166af, 0x00000000, 0x00000000, 0x00000000, + 0x80000000 + // @formatter:on + }); + + private ArrayDataPointerConstant log2 = pointerConstant(8, new int[]{ + // @formatter:off + 0xfefa3800, 0x3fa62e42, + }); + private ArrayDataPointerConstant log28 = pointerConstant(8, new int[]{ + 0x93c76730, 0x3ceef357 + // @formatter:on + }); + + private ArrayDataPointerConstant coeff = pointerConstant(16, new int[]{ + // @formatter:off + 0x92492492, 0x3fc24924, 0x00000000, 0xbfd00000, + }); + private ArrayDataPointerConstant coeff16 = pointerConstant(16, new int[]{ + 0x3d6fb175, 0xbfc5555e, 0x55555555, 0x3fd55555, + }); + private ArrayDataPointerConstant coeff32 = pointerConstant(16, new int[]{ + 0x9999999a, 0x3fc99999, 0x00000000, 0xbfe00000 + // @formatter:on + }); + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + // registers, + // input: xmm0 + // scratch: xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 + // rax, rdx, rcx, r8, r11 + Label block0 = new Label(); + Label block1 = new Label(); + Label block2 = new Label(); + Label block3 = new Label(); + Label block4 = new Label(); + Label block5 = new Label(); + Label block6 = new Label(); + Label block7 = new Label(); + Label block8 = new Label(); + Label block9 = new Label(); + + masm.subq(rsp, 24); + masm.movsd(new AMD64Address(rsp, 0), xmm0); + masm.movq(rax, 0x3ff0000000000000L); + masm.movdq(xmm2, rax); + masm.movq(rdx, 0x77f0000000000000L); + masm.movdq(xmm3, rdx); + masm.movl(rcx, 32768); + masm.movdl(xmm4, rcx); + masm.movq(r8, 0xffffe00000000000L); + masm.movdq(xmm5, r8); + masm.movdqu(xmm1, xmm0); + masm.pextrw(rax, xmm0, 3); + masm.por(xmm0, xmm2); + masm.movl(rcx, 16352); + masm.psrlq(xmm0, 27); + masm.leaq(r11, recordExternalAddress(crb, lTbl)); + masm.psrld(xmm0, 2); + masm.rcpps(xmm0, xmm0); + masm.psllq(xmm1, 12); + masm.pshufd(xmm6, xmm5, 228); + masm.psrlq(xmm1, 12); + masm.subl(rax, 16); + masm.cmpl(rax, 32736); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block0); + + masm.bind(block1); + masm.paddd(xmm0, xmm4); + masm.por(xmm1, xmm3); + masm.movdl(rdx, xmm0); + masm.psllq(xmm0, 29); + masm.pand(xmm5, xmm1); + masm.pand(xmm0, xmm6); + masm.subsd(xmm1, xmm5); + masm.mulpd(xmm5, xmm0); + masm.andl(rax, 32752); + masm.subl(rax, rcx); + masm.cvtsi2sdl(xmm7, rax); + masm.mulsd(xmm1, xmm0); + masm.movq(xmm6, recordExternalAddress(crb, log2)); // 0xfefa3800, 0x3fa62e42 + masm.movdqu(xmm3, recordExternalAddress(crb, coeff)); // 0x92492492, 0x3fc24924, + // 0x00000000, 0xbfd00000 + masm.subsd(xmm5, xmm2); + masm.andl(rdx, 16711680); + masm.shrl(rdx, 12); + masm.movdqu(xmm0, new AMD64Address(r11, rdx, AMD64Address.Scale.Times1)); + masm.movdqu(xmm4, recordExternalAddress(crb, coeff16)); // 0x3d6fb175, 0xbfc5555e, + // 0x55555555, 0x3fd55555 + masm.addsd(xmm1, xmm5); + masm.movdqu(xmm2, recordExternalAddress(crb, coeff32)); // 0x9999999a, 0x3fc99999, + // 0x00000000, 0xbfe00000 + masm.mulsd(xmm6, xmm7); + if (masm.supports(AMD64.CPUFeature.SSE3)) { + masm.movddup(xmm5, xmm1); + } else { + masm.movdqu(xmm5, xmm1); + masm.movlhps(xmm5, xmm5); + } + masm.mulsd(xmm7, recordExternalAddress(crb, log28)); // 0x93c76730, 0x3ceef357 + masm.mulsd(xmm3, xmm1); + masm.addsd(xmm0, xmm6); + masm.mulpd(xmm4, xmm5); + masm.mulpd(xmm5, xmm5); + if (masm.supports(AMD64.CPUFeature.SSE3)) { + masm.movddup(xmm6, xmm0); + } else { + masm.movdqu(xmm6, xmm0); + masm.movlhps(xmm6, xmm6); + } + masm.addsd(xmm0, xmm1); + masm.addpd(xmm4, xmm2); + masm.mulpd(xmm3, xmm5); + masm.subsd(xmm6, xmm0); + masm.mulsd(xmm4, xmm1); + masm.pshufd(xmm2, xmm0, 238); + masm.addsd(xmm1, xmm6); + masm.mulsd(xmm5, xmm5); + masm.addsd(xmm7, xmm2); + masm.addpd(xmm4, xmm3); + masm.addsd(xmm1, xmm7); + masm.mulpd(xmm4, xmm5); + masm.addsd(xmm1, xmm4); + masm.pshufd(xmm5, xmm4, 238); + masm.addsd(xmm1, xmm5); + masm.addsd(xmm0, xmm1); + masm.jmp(block9); + + masm.bind(block0); + masm.movq(xmm0, new AMD64Address(rsp, 0)); + masm.movq(xmm1, new AMD64Address(rsp, 0)); + masm.addl(rax, 16); + masm.cmpl(rax, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block2); + masm.cmpl(rax, 16); + masm.jcc(AMD64Assembler.ConditionFlag.Below, block3); + + masm.bind(block4); + masm.addsd(xmm0, xmm0); + masm.jmp(block9); + + masm.bind(block5); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block4); + masm.cmpl(rdx, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block4); + masm.jmp(block6); + + masm.bind(block3); + masm.xorpd(xmm1, xmm1); + masm.addsd(xmm1, xmm0); + masm.movdl(rdx, xmm1); + masm.psrlq(xmm1, 32); + masm.movdl(rcx, xmm1); + masm.orl(rdx, rcx); + masm.cmpl(rdx, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block7); + masm.xorpd(xmm1, xmm1); + masm.movl(rax, 18416); + masm.pinsrw(xmm1, rax, 3); + masm.mulsd(xmm0, xmm1); + masm.movdqu(xmm1, xmm0); + masm.pextrw(rax, xmm0, 3); + masm.por(xmm0, xmm2); + masm.psrlq(xmm0, 27); + masm.movl(rcx, 18416); + masm.psrld(xmm0, 2); + masm.rcpps(xmm0, xmm0); + masm.psllq(xmm1, 12); + masm.pshufd(xmm6, xmm5, 228); + masm.psrlq(xmm1, 12); + masm.jmp(block1); + + masm.bind(block2); + masm.movdl(rdx, xmm1); + masm.psrlq(xmm1, 32); + masm.movdl(rcx, xmm1); + masm.addl(rcx, rcx); + masm.cmpl(rcx, -2097152); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block5); + masm.orl(rdx, rcx); + masm.cmpl(rdx, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block7); + + masm.bind(block6); + masm.xorpd(xmm1, xmm1); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 32752); + masm.pinsrw(xmm1, rax, 3); + masm.mulsd(xmm0, xmm1); + masm.movl(new AMD64Address(rsp, 16), 3); + masm.jmp(block8); + masm.bind(block7); + masm.xorpd(xmm1, xmm1); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 49136); + masm.pinsrw(xmm0, rax, 3); + masm.divsd(xmm0, xmm1); + masm.movl(new AMD64Address(rsp, 16), 2); + + masm.bind(block8); + masm.movq(new AMD64Address(rsp, 8), xmm0); + + masm.movq(xmm0, new AMD64Address(rsp, 8)); + + masm.bind(block9); + masm.addq(rsp, 24); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathPowOp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathPowOp.java new file mode 100644 index 00000000000..d075bfdc8e7 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathPowOp.java @@ -0,0 +1,1979 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, Intel Corporation. All rights reserved. + * Intel Math Library (LIBM) Source Code + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.amd64; + +import static jdk.vm.ci.amd64.AMD64.r10; +import static jdk.vm.ci.amd64.AMD64.r11; +import static jdk.vm.ci.amd64.AMD64.r8; +import static jdk.vm.ci.amd64.AMD64.r9; +import static jdk.vm.ci.amd64.AMD64.rax; +import static jdk.vm.ci.amd64.AMD64.rcx; +import static jdk.vm.ci.amd64.AMD64.rdx; +import static jdk.vm.ci.amd64.AMD64.rsp; +import static jdk.vm.ci.amd64.AMD64.xmm0; +import static jdk.vm.ci.amd64.AMD64.xmm1; +import static jdk.vm.ci.amd64.AMD64.xmm2; +import static jdk.vm.ci.amd64.AMD64.xmm3; +import static jdk.vm.ci.amd64.AMD64.xmm4; +import static jdk.vm.ci.amd64.AMD64.xmm5; +import static jdk.vm.ci.amd64.AMD64.xmm6; +import static jdk.vm.ci.amd64.AMD64.xmm7; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.pointerConstant; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.recordExternalAddress; + +import org.graalvm.compiler.asm.Label; +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.asm.amd64.AMD64Assembler; +import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.lir.LIRInstructionClass; +import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; + +import jdk.vm.ci.code.Register; + +/** + * 
+ *                     ALGORITHM DESCRIPTION  - POW()
+ *                     ---------------------
+ *
+ *    Let x=2^k * mx, mx in [1,2)
+ *
+ *    log2(x) calculation:
+ *
+ *    Get B~1/mx based on the output of rcpps instruction (B0)
+ *    B = int((B0*LH*2^9+0.5))/2^9
+ *    LH is a short approximation for log2(e)
+ *
+ *    Reduced argument, scaled by LH:
+ *                r=B*mx-LH (computed accurately in high and low parts)
+ *
+ *    log2(x) result:  k - log2(B) + p(r)
+ *             p(r) is a degree 8 polynomial
+ *             -log2(B) read from data table (high, low parts)
+ *             log2(x) is formed from high and low parts
+ *    For |x| in [1-1/32, 1+1/16), a slower but more accurate computation
+ *    based om the same table design is performed.
+ *
+ *   Main path is taken if | floor(log2(|log2(|x|)|) + floor(log2|y|) | < 8,
+ *   to filter out all potential OF/UF cases.
+ *   exp2(y*log2(x)) is computed using an 8-bit index table and a degree 5
+ *   polynomial
+ *
+ * Special cases:
+ *  pow(-0,y) = -INF and raises the divide-by-zero exception for y an odd
+ *  integer < 0.
+ *  pow(-0,y) = +INF and raises the divide-by-zero exception for y < 0 and
+ *  not an odd integer.
+ *  pow(-0,y) = -0 for y an odd integer > 0.
+ *  pow(-0,y) = +0 for y > 0 and not an odd integer.
+ *  pow(-1,-INF) = NaN.
+ *  pow(+1,y) = NaN for any y, even a NaN.
+ *  pow(x,-0) = 1 for any x, even a NaN.
+ *  pow(x,y) = a NaN and raises the invalid exception for finite x < 0 and
+ *  finite non-integer y.
+ *  pow(x,-INF) = +INF for |x|<1.
+ *  pow(x,-INF) = +0 for |x|>1.
+ *  pow(x,+INF) = +0 for |x|<1.
+ *  pow(x,+INF) = +INF for |x|>1.
+ *  pow(-INF,y) = -0 for y an odd integer < 0.
+ *  pow(-INF,y) = +0 for y < 0 and not an odd integer.
+ *  pow(-INF,y) = -INF for y an odd integer > 0.
+ *  pow(-INF,y) = +INF for y > 0 and not an odd integer.
+ *  pow(+INF,y) = +0 for y <0.
+ *  pow(+INF,y) = +INF for y >0.
+ *
+ */ +public final class AMD64MathPowOp extends AMD64MathIntrinsicBinaryOp { + + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64MathPowOp.class); + + public AMD64MathPowOp() { + super(TYPE, /* GPR */ rax, rcx, rdx, r8, r9, r10, r11, + /* XMM */ xmm2, xmm3, xmm4, xmm5, xmm6, xmm7); + } + + private ArrayDataPointerConstant highsigmask = pointerConstant(16, new int[]{ + //@formatter:off + 0x00000000, 0xfffff800, 0x00000000, 0xfffff800 + //@formatter:on + }); + + private ArrayDataPointerConstant log2E = pointerConstant(16, new int[]{ + //@formatter:off + 0x00000000, 0x3ff72000, 0x161bb241, 0xbf5dabe1 + //@formatter:on + }); + + private ArrayDataPointerConstant highmaskY = pointerConstant(8, new int[]{ + //@formatter:off + 0x00000000, 0xfffffff8, + }); + private ArrayDataPointerConstant highmaskY8 = pointerConstant(8, new int[]{ + 0x00000000, 0xffffffff + //@formatter:on + }); + + private ArrayDataPointerConstant tExp = pointerConstant(16, new int[]{ + //@formatter:off + 0x00000000, 0x3ff00000, 0x00000000, 0x3b700000, 0xfa5abcbf, + 0x3ff00b1a, 0xa7609f71, 0xbc84f6b2, 0xa9fb3335, 0x3ff0163d, + 0x9ab8cdb7, 0x3c9b6129, 0x143b0281, 0x3ff02168, 0x0fc54eb6, + 0xbc82bf31, 0x3e778061, 0x3ff02c9a, 0x535b085d, 0xbc719083, + 0x2e11bbcc, 0x3ff037d4, 0xeeade11a, 0x3c656811, 0xe86e7f85, + 0x3ff04315, 0x1977c96e, 0xbc90a31c, 0x72f654b1, 0x3ff04e5f, + 0x3aa0d08c, 0x3c84c379, 0xd3158574, 0x3ff059b0, 0xa475b465, + 0x3c8d73e2, 0x0e3c1f89, 0x3ff0650a, 0x5799c397, 0xbc95cb7b, + 0x29ddf6de, 0x3ff0706b, 0xe2b13c27, 0xbc8c91df, 0x2b72a836, + 0x3ff07bd4, 0x54458700, 0x3c832334, 0x18759bc8, 0x3ff08745, + 0x4bb284ff, 0x3c6186be, 0xf66607e0, 0x3ff092bd, 0x800a3fd1, + 0xbc968063, 0xcac6f383, 0x3ff09e3e, 0x18316136, 0x3c914878, + 0x9b1f3919, 0x3ff0a9c7, 0x873d1d38, 0x3c85d16c, 0x6cf9890f, + 0x3ff0b558, 0x4adc610b, 0x3c98a62e, 0x45e46c85, 0x3ff0c0f1, + 0x06d21cef, 0x3c94f989, 0x2b7247f7, 0x3ff0cc92, 0x16e24f71, + 0x3c901edc, 0x23395dec, 0x3ff0d83b, 0xe43f316a, 0xbc9bc14d, + 0x32d3d1a2, 0x3ff0e3ec, 0x27c57b52, 0x3c403a17, 0x5fdfa9c5, + 0x3ff0efa5, 0xbc54021b, 0xbc949db9, 0xaffed31b, 0x3ff0fb66, + 0xc44ebd7b, 0xbc6b9bed, 0x28d7233e, 0x3ff10730, 0x1692fdd5, + 0x3c8d46eb, 0xd0125b51, 0x3ff11301, 0x39449b3a, 0xbc96c510, + 0xab5e2ab6, 0x3ff11edb, 0xf703fb72, 0xbc9ca454, 0xc06c31cc, + 0x3ff12abd, 0xb36ca5c7, 0xbc51b514, 0x14f204ab, 0x3ff136a8, + 0xba48dcf0, 0xbc67108f, 0xaea92de0, 0x3ff1429a, 0x9af1369e, + 0xbc932fbf, 0x934f312e, 0x3ff14e95, 0x39bf44ab, 0xbc8b91e8, + 0xc8a58e51, 0x3ff15a98, 0xb9eeab0a, 0x3c82406a, 0x5471c3c2, + 0x3ff166a4, 0x82ea1a32, 0x3c58f23b, 0x3c7d517b, 0x3ff172b8, + 0xb9d78a76, 0xbc819041, 0x8695bbc0, 0x3ff17ed4, 0xe2ac5a64, + 0x3c709e3f, 0x388c8dea, 0x3ff18af9, 0xd1970f6c, 0xbc911023, + 0x58375d2f, 0x3ff19726, 0x85f17e08, 0x3c94aadd, 0xeb6fcb75, + 0x3ff1a35b, 0x7b4968e4, 0x3c8e5b4c, 0xf8138a1c, 0x3ff1af99, + 0xa4b69280, 0x3c97bf85, 0x84045cd4, 0x3ff1bbe0, 0x352ef607, + 0xbc995386, 0x95281c6b, 0x3ff1c82f, 0x8010f8c9, 0x3c900977, + 0x3168b9aa, 0x3ff1d487, 0x00a2643c, 0x3c9e016e, 0x5eb44027, + 0x3ff1e0e7, 0x088cb6de, 0xbc96fdd8, 0x22fcd91d, 0x3ff1ed50, + 0x027bb78c, 0xbc91df98, 0x8438ce4d, 0x3ff1f9c1, 0xa097af5c, + 0xbc9bf524, 0x88628cd6, 0x3ff2063b, 0x814a8495, 0x3c8dc775, + 0x3578a819, 0x3ff212be, 0x2cfcaac9, 0x3c93592d, 0x917ddc96, + 0x3ff21f49, 0x9494a5ee, 0x3c82a97e, 0xa27912d1, 0x3ff22bdd, + 0x5577d69f, 0x3c8d34fb, 0x6e756238, 0x3ff2387a, 0xb6c70573, + 0x3c99b07e, 0xfb82140a, 0x3ff2451f, 0x911ca996, 0x3c8acfcc, + 0x4fb2a63f, 0x3ff251ce, 0xbef4f4a4, 0x3c8ac155, 0x711ece75, + 0x3ff25e85, 0x4ac31b2c, 0x3c93e1a2, 0x65e27cdd, 0x3ff26b45, + 0x9940e9d9, 0x3c82bd33, 0x341ddf29, 0x3ff2780e, 0x05f9e76c, + 0x3c9e067c, 0xe1f56381, 0x3ff284df, 0x8c3f0d7e, 0xbc9a4c3a, + 0x7591bb70, 0x3ff291ba, 0x28401cbd, 0xbc82cc72, 0xf51fdee1, + 0x3ff29e9d, 0xafad1255, 0x3c8612e8, 0x66d10f13, 0x3ff2ab8a, + 0x191690a7, 0xbc995743, 0xd0dad990, 0x3ff2b87f, 0xd6381aa4, + 0xbc410adc, 0x39771b2f, 0x3ff2c57e, 0xa6eb5124, 0xbc950145, + 0xa6e4030b, 0x3ff2d285, 0x54db41d5, 0x3c900247, 0x1f641589, + 0x3ff2df96, 0xfbbce198, 0x3c9d16cf, 0xa93e2f56, 0x3ff2ecaf, + 0x45d52383, 0x3c71ca0f, 0x4abd886b, 0x3ff2f9d2, 0x532bda93, + 0xbc653c55, 0x0a31b715, 0x3ff306fe, 0xd23182e4, 0x3c86f46a, + 0xedeeb2fd, 0x3ff31432, 0xf3f3fcd1, 0x3c8959a3, 0xfc4cd831, + 0x3ff32170, 0x8e18047c, 0x3c8a9ce7, 0x3ba8ea32, 0x3ff32eb8, + 0x3cb4f318, 0xbc9c45e8, 0xb26416ff, 0x3ff33c08, 0x843659a6, + 0x3c932721, 0x66e3fa2d, 0x3ff34962, 0x930881a4, 0xbc835a75, + 0x5f929ff1, 0x3ff356c5, 0x5c4e4628, 0xbc8b5cee, 0xa2de883b, + 0x3ff36431, 0xa06cb85e, 0xbc8c3144, 0x373aa9cb, 0x3ff371a7, + 0xbf42eae2, 0xbc963aea, 0x231e754a, 0x3ff37f26, 0x9eceb23c, + 0xbc99f5ca, 0x6d05d866, 0x3ff38cae, 0x3c9904bd, 0xbc9e958d, + 0x1b7140ef, 0x3ff39a40, 0xfc8e2934, 0xbc99a9a5, 0x34e59ff7, + 0x3ff3a7db, 0xd661f5e3, 0xbc75e436, 0xbfec6cf4, 0x3ff3b57f, + 0xe26fff18, 0x3c954c66, 0xc313a8e5, 0x3ff3c32d, 0x375d29c3, + 0xbc9efff8, 0x44ede173, 0x3ff3d0e5, 0x8c284c71, 0x3c7fe8d0, + 0x4c123422, 0x3ff3dea6, 0x11f09ebc, 0x3c8ada09, 0xdf1c5175, + 0x3ff3ec70, 0x7b8c9bca, 0xbc8af663, 0x04ac801c, 0x3ff3fa45, + 0xf956f9f3, 0xbc97d023, 0xc367a024, 0x3ff40822, 0xb6f4d048, + 0x3c8bddf8, 0x21f72e2a, 0x3ff4160a, 0x1c309278, 0xbc5ef369, + 0x2709468a, 0x3ff423fb, 0xc0b314dd, 0xbc98462d, 0xd950a897, + 0x3ff431f5, 0xe35f7999, 0xbc81c7dd, 0x3f84b9d4, 0x3ff43ffa, + 0x9704c003, 0x3c8880be, 0x6061892d, 0x3ff44e08, 0x04ef80d0, + 0x3c489b7a, 0x42a7d232, 0x3ff45c20, 0x82fb1f8e, 0xbc686419, + 0xed1d0057, 0x3ff46a41, 0xd1648a76, 0x3c9c944b, 0x668b3237, + 0x3ff4786d, 0xed445733, 0xbc9c20f0, 0xb5c13cd0, 0x3ff486a2, + 0xb69062f0, 0x3c73c1a3, 0xe192aed2, 0x3ff494e1, 0x5e499ea0, + 0xbc83b289, 0xf0d7d3de, 0x3ff4a32a, 0xf3d1be56, 0x3c99cb62, + 0xea6db7d7, 0x3ff4b17d, 0x7f2897f0, 0xbc8125b8, 0xd5362a27, + 0x3ff4bfda, 0xafec42e2, 0x3c7d4397, 0xb817c114, 0x3ff4ce41, + 0x690abd5d, 0x3c905e29, 0x99fddd0d, 0x3ff4dcb2, 0xbc6a7833, + 0x3c98ecdb, 0x81d8abff, 0x3ff4eb2d, 0x2e5d7a52, 0xbc95257d, + 0x769d2ca7, 0x3ff4f9b2, 0xd25957e3, 0xbc94b309, 0x7f4531ee, + 0x3ff50841, 0x49b7465f, 0x3c7a249b, 0xa2cf6642, 0x3ff516da, + 0x69bd93ef, 0xbc8f7685, 0xe83f4eef, 0x3ff5257d, 0x43efef71, + 0xbc7c998d, 0x569d4f82, 0x3ff5342b, 0x1db13cad, 0xbc807abe, + 0xf4f6ad27, 0x3ff542e2, 0x192d5f7e, 0x3c87926d, 0xca5d920f, + 0x3ff551a4, 0xefede59b, 0xbc8d689c, 0xdde910d2, 0x3ff56070, + 0x168eebf0, 0xbc90fb6e, 0x36b527da, 0x3ff56f47, 0x011d93ad, + 0x3c99bb2c, 0xdbe2c4cf, 0x3ff57e27, 0x8a57b9c4, 0xbc90b98c, + 0xd497c7fd, 0x3ff58d12, 0x5b9a1de8, 0x3c8295e1, 0x27ff07cc, + 0x3ff59c08, 0xe467e60f, 0xbc97e2ce, 0xdd485429, 0x3ff5ab07, + 0x054647ad, 0x3c96324c, 0xfba87a03, 0x3ff5ba11, 0x4c233e1a, + 0xbc9b77a1, 0x8a5946b7, 0x3ff5c926, 0x816986a2, 0x3c3c4b1b, + 0x90998b93, 0x3ff5d845, 0xa8b45643, 0xbc9cd6a7, 0x15ad2148, + 0x3ff5e76f, 0x3080e65e, 0x3c9ba6f9, 0x20dceb71, 0x3ff5f6a3, + 0xe3cdcf92, 0xbc89eadd, 0xb976dc09, 0x3ff605e1, 0x9b56de47, + 0xbc93e242, 0xe6cdf6f4, 0x3ff6152a, 0x4ab84c27, 0x3c9e4b3e, + 0xb03a5585, 0x3ff6247e, 0x7e40b497, 0xbc9383c1, 0x1d1929fd, + 0x3ff633dd, 0xbeb964e5, 0x3c984710, 0x34ccc320, 0x3ff64346, + 0x759d8933, 0xbc8c483c, 0xfebc8fb7, 0x3ff652b9, 0xc9a73e09, + 0xbc9ae3d5, 0x82552225, 0x3ff66238, 0x87591c34, 0xbc9bb609, + 0xc70833f6, 0x3ff671c1, 0x586c6134, 0xbc8e8732, 0xd44ca973, + 0x3ff68155, 0x44f73e65, 0x3c6038ae, 0xb19e9538, 0x3ff690f4, + 0x9aeb445d, 0x3c8804bd, 0x667f3bcd, 0x3ff6a09e, 0x13b26456, + 0xbc9bdd34, 0xfa75173e, 0x3ff6b052, 0x2c9a9d0e, 0x3c7a38f5, + 0x750bdabf, 0x3ff6c012, 0x67ff0b0d, 0xbc728956, 0xddd47645, + 0x3ff6cfdc, 0xb6f17309, 0x3c9c7aa9, 0x3c651a2f, 0x3ff6dfb2, + 0x683c88ab, 0xbc6bbe3a, 0x98593ae5, 0x3ff6ef92, 0x9e1ac8b2, + 0xbc90b974, 0xf9519484, 0x3ff6ff7d, 0x25860ef6, 0xbc883c0f, + 0x66f42e87, 0x3ff70f74, 0xd45aa65f, 0x3c59d644, 0xe8ec5f74, + 0x3ff71f75, 0x86887a99, 0xbc816e47, 0x86ead08a, 0x3ff72f82, + 0x2cd62c72, 0xbc920aa0, 0x48a58174, 0x3ff73f9a, 0x6c65d53c, + 0xbc90a8d9, 0x35d7cbfd, 0x3ff74fbd, 0x618a6e1c, 0x3c9047fd, + 0x564267c9, 0x3ff75feb, 0x57316dd3, 0xbc902459, 0xb1ab6e09, + 0x3ff77024, 0x169147f8, 0x3c9b7877, 0x4fde5d3f, 0x3ff78069, + 0x0a02162d, 0x3c9866b8, 0x38ac1cf6, 0x3ff790b9, 0x62aadd3e, + 0x3c9349a8, 0x73eb0187, 0x3ff7a114, 0xee04992f, 0xbc841577, + 0x0976cfdb, 0x3ff7b17b, 0x8468dc88, 0xbc9bebb5, 0x0130c132, + 0x3ff7c1ed, 0xd1164dd6, 0x3c9f124c, 0x62ff86f0, 0x3ff7d26a, + 0xfb72b8b4, 0x3c91bddb, 0x36cf4e62, 0x3ff7e2f3, 0xba15797e, + 0x3c705d02, 0x8491c491, 0x3ff7f387, 0xcf9311ae, 0xbc807f11, + 0x543e1a12, 0x3ff80427, 0x626d972b, 0xbc927c86, 0xadd106d9, + 0x3ff814d2, 0x0d151d4d, 0x3c946437, 0x994cce13, 0x3ff82589, + 0xd41532d8, 0xbc9d4c1d, 0x1eb941f7, 0x3ff8364c, 0x31df2bd5, + 0x3c999b9a, 0x4623c7ad, 0x3ff8471a, 0xa341cdfb, 0xbc88d684, + 0x179f5b21, 0x3ff857f4, 0xf8b216d0, 0xbc5ba748, 0x9b4492ed, + 0x3ff868d9, 0x9bd4f6ba, 0xbc9fc6f8, 0xd931a436, 0x3ff879ca, + 0xd2db47bd, 0x3c85d2d7, 0xd98a6699, 0x3ff88ac7, 0xf37cb53a, + 0x3c9994c2, 0xa478580f, 0x3ff89bd0, 0x4475202a, 0x3c9d5395, + 0x422aa0db, 0x3ff8ace5, 0x56864b27, 0x3c96e9f1, 0xbad61778, + 0x3ff8be05, 0xfc43446e, 0x3c9ecb5e, 0x16b5448c, 0x3ff8cf32, + 0x32e9e3aa, 0xbc70d55e, 0x5e0866d9, 0x3ff8e06a, 0x6fc9b2e6, + 0xbc97114a, 0x99157736, 0x3ff8f1ae, 0xa2e3976c, 0x3c85cc13, + 0xd0282c8a, 0x3ff902fe, 0x85fe3fd2, 0x3c9592ca, 0x0b91ffc6, + 0x3ff9145b, 0x2e582524, 0xbc9dd679, 0x53aa2fe2, 0x3ff925c3, + 0xa639db7f, 0xbc83455f, 0xb0cdc5e5, 0x3ff93737, 0x81b57ebc, + 0xbc675fc7, 0x2b5f98e5, 0x3ff948b8, 0x797d2d99, 0xbc8dc3d6, + 0xcbc8520f, 0x3ff95a44, 0x96a5f039, 0xbc764b7c, 0x9a7670b3, + 0x3ff96bdd, 0x7f19c896, 0xbc5ba596, 0x9fde4e50, 0x3ff97d82, + 0x7c1b85d1, 0xbc9d185b, 0xe47a22a2, 0x3ff98f33, 0xa24c78ec, + 0x3c7cabda, 0x70ca07ba, 0x3ff9a0f1, 0x91cee632, 0xbc9173bd, + 0x4d53fe0d, 0x3ff9b2bb, 0x4df6d518, 0xbc9dd84e, 0x82a3f090, + 0x3ff9c491, 0xb071f2be, 0x3c7c7c46, 0x194bb8d5, 0x3ff9d674, + 0xa3dd8233, 0xbc9516be, 0x19e32323, 0x3ff9e863, 0x78e64c6e, + 0x3c7824ca, 0x8d07f29e, 0x3ff9fa5e, 0xaaf1face, 0xbc84a9ce, + 0x7b5de565, 0x3ffa0c66, 0x5d1cd533, 0xbc935949, 0xed8eb8bb, + 0x3ffa1e7a, 0xee8be70e, 0x3c9c6618, 0xec4a2d33, 0x3ffa309b, + 0x7ddc36ab, 0x3c96305c, 0x80460ad8, 0x3ffa42c9, 0x589fb120, + 0xbc9aa780, 0xb23e255d, 0x3ffa5503, 0xdb8d41e1, 0xbc9d2f6e, + 0x8af46052, 0x3ffa674a, 0x30670366, 0x3c650f56, 0x1330b358, + 0x3ffa799e, 0xcac563c7, 0x3c9bcb7e, 0x53c12e59, 0x3ffa8bfe, + 0xb2ba15a9, 0xbc94f867, 0x5579fdbf, 0x3ffa9e6b, 0x0ef7fd31, + 0x3c90fac9, 0x21356eba, 0x3ffab0e5, 0xdae94545, 0x3c889c31, + 0xbfd3f37a, 0x3ffac36b, 0xcae76cd0, 0xbc8f9234, 0x3a3c2774, + 0x3ffad5ff, 0xb6b1b8e5, 0x3c97ef3b, 0x995ad3ad, 0x3ffae89f, + 0x345dcc81, 0x3c97a1cd, 0xe622f2ff, 0x3ffafb4c, 0x0f315ecd, + 0xbc94b2fc, 0x298db666, 0x3ffb0e07, 0x4c80e425, 0xbc9bdef5, + 0x6c9a8952, 0x3ffb20ce, 0x4a0756cc, 0x3c94dd02, 0xb84f15fb, + 0x3ffb33a2, 0x3084d708, 0xbc62805e, 0x15b749b1, 0x3ffb4684, + 0xe9df7c90, 0xbc7f763d, 0x8de5593a, 0x3ffb5972, 0xbbba6de3, + 0xbc9c71df, 0x29f1c52a, 0x3ffb6c6e, 0x52883f6e, 0x3c92a8f3, + 0xf2fb5e47, 0x3ffb7f76, 0x7e54ac3b, 0xbc75584f, 0xf22749e4, + 0x3ffb928c, 0x54cb65c6, 0xbc9b7216, 0x30a1064a, 0x3ffba5b0, + 0x0e54292e, 0xbc9efcd3, 0xb79a6f1f, 0x3ffbb8e0, 0xc9696205, + 0xbc3f52d1, 0x904bc1d2, 0x3ffbcc1e, 0x7a2d9e84, 0x3c823dd0, + 0xc3f3a207, 0x3ffbdf69, 0x60ea5b53, 0xbc3c2623, 0x5bd71e09, + 0x3ffbf2c2, 0x3f6b9c73, 0xbc9efdca, 0x6141b33d, 0x3ffc0628, + 0xa1fbca34, 0xbc8d8a5a, 0xdd85529c, 0x3ffc199b, 0x895048dd, + 0x3c811065, 0xd9fa652c, 0x3ffc2d1c, 0x17c8a5d7, 0xbc96e516, + 0x5fffd07a, 0x3ffc40ab, 0xe083c60a, 0x3c9b4537, 0x78fafb22, + 0x3ffc5447, 0x2493b5af, 0x3c912f07, 0x2e57d14b, 0x3ffc67f1, + 0xff483cad, 0x3c92884d, 0x8988c933, 0x3ffc7ba8, 0xbe255559, + 0xbc8e76bb, 0x9406e7b5, 0x3ffc8f6d, 0x48805c44, 0x3c71acbc, + 0x5751c4db, 0x3ffca340, 0xd10d08f5, 0xbc87f2be, 0xdcef9069, + 0x3ffcb720, 0xd1e949db, 0x3c7503cb, 0x2e6d1675, 0x3ffccb0f, + 0x86009092, 0xbc7d220f, 0x555dc3fa, 0x3ffcdf0b, 0x53829d72, + 0xbc8dd83b, 0x5b5bab74, 0x3ffcf315, 0xb86dff57, 0xbc9a08e9, + 0x4a07897c, 0x3ffd072d, 0x43797a9c, 0xbc9cbc37, 0x2b08c968, + 0x3ffd1b53, 0x219a36ee, 0x3c955636, 0x080d89f2, 0x3ffd2f87, + 0x719d8578, 0xbc9d487b, 0xeacaa1d6, 0x3ffd43c8, 0xbf5a1614, + 0x3c93db53, 0xdcfba487, 0x3ffd5818, 0xd75b3707, 0x3c82ed02, + 0xe862e6d3, 0x3ffd6c76, 0x4a8165a0, 0x3c5fe87a, 0x16c98398, + 0x3ffd80e3, 0x8beddfe8, 0xbc911ec1, 0x71ff6075, 0x3ffd955d, + 0xbb9af6be, 0x3c9a052d, 0x03db3285, 0x3ffda9e6, 0x696db532, + 0x3c9c2300, 0xd63a8315, 0x3ffdbe7c, 0x926b8be4, 0xbc9b76f1, + 0xf301b460, 0x3ffdd321, 0x78f018c3, 0x3c92da57, 0x641c0658, + 0x3ffde7d5, 0x8e79ba8f, 0xbc9ca552, 0x337b9b5f, 0x3ffdfc97, + 0x4f184b5c, 0xbc91a5cd, 0x6b197d17, 0x3ffe1167, 0xbd5c7f44, + 0xbc72b529, 0x14f5a129, 0x3ffe2646, 0x817a1496, 0xbc97b627, + 0x3b16ee12, 0x3ffe3b33, 0x31fdc68b, 0xbc99f4a4, 0xe78b3ff6, + 0x3ffe502e, 0x80a9cc8f, 0x3c839e89, 0x24676d76, 0x3ffe6539, + 0x7522b735, 0xbc863ff8, 0xfbc74c83, 0x3ffe7a51, 0xca0c8de2, + 0x3c92d522, 0x77cdb740, 0x3ffe8f79, 0x80b054b1, 0xbc910894, + 0xa2a490da, 0x3ffea4af, 0x179c2893, 0xbc9e9c23, 0x867cca6e, + 0x3ffeb9f4, 0x2293e4f2, 0x3c94832f, 0x2d8e67f1, 0x3ffecf48, + 0xb411ad8c, 0xbc9c93f3, 0xa2188510, 0x3ffee4aa, 0xa487568d, + 0x3c91c68d, 0xee615a27, 0x3ffefa1b, 0x86a4b6b0, 0x3c9dc7f4, + 0x1cb6412a, 0x3fff0f9c, 0x65181d45, 0xbc932200, 0x376bba97, + 0x3fff252b, 0xbf0d8e43, 0x3c93a1a5, 0x48dd7274, 0x3fff3ac9, + 0x3ed837de, 0xbc795a5a, 0x5b6e4540, 0x3fff5076, 0x2dd8a18b, + 0x3c99d3e1, 0x798844f8, 0x3fff6632, 0x3539343e, 0x3c9fa37b, + 0xad9cbe14, 0x3fff7bfd, 0xd006350a, 0xbc9dbb12, 0x02243c89, + 0x3fff91d8, 0xa779f689, 0xbc612ea8, 0x819e90d8, 0x3fffa7c1, + 0xf3a5931e, 0x3c874853, 0x3692d514, 0x3fffbdba, 0x15098eb6, + 0xbc796773, 0x2b8f71f1, 0x3fffd3c2, 0x966579e7, 0x3c62eb74, + 0x6b2a23d9, 0x3fffe9d9, 0x7442fde3, 0x3c74a603 + //@formatter:on + }); + + private ArrayDataPointerConstant eCoeff = pointerConstant(16, new int[]{ + //@formatter:off + 0xe78a6731, 0x3f55d87f, 0xd704a0c0, 0x3fac6b08, + }); + private ArrayDataPointerConstant eCoeff16 = pointerConstant(16, new int[]{ + 0x6fba4e77, 0x3f83b2ab, 0xff82c58f, 0x3fcebfbd, + }); + private ArrayDataPointerConstant eCoeff32 = pointerConstant(16, new int[]{ + 0xfefa39ef, 0x3fe62e42, 0x00000000, 0x00000000 + //@formatter:on + }); + + private ArrayDataPointerConstant coeffH = pointerConstant(8, new int[]{ + //@formatter:off + 0x00000000, 0xbfd61a00, + }); + private ArrayDataPointerConstant coeffH8 = pointerConstant(8, new int[]{ + 0x00000000, 0xbf5dabe1 + //@formatter:on + }); + + private ArrayDataPointerConstant highmaskLogX = pointerConstant(16, new int[]{ + //@formatter:off + 0xf8000000, 0xffffffff, 0x00000000, 0xfffff800 + //@formatter:on + }); + + private ArrayDataPointerConstant halfmask = pointerConstant(8, new int[]{ + //@formatter:off + 0xf8000000, 0xffffffff, 0xf8000000, 0xffffffff + //@formatter:on + }); + + private ArrayDataPointerConstant coeff = pointerConstant(16, new int[]{ + //@formatter:off + 0x6dc96112, 0xbf836578, 0xee241472, 0xbf9b0301, + }); + private ArrayDataPointerConstant coeff16 = pointerConstant(16, new int[]{ + 0x9f95985a, 0xbfb528db, 0xb3841d2a, 0xbfd619b6, + }); + private ArrayDataPointerConstant coeff32 = pointerConstant(16, new int[]{ + 0x518775e3, 0x3f9004f2, 0xac8349bb, 0x3fa76c9b, + }); + private ArrayDataPointerConstant coeff48 = pointerConstant(16, new int[]{ + 0x486ececc, 0x3fc4635e, 0x161bb241, 0xbf5dabe1, + }); + private ArrayDataPointerConstant coeff64 = pointerConstant(16, new int[]{ + 0x9f95985a, 0xbfb528db, 0xf8b5787d, 0x3ef2531e, + }); + private ArrayDataPointerConstant coeff80 = pointerConstant(16, new int[]{ + 0x486ececb, 0x3fc4635e, 0x412055cc, 0xbdd61bb2 + //@formatter:on + }); + + private ArrayDataPointerConstant lTbl = pointerConstant(16, new int[]{ + //@formatter:off + 0x00000000, 0x3ff00000, 0x00000000, 0x00000000, 0x20000000, + 0x3feff00a, 0x96621f95, 0x3e5b1856, 0xe0000000, 0x3fefe019, + 0xe5916f9e, 0xbe325278, 0x00000000, 0x3fefd02f, 0x859a1062, + 0x3e595fb7, 0xc0000000, 0x3fefc049, 0xb245f18f, 0xbe529c38, + 0xe0000000, 0x3fefb069, 0xad2880a7, 0xbe501230, 0x60000000, + 0x3fefa08f, 0xc8e72420, 0x3e597bd1, 0x80000000, 0x3fef90ba, + 0xc30c4500, 0xbe5d6c75, 0xe0000000, 0x3fef80ea, 0x02c63f43, + 0x3e2e1318, 0xc0000000, 0x3fef7120, 0xb3d4cccc, 0xbe44c52a, + 0x00000000, 0x3fef615c, 0xdbd91397, 0xbe4e7d6c, 0xa0000000, + 0x3fef519c, 0x65c5cd68, 0xbe522dc8, 0xa0000000, 0x3fef41e2, + 0x46d1306c, 0xbe5a840e, 0xe0000000, 0x3fef322d, 0xd2980e94, + 0x3e5071af, 0xa0000000, 0x3fef227e, 0x773abade, 0xbe5891e5, + 0xa0000000, 0x3fef12d4, 0xdc6bf46b, 0xbe5cccbe, 0xe0000000, + 0x3fef032f, 0xbc7247fa, 0xbe2bab83, 0x80000000, 0x3feef390, + 0xbcaa1e46, 0xbe53bb3b, 0x60000000, 0x3feee3f6, 0x5f6c682d, + 0xbe54c619, 0x80000000, 0x3feed461, 0x5141e368, 0xbe4b6d86, + 0xe0000000, 0x3feec4d1, 0xec678f76, 0xbe369af6, 0x80000000, + 0x3feeb547, 0x41301f55, 0xbe2d4312, 0x60000000, 0x3feea5c2, + 0x676da6bd, 0xbe4d8dd0, 0x60000000, 0x3fee9642, 0x57a891c4, + 0x3e51f991, 0xa0000000, 0x3fee86c7, 0xe4eb491e, 0x3e579bf9, + 0x20000000, 0x3fee7752, 0xfddc4a2c, 0xbe3356e6, 0xc0000000, + 0x3fee67e1, 0xd75b5bf1, 0xbe449531, 0x80000000, 0x3fee5876, + 0xbd423b8e, 0x3df54fe4, 0x60000000, 0x3fee4910, 0x330e51b9, + 0x3e54289c, 0x80000000, 0x3fee39af, 0x8651a95f, 0xbe55aad6, + 0xa0000000, 0x3fee2a53, 0x5e98c708, 0xbe2fc4a9, 0xe0000000, + 0x3fee1afc, 0x0989328d, 0x3e23958c, 0x40000000, 0x3fee0bab, + 0xee642abd, 0xbe425dd8, 0xa0000000, 0x3fedfc5e, 0xc394d236, + 0x3e526362, 0x20000000, 0x3feded17, 0xe104aa8e, 0x3e4ce247, + 0xc0000000, 0x3fedddd4, 0x265a9be4, 0xbe5bb77a, 0x40000000, + 0x3fedce97, 0x0ecac52f, 0x3e4a7cb1, 0xe0000000, 0x3fedbf5e, + 0x124cb3b8, 0x3e257024, 0x80000000, 0x3fedb02b, 0xe6d4febe, + 0xbe2033ee, 0x20000000, 0x3feda0fd, 0x39cca00e, 0xbe3ddabc, + 0xc0000000, 0x3fed91d3, 0xef8a552a, 0xbe543390, 0x40000000, + 0x3fed82af, 0xb8e85204, 0x3e513850, 0xe0000000, 0x3fed738f, + 0x3d59fe08, 0xbe5db728, 0x40000000, 0x3fed6475, 0x3aa7ead1, + 0x3e58804b, 0xc0000000, 0x3fed555f, 0xf8a35ba9, 0xbe5298b0, + 0x00000000, 0x3fed464f, 0x9a88dd15, 0x3e5a8cdb, 0x40000000, + 0x3fed3743, 0xb0b0a190, 0x3e598635, 0x80000000, 0x3fed283c, + 0xe2113295, 0xbe5c1119, 0x80000000, 0x3fed193a, 0xafbf1728, + 0xbe492e9c, 0x60000000, 0x3fed0a3d, 0xe4a4ccf3, 0x3e19b90e, + 0x20000000, 0x3fecfb45, 0xba3cbeb8, 0x3e406b50, 0xc0000000, + 0x3fecec51, 0x110f7ddd, 0x3e0d6806, 0x40000000, 0x3fecdd63, + 0x7dd7d508, 0xbe5a8943, 0x80000000, 0x3fecce79, 0x9b60f271, + 0xbe50676a, 0x80000000, 0x3fecbf94, 0x0b9ad660, 0x3e59174f, + 0x60000000, 0x3fecb0b4, 0x00823d9c, 0x3e5bbf72, 0x20000000, + 0x3feca1d9, 0x38a6ec89, 0xbe4d38f9, 0x80000000, 0x3fec9302, + 0x3a0b7d8e, 0x3e53dbfd, 0xc0000000, 0x3fec8430, 0xc6826b34, + 0xbe27c5c9, 0xc0000000, 0x3fec7563, 0x0c706381, 0xbe593653, + 0x60000000, 0x3fec669b, 0x7df34ec7, 0x3e461ab5, 0xe0000000, + 0x3fec57d7, 0x40e5e7e8, 0xbe5c3dae, 0x00000000, 0x3fec4919, + 0x5602770f, 0xbe55219d, 0xc0000000, 0x3fec3a5e, 0xec7911eb, + 0x3e5a5d25, 0x60000000, 0x3fec2ba9, 0xb39ea225, 0xbe53c00b, + 0x80000000, 0x3fec1cf8, 0x967a212e, 0x3e5a8ddf, 0x60000000, + 0x3fec0e4c, 0x580798bd, 0x3e5f53ab, 0x00000000, 0x3febffa5, + 0xb8282df6, 0xbe46b874, 0x20000000, 0x3febf102, 0xe33a6729, + 0x3e54963f, 0x00000000, 0x3febe264, 0x3b53e88a, 0xbe3adce1, + 0x60000000, 0x3febd3ca, 0xc2585084, 0x3e5cde9f, 0x80000000, + 0x3febc535, 0xa335c5ee, 0xbe39fd9c, 0x20000000, 0x3febb6a5, + 0x7325b04d, 0x3e42ba15, 0x60000000, 0x3feba819, 0x1564540f, + 0x3e3a9f35, 0x40000000, 0x3feb9992, 0x83fff592, 0xbe5465ce, + 0xa0000000, 0x3feb8b0f, 0xb9da63d3, 0xbe4b1a0a, 0x80000000, + 0x3feb7c91, 0x6d6f1ea4, 0x3e557657, 0x00000000, 0x3feb6e18, + 0x5e80a1bf, 0x3e4ddbb6, 0x00000000, 0x3feb5fa3, 0x1c9eacb5, + 0x3e592877, 0xa0000000, 0x3feb5132, 0x6d40beb3, 0xbe51858c, + 0xa0000000, 0x3feb42c6, 0xd740c67b, 0x3e427ad2, 0x40000000, + 0x3feb345f, 0xa3e0ccee, 0xbe5c2fc4, 0x40000000, 0x3feb25fc, + 0x8e752b50, 0xbe3da3c2, 0xc0000000, 0x3feb179d, 0xa892e7de, + 0x3e1fb481, 0xc0000000, 0x3feb0943, 0x21ed71e9, 0xbe365206, + 0x20000000, 0x3feafaee, 0x0e1380a3, 0x3e5c5b7b, 0x20000000, + 0x3feaec9d, 0x3c3d640e, 0xbe5dbbd0, 0x60000000, 0x3feade50, + 0x8f97a715, 0x3e3a8ec5, 0x20000000, 0x3fead008, 0x23ab2839, + 0x3e2fe98a, 0x40000000, 0x3feac1c4, 0xf4bbd50f, 0x3e54d8f6, + 0xe0000000, 0x3feab384, 0x14757c4d, 0xbe48774c, 0xc0000000, + 0x3feaa549, 0x7c7b0eea, 0x3e5b51bb, 0x20000000, 0x3fea9713, + 0xf56f7013, 0x3e386200, 0xe0000000, 0x3fea88e0, 0xbe428ebe, + 0xbe514af5, 0xe0000000, 0x3fea7ab2, 0x8d0e4496, 0x3e4f9165, + 0x60000000, 0x3fea6c89, 0xdbacc5d5, 0xbe5c063b, 0x20000000, + 0x3fea5e64, 0x3f19d970, 0xbe5a0c8c, 0x20000000, 0x3fea5043, + 0x09ea3e6b, 0x3e5065dc, 0x80000000, 0x3fea4226, 0x78df246c, + 0x3e5e05f6, 0x40000000, 0x3fea340e, 0x4057d4a0, 0x3e431b2b, + 0x40000000, 0x3fea25fa, 0x82867bb5, 0x3e4b76be, 0xa0000000, + 0x3fea17ea, 0x9436f40a, 0xbe5aad39, 0x20000000, 0x3fea09df, + 0x4b5253b3, 0x3e46380b, 0x00000000, 0x3fe9fbd8, 0x8fc52466, + 0xbe386f9b, 0x20000000, 0x3fe9edd5, 0x22d3f344, 0xbe538347, + 0x60000000, 0x3fe9dfd6, 0x1ac33522, 0x3e5dbc53, 0x00000000, + 0x3fe9d1dc, 0xeabdff1d, 0x3e40fc0c, 0xe0000000, 0x3fe9c3e5, + 0xafd30e73, 0xbe585e63, 0xe0000000, 0x3fe9b5f3, 0xa52f226a, + 0xbe43e8f9, 0x20000000, 0x3fe9a806, 0xecb8698d, 0xbe515b36, + 0x80000000, 0x3fe99a1c, 0xf2b4e89d, 0x3e48b62b, 0x20000000, + 0x3fe98c37, 0x7c9a88fb, 0x3e44414c, 0x00000000, 0x3fe97e56, + 0xda015741, 0xbe5d13ba, 0xe0000000, 0x3fe97078, 0x5fdace06, + 0x3e51b947, 0x00000000, 0x3fe962a0, 0x956ca094, 0x3e518785, + 0x40000000, 0x3fe954cb, 0x01164c1d, 0x3e5d5b57, 0xc0000000, + 0x3fe946fa, 0xe63b3767, 0xbe4f84e7, 0x40000000, 0x3fe9392e, + 0xe57cc2a9, 0x3e34eda3, 0xe0000000, 0x3fe92b65, 0x8c75b544, + 0x3e5766a0, 0xc0000000, 0x3fe91da1, 0x37d1d087, 0xbe5e2ab1, + 0x80000000, 0x3fe90fe1, 0xa953dc20, 0x3e5fa1f3, 0x80000000, + 0x3fe90225, 0xdbd3f369, 0x3e47d6db, 0xa0000000, 0x3fe8f46d, + 0x1c9be989, 0xbe5e2b0a, 0xa0000000, 0x3fe8e6b9, 0x3c93d76a, + 0x3e5c8618, 0xe0000000, 0x3fe8d909, 0x2182fc9a, 0xbe41aa9e, + 0x20000000, 0x3fe8cb5e, 0xe6b3539d, 0xbe530d19, 0x60000000, + 0x3fe8bdb6, 0x49e58cc3, 0xbe3bb374, 0xa0000000, 0x3fe8b012, + 0xa7cfeb8f, 0x3e56c412, 0x00000000, 0x3fe8a273, 0x8d52bc19, + 0x3e1429b8, 0x60000000, 0x3fe894d7, 0x4dc32c6c, 0xbe48604c, + 0xc0000000, 0x3fe8873f, 0x0c868e56, 0xbe564ee5, 0x00000000, + 0x3fe879ac, 0x56aee828, 0x3e5e2fd8, 0x60000000, 0x3fe86c1c, + 0x7ceab8ec, 0x3e493365, 0xc0000000, 0x3fe85e90, 0x78d4dadc, + 0xbe4f7f25, 0x00000000, 0x3fe85109, 0x0ccd8280, 0x3e31e7a2, + 0x40000000, 0x3fe84385, 0x34ba4e15, 0x3e328077, 0x80000000, + 0x3fe83605, 0xa670975a, 0xbe53eee5, 0xa0000000, 0x3fe82889, + 0xf61b77b2, 0xbe43a20a, 0xa0000000, 0x3fe81b11, 0x13e6643b, + 0x3e5e5fe5, 0xc0000000, 0x3fe80d9d, 0x82cc94e8, 0xbe5ff1f9, + 0xa0000000, 0x3fe8002d, 0x8a0c9c5d, 0xbe42b0e7, 0x60000000, + 0x3fe7f2c1, 0x22a16f01, 0x3e5d9ea0, 0x20000000, 0x3fe7e559, + 0xc38cd451, 0x3e506963, 0xc0000000, 0x3fe7d7f4, 0x9902bc71, + 0x3e4503d7, 0x40000000, 0x3fe7ca94, 0xdef2a3c0, 0x3e3d98ed, + 0xa0000000, 0x3fe7bd37, 0xed49abb0, 0x3e24c1ff, 0xe0000000, + 0x3fe7afde, 0xe3b0be70, 0xbe40c467, 0x00000000, 0x3fe7a28a, + 0xaf9f193c, 0xbe5dff6c, 0xe0000000, 0x3fe79538, 0xb74cf6b6, + 0xbe258ed0, 0xa0000000, 0x3fe787eb, 0x1d9127c7, 0x3e345fb0, + 0x40000000, 0x3fe77aa2, 0x1028c21d, 0xbe4619bd, 0xa0000000, + 0x3fe76d5c, 0x7cb0b5e4, 0x3e40f1a2, 0xe0000000, 0x3fe7601a, + 0x2b1bc4ad, 0xbe32e8bb, 0xe0000000, 0x3fe752dc, 0x6839f64e, + 0x3e41f57b, 0xc0000000, 0x3fe745a2, 0xc4121f7e, 0xbe52c40a, + 0x60000000, 0x3fe7386c, 0xd6852d72, 0xbe5c4e6b, 0xc0000000, + 0x3fe72b39, 0x91d690f7, 0xbe57f88f, 0xe0000000, 0x3fe71e0a, + 0x627a2159, 0xbe4425d5, 0xc0000000, 0x3fe710df, 0x50a54033, + 0x3e422b7e, 0x60000000, 0x3fe703b8, 0x3b0b5f91, 0x3e5d3857, + 0xe0000000, 0x3fe6f694, 0x84d628a2, 0xbe51f090, 0x00000000, + 0x3fe6e975, 0x306d8894, 0xbe414d83, 0xe0000000, 0x3fe6dc58, + 0x30bf24aa, 0xbe4650ca, 0x80000000, 0x3fe6cf40, 0xd4628d69, + 0xbe5db007, 0xc0000000, 0x3fe6c22b, 0xa2aae57b, 0xbe31d279, + 0xc0000000, 0x3fe6b51a, 0x860edf7e, 0xbe2d4c4a, 0x80000000, + 0x3fe6a80d, 0xf3559341, 0xbe5f7e98, 0xe0000000, 0x3fe69b03, + 0xa885899e, 0xbe5c2011, 0xe0000000, 0x3fe68dfd, 0x2bdc6d37, + 0x3e224a82, 0xa0000000, 0x3fe680fb, 0xc12ad1b9, 0xbe40cf56, + 0x00000000, 0x3fe673fd, 0x1bcdf659, 0xbdf52f2d, 0x00000000, + 0x3fe66702, 0x5df10408, 0x3e5663e0, 0xc0000000, 0x3fe65a0a, + 0xa4070568, 0xbe40b12f, 0x00000000, 0x3fe64d17, 0x71c54c47, + 0x3e5f5e8b, 0x00000000, 0x3fe64027, 0xbd4b7e83, 0x3e42ead6, + 0xa0000000, 0x3fe6333a, 0x61598bd2, 0xbe4c48d4, 0xc0000000, + 0x3fe62651, 0x6f538d61, 0x3e548401, 0xa0000000, 0x3fe6196c, + 0x14344120, 0xbe529af6, 0x00000000, 0x3fe60c8b, 0x5982c587, + 0xbe3e1e4f, 0x00000000, 0x3fe5ffad, 0xfe51d4ea, 0xbe4c897a, + 0x80000000, 0x3fe5f2d2, 0xfd46ebe1, 0x3e552e00, 0xa0000000, + 0x3fe5e5fb, 0xa4695699, 0x3e5ed471, 0x60000000, 0x3fe5d928, + 0x80d118ae, 0x3e456b61, 0xa0000000, 0x3fe5cc58, 0x304c330b, + 0x3e54dc29, 0x80000000, 0x3fe5bf8c, 0x0af2dedf, 0xbe3aa9bd, + 0xe0000000, 0x3fe5b2c3, 0x15fc9258, 0xbe479a37, 0xc0000000, + 0x3fe5a5fe, 0x9292c7ea, 0x3e188650, 0x20000000, 0x3fe5993d, + 0x33b4d380, 0x3e5d6d93, 0x20000000, 0x3fe58c7f, 0x02fd16c7, + 0x3e2fe961, 0xa0000000, 0x3fe57fc4, 0x4a05edb6, 0xbe4d55b4, + 0xa0000000, 0x3fe5730d, 0x3d443abb, 0xbe5e6954, 0x00000000, + 0x3fe5665a, 0x024acfea, 0x3e50e61b, 0x00000000, 0x3fe559aa, + 0xcc9edd09, 0xbe325403, 0x60000000, 0x3fe54cfd, 0x1fe26950, + 0x3e5d500e, 0x60000000, 0x3fe54054, 0x6c5ae164, 0xbe4a79b4, + 0xc0000000, 0x3fe533ae, 0x154b0287, 0xbe401571, 0xa0000000, + 0x3fe5270c, 0x0673f401, 0xbe56e56b, 0xe0000000, 0x3fe51a6d, + 0x751b639c, 0x3e235269, 0xa0000000, 0x3fe50dd2, 0x7c7b2bed, + 0x3ddec887, 0xc0000000, 0x3fe5013a, 0xafab4e17, 0x3e5e7575, + 0x60000000, 0x3fe4f4a6, 0x2e308668, 0x3e59aed6, 0x80000000, + 0x3fe4e815, 0xf33e2a76, 0xbe51f184, 0xe0000000, 0x3fe4db87, + 0x839f3e3e, 0x3e57db01, 0xc0000000, 0x3fe4cefd, 0xa9eda7bb, + 0x3e535e0f, 0x00000000, 0x3fe4c277, 0x2a8f66a5, 0x3e5ce451, + 0xc0000000, 0x3fe4b5f3, 0x05192456, 0xbe4e8518, 0xc0000000, + 0x3fe4a973, 0x4aa7cd1d, 0x3e46784a, 0x40000000, 0x3fe49cf7, + 0x8e23025e, 0xbe5749f2, 0x00000000, 0x3fe4907e, 0x18d30215, + 0x3e360f39, 0x20000000, 0x3fe48408, 0x63dcf2f3, 0x3e5e00fe, + 0xc0000000, 0x3fe47795, 0x46182d09, 0xbe5173d9, 0xa0000000, + 0x3fe46b26, 0x8f0e62aa, 0xbe48f281, 0xe0000000, 0x3fe45eba, + 0x5775c40c, 0xbe56aad4, 0x60000000, 0x3fe45252, 0x0fe25f69, + 0x3e48bd71, 0x40000000, 0x3fe445ed, 0xe9989ec5, 0x3e590d97, + 0x80000000, 0x3fe4398b, 0xb3d9ffe3, 0x3e479dbc, 0x20000000, + 0x3fe42d2d, 0x388e4d2e, 0xbe5eed80, 0xe0000000, 0x3fe420d1, + 0x6f797c18, 0x3e554b4c, 0x20000000, 0x3fe4147a, 0x31048bb4, + 0xbe5b1112, 0x80000000, 0x3fe40825, 0x2efba4f9, 0x3e48ebc7, + 0x40000000, 0x3fe3fbd4, 0x50201119, 0x3e40b701, 0x40000000, + 0x3fe3ef86, 0x0a4db32c, 0x3e551de8, 0xa0000000, 0x3fe3e33b, + 0x0c9c148b, 0xbe50c1f6, 0x20000000, 0x3fe3d6f4, 0xc9129447, + 0x3e533fa0, 0x00000000, 0x3fe3cab0, 0xaae5b5a0, 0xbe22b68e, + 0x20000000, 0x3fe3be6f, 0x02305e8a, 0xbe54fc08, 0x60000000, + 0x3fe3b231, 0x7f908258, 0x3e57dc05, 0x00000000, 0x3fe3a5f7, + 0x1a09af78, 0x3e08038b, 0xe0000000, 0x3fe399bf, 0x490643c1, + 0xbe5dbe42, 0xe0000000, 0x3fe38d8b, 0x5e8ad724, 0xbe3c2b72, + 0x20000000, 0x3fe3815b, 0xc67196b6, 0x3e1713cf, 0xa0000000, + 0x3fe3752d, 0x6182e429, 0xbe3ec14c, 0x40000000, 0x3fe36903, + 0xab6eb1ae, 0x3e5a2cc5, 0x40000000, 0x3fe35cdc, 0xfe5dc064, + 0xbe5c5878, 0x40000000, 0x3fe350b8, 0x0ba6b9e4, 0x3e51619b, + 0x80000000, 0x3fe34497, 0x857761aa, 0x3e5fff53, 0x00000000, + 0x3fe3387a, 0xf872d68c, 0x3e484f4d, 0xa0000000, 0x3fe32c5f, + 0x087e97c2, 0x3e52842e, 0x80000000, 0x3fe32048, 0x73d6d0c0, + 0xbe503edf, 0x80000000, 0x3fe31434, 0x0c1456a1, 0xbe5f72ad, + 0xa0000000, 0x3fe30823, 0x83a1a4d5, 0xbe5e65cc, 0xe0000000, + 0x3fe2fc15, 0x855a7390, 0xbe506438, 0x40000000, 0x3fe2f00b, + 0xa2898287, 0x3e3d22a2, 0xe0000000, 0x3fe2e403, 0x8b56f66f, + 0xbe5aa5fd, 0x80000000, 0x3fe2d7ff, 0x52db119a, 0x3e3a2e3d, + 0x60000000, 0x3fe2cbfe, 0xe2ddd4c0, 0xbe586469, 0x40000000, + 0x3fe2c000, 0x6b01bf10, 0x3e352b9d, 0x40000000, 0x3fe2b405, + 0xb07a1cdf, 0x3e5c5cda, 0x80000000, 0x3fe2a80d, 0xc7b5f868, + 0xbe5668b3, 0xc0000000, 0x3fe29c18, 0x185edf62, 0xbe563d66, + 0x00000000, 0x3fe29027, 0xf729e1cc, 0x3e59a9a0, 0x80000000, + 0x3fe28438, 0x6433c727, 0xbe43cc89, 0x00000000, 0x3fe2784d, + 0x41782631, 0xbe30750c, 0xa0000000, 0x3fe26c64, 0x914911b7, + 0xbe58290e, 0x40000000, 0x3fe2607f, 0x3dcc73e1, 0xbe4269cd, + 0x00000000, 0x3fe2549d, 0x2751bf70, 0xbe5a6998, 0xc0000000, + 0x3fe248bd, 0x4248b9fb, 0xbe4ddb00, 0x80000000, 0x3fe23ce1, + 0xf35cf82f, 0x3e561b71, 0x60000000, 0x3fe23108, 0x8e481a2d, + 0x3e518fb9, 0x60000000, 0x3fe22532, 0x5ab96edc, 0xbe5fafc5, + 0x40000000, 0x3fe2195f, 0x80943911, 0xbe07f819, 0x40000000, + 0x3fe20d8f, 0x386f2d6c, 0xbe54ba8b, 0x40000000, 0x3fe201c2, + 0xf29664ac, 0xbe5eb815, 0x20000000, 0x3fe1f5f8, 0x64f03390, + 0x3e5e320c, 0x20000000, 0x3fe1ea31, 0x747ff696, 0x3e5ef0a5, + 0x40000000, 0x3fe1de6d, 0x3e9ceb51, 0xbe5f8d27, 0x20000000, + 0x3fe1d2ac, 0x4ae0b55e, 0x3e5faa21, 0x20000000, 0x3fe1c6ee, + 0x28569a5e, 0x3e598a4f, 0x20000000, 0x3fe1bb33, 0x54b33e07, + 0x3e46130a, 0x20000000, 0x3fe1af7b, 0x024f1078, 0xbe4dbf93, + 0x00000000, 0x3fe1a3c6, 0xb0783bfa, 0x3e419248, 0xe0000000, + 0x3fe19813, 0x2f02b836, 0x3e4e02b7, 0xc0000000, 0x3fe18c64, + 0x28dec9d4, 0x3e09064f, 0x80000000, 0x3fe180b8, 0x45cbf406, + 0x3e5b1f46, 0x40000000, 0x3fe1750f, 0x03d9964c, 0x3e5b0a79, + 0x00000000, 0x3fe16969, 0x8b5b882b, 0xbe238086, 0xa0000000, + 0x3fe15dc5, 0x73bad6f8, 0xbdf1fca4, 0x20000000, 0x3fe15225, + 0x5385769c, 0x3e5e8d76, 0xa0000000, 0x3fe14687, 0x1676dc6b, + 0x3e571d08, 0x20000000, 0x3fe13aed, 0xa8c41c7f, 0xbe598a25, + 0x60000000, 0x3fe12f55, 0xc4e1aaf0, 0x3e435277, 0xa0000000, + 0x3fe123c0, 0x403638e1, 0xbe21aa7c, 0xc0000000, 0x3fe1182e, + 0x557a092b, 0xbdd0116b, 0xc0000000, 0x3fe10c9f, 0x7d779f66, + 0x3e4a61ba, 0xc0000000, 0x3fe10113, 0x2b09c645, 0xbe5d586e, + 0x20000000, 0x3fe0ea04, 0xea2cad46, 0x3e5aa97c, 0x20000000, + 0x3fe0d300, 0x23190e54, 0x3e50f1a7, 0xa0000000, 0x3fe0bc07, + 0x1379a5a6, 0xbe51619d, 0x60000000, 0x3fe0a51a, 0x926a3d4a, + 0x3e5cf019, 0xa0000000, 0x3fe08e38, 0xa8c24358, 0x3e35241e, + 0x20000000, 0x3fe07762, 0x24317e7a, 0x3e512cfa, 0x00000000, + 0x3fe06097, 0xfd9cf274, 0xbe55bef3, 0x00000000, 0x3fe049d7, + 0x3689b49d, 0xbe36d26d, 0x40000000, 0x3fe03322, 0xf72ef6c4, + 0xbe54cd08, 0xa0000000, 0x3fe01c78, 0x23702d2d, 0xbe5900bf, + 0x00000000, 0x3fe005da, 0x3f59c14c, 0x3e57d80b, 0x40000000, + 0x3fdfde8d, 0xad67766d, 0xbe57fad4, 0x40000000, 0x3fdfb17c, + 0x644f4ae7, 0x3e1ee43b, 0x40000000, 0x3fdf8481, 0x903234d2, + 0x3e501a86, 0x40000000, 0x3fdf579c, 0xafe9e509, 0xbe267c3e, + 0x00000000, 0x3fdf2acd, 0xb7dfda0b, 0xbe48149b, 0x40000000, + 0x3fdefe13, 0x3b94305e, 0x3e5f4ea7, 0x80000000, 0x3fded16f, + 0x5d95da61, 0xbe55c198, 0x00000000, 0x3fdea4e1, 0x406960c9, + 0xbdd99a19, 0x00000000, 0x3fde7868, 0xd22f3539, 0x3e470c78, + 0x80000000, 0x3fde4c04, 0x83eec535, 0xbe3e1232, 0x40000000, + 0x3fde1fb6, 0x3dfbffcb, 0xbe4b7d71, 0x40000000, 0x3fddf37d, + 0x7e1be4e0, 0xbe5b8f8f, 0x40000000, 0x3fddc759, 0x46dae887, + 0xbe350458, 0x80000000, 0x3fdd9b4a, 0xed6ecc49, 0xbe5f0045, + 0x80000000, 0x3fdd6f50, 0x2e9e883c, 0x3e2915da, 0x80000000, + 0x3fdd436b, 0xf0bccb32, 0x3e4a68c9, 0x80000000, 0x3fdd179b, + 0x9bbfc779, 0xbe54a26a, 0x00000000, 0x3fdcebe0, 0x7cea33ab, + 0x3e43c6b7, 0x40000000, 0x3fdcc039, 0xe740fd06, 0x3e5526c2, + 0x40000000, 0x3fdc94a7, 0x9eadeb1a, 0xbe396d8d, 0xc0000000, + 0x3fdc6929, 0xf0a8f95a, 0xbe5c0ab2, 0x80000000, 0x3fdc3dc0, + 0x6ee2693b, 0x3e0992e6, 0xc0000000, 0x3fdc126b, 0x5ac6b581, + 0xbe2834b6, 0x40000000, 0x3fdbe72b, 0x8cc226ff, 0x3e3596a6, + 0x00000000, 0x3fdbbbff, 0xf92a74bb, 0x3e3c5813, 0x00000000, + 0x3fdb90e7, 0x479664c0, 0xbe50d644, 0x00000000, 0x3fdb65e3, + 0x5004975b, 0xbe55258f, 0x00000000, 0x3fdb3af3, 0xe4b23194, + 0xbe588407, 0xc0000000, 0x3fdb1016, 0xe65d4d0a, 0x3e527c26, + 0x80000000, 0x3fdae54e, 0x814fddd6, 0x3e5962a2, 0x40000000, + 0x3fdaba9a, 0xe19d0913, 0xbe562f4e, 0x80000000, 0x3fda8ff9, + 0x43cfd006, 0xbe4cfdeb, 0x40000000, 0x3fda656c, 0x686f0a4e, + 0x3e5e47a8, 0xc0000000, 0x3fda3af2, 0x7200d410, 0x3e5e1199, + 0xc0000000, 0x3fda108c, 0xabd2266e, 0x3e5ee4d1, 0x40000000, + 0x3fd9e63a, 0x396f8f2c, 0x3e4dbffb, 0x00000000, 0x3fd9bbfb, + 0xe32b25dd, 0x3e5c3a54, 0x40000000, 0x3fd991cf, 0x431e4035, + 0xbe457925, 0x80000000, 0x3fd967b6, 0x7bed3dd3, 0x3e40c61d, + 0x00000000, 0x3fd93db1, 0xd7449365, 0x3e306419, 0x80000000, + 0x3fd913be, 0x1746e791, 0x3e56fcfc, 0x40000000, 0x3fd8e9df, + 0xf3a9028b, 0xbe5041b9, 0xc0000000, 0x3fd8c012, 0x56840c50, + 0xbe26e20a, 0x40000000, 0x3fd89659, 0x19763102, 0xbe51f466, + 0x80000000, 0x3fd86cb2, 0x7032de7c, 0xbe4d298a, 0x80000000, + 0x3fd8431e, 0xdeb39fab, 0xbe4361eb, 0x40000000, 0x3fd8199d, + 0x5d01cbe0, 0xbe5425b3, 0x80000000, 0x3fd7f02e, 0x3ce99aa9, + 0x3e146fa8, 0x80000000, 0x3fd7c6d2, 0xd1a262b9, 0xbe5a1a69, + 0xc0000000, 0x3fd79d88, 0x8606c236, 0x3e423a08, 0x80000000, + 0x3fd77451, 0x8fd1e1b7, 0x3e5a6a63, 0xc0000000, 0x3fd74b2c, + 0xe491456a, 0x3e42c1ca, 0x40000000, 0x3fd7221a, 0x4499a6d7, + 0x3e36a69a, 0x00000000, 0x3fd6f91a, 0x5237df94, 0xbe0f8f02, + 0x00000000, 0x3fd6d02c, 0xb6482c6e, 0xbe5abcf7, 0x00000000, + 0x3fd6a750, 0x1919fd61, 0xbe57ade2, 0x00000000, 0x3fd67e86, + 0xaa7a994d, 0xbe3f3fbd, 0x00000000, 0x3fd655ce, 0x67db014c, + 0x3e33c550, 0x00000000, 0x3fd62d28, 0xa82856b7, 0xbe1409d1, + 0xc0000000, 0x3fd60493, 0x1e6a300d, 0x3e55d899, 0x80000000, + 0x3fd5dc11, 0x1222bd5c, 0xbe35bfc0, 0xc0000000, 0x3fd5b3a0, + 0x6e8dc2d3, 0x3e5d4d79, 0x00000000, 0x3fd58b42, 0xe0e4ace6, + 0xbe517303, 0x80000000, 0x3fd562f4, 0xb306e0a8, 0x3e5edf0f, + 0xc0000000, 0x3fd53ab8, 0x6574bc54, 0x3e5ee859, 0x80000000, + 0x3fd5128e, 0xea902207, 0x3e5f6188, 0xc0000000, 0x3fd4ea75, + 0x9f911d79, 0x3e511735, 0x80000000, 0x3fd4c26e, 0xf9c77397, + 0xbe5b1643, 0x40000000, 0x3fd49a78, 0x15fc9258, 0x3e479a37, + 0x80000000, 0x3fd47293, 0xd5a04dd9, 0xbe426e56, 0xc0000000, + 0x3fd44abf, 0xe04042f5, 0x3e56f7c6, 0x40000000, 0x3fd422fd, + 0x1d8bf2c8, 0x3e5d8810, 0x00000000, 0x3fd3fb4c, 0x88a8ddee, + 0xbe311454, 0xc0000000, 0x3fd3d3ab, 0x3e3b5e47, 0xbe5d1b72, + 0x40000000, 0x3fd3ac1c, 0xc2ab5d59, 0x3e31b02b, 0xc0000000, + 0x3fd3849d, 0xd4e34b9e, 0x3e51cb2f, 0x40000000, 0x3fd35d30, + 0x177204fb, 0xbe2b8cd7, 0x80000000, 0x3fd335d3, 0xfcd38c82, + 0xbe4356e1, 0x80000000, 0x3fd30e87, 0x64f54acc, 0xbe4e6224, + 0x00000000, 0x3fd2e74c, 0xaa7975d9, 0x3e5dc0fe, 0x80000000, + 0x3fd2c021, 0x516dab3f, 0xbe50ffa3, 0x40000000, 0x3fd29907, + 0x2bfb7313, 0x3e5674a2, 0xc0000000, 0x3fd271fd, 0x0549fc99, + 0x3e385d29, 0xc0000000, 0x3fd24b04, 0x55b63073, 0xbe500c6d, + 0x00000000, 0x3fd2241c, 0x3f91953a, 0x3e389977, 0xc0000000, + 0x3fd1fd43, 0xa1543f71, 0xbe3487ab, 0xc0000000, 0x3fd1d67b, + 0x4ec8867c, 0x3df6a2dc, 0x00000000, 0x3fd1afc4, 0x4328e3bb, + 0x3e41d9c0, 0x80000000, 0x3fd1891c, 0x2e1cda84, 0x3e3bdd87, + 0x40000000, 0x3fd16285, 0x4b5331ae, 0xbe53128e, 0x00000000, + 0x3fd13bfe, 0xb9aec164, 0xbe52ac98, 0xc0000000, 0x3fd11586, + 0xd91e1316, 0xbe350630, 0x80000000, 0x3fd0ef1f, 0x7cacc12c, + 0x3e3f5219, 0x40000000, 0x3fd0c8c8, 0xbce277b7, 0x3e3d30c0, + 0x00000000, 0x3fd0a281, 0x2a63447d, 0xbe541377, 0x80000000, + 0x3fd07c49, 0xfac483b5, 0xbe5772ec, 0xc0000000, 0x3fd05621, + 0x36b8a570, 0xbe4fd4bd, 0xc0000000, 0x3fd03009, 0xbae505f7, + 0xbe450388, 0x80000000, 0x3fd00a01, 0x3e35aead, 0xbe5430fc, + 0x80000000, 0x3fcfc811, 0x707475ac, 0x3e38806e, 0x80000000, + 0x3fcf7c3f, 0xc91817fc, 0xbe40ccea, 0x80000000, 0x3fcf308c, + 0xae05d5e9, 0xbe4919b8, 0x80000000, 0x3fcee4f8, 0xae6cc9e6, + 0xbe530b94, 0x00000000, 0x3fce9983, 0x1efe3e8e, 0x3e57747e, + 0x00000000, 0x3fce4e2d, 0xda78d9bf, 0xbe59a608, 0x00000000, + 0x3fce02f5, 0x8abe2c2e, 0x3e4a35ad, 0x00000000, 0x3fcdb7dc, + 0x1495450d, 0xbe0872cc, 0x80000000, 0x3fcd6ce1, 0x86ee0ba0, + 0xbe4f59a0, 0x00000000, 0x3fcd2205, 0xe81ca888, 0x3e5402c3, + 0x00000000, 0x3fccd747, 0x3b4424b9, 0x3e5dfdc3, 0x80000000, + 0x3fcc8ca7, 0xd305b56c, 0x3e202da6, 0x00000000, 0x3fcc4226, + 0x399a6910, 0xbe482a1c, 0x80000000, 0x3fcbf7c2, 0x747f7938, + 0xbe587372, 0x80000000, 0x3fcbad7c, 0x6fc246a0, 0x3e50d83d, + 0x00000000, 0x3fcb6355, 0xee9e9be5, 0xbe5c35bd, 0x80000000, + 0x3fcb194a, 0x8416c0bc, 0x3e546d4f, 0x00000000, 0x3fcacf5e, + 0x49f7f08f, 0x3e56da76, 0x00000000, 0x3fca858f, 0x5dc30de2, + 0x3e5f390c, 0x00000000, 0x3fca3bde, 0x950583b6, 0xbe5e4169, + 0x80000000, 0x3fc9f249, 0x33631553, 0x3e52aeb1, 0x00000000, + 0x3fc9a8d3, 0xde8795a6, 0xbe59a504, 0x00000000, 0x3fc95f79, + 0x076bf41e, 0x3e5122fe, 0x80000000, 0x3fc9163c, 0x2914c8e7, + 0x3e3dd064, 0x00000000, 0x3fc8cd1d, 0x3a30eca3, 0xbe21b4aa, + 0x80000000, 0x3fc8841a, 0xb2a96650, 0xbe575444, 0x80000000, + 0x3fc83b34, 0x2376c0cb, 0xbe2a74c7, 0x80000000, 0x3fc7f26b, + 0xd8a0b653, 0xbe5181b6, 0x00000000, 0x3fc7a9bf, 0x32257882, + 0xbe4a78b4, 0x00000000, 0x3fc7612f, 0x1eee8bd9, 0xbe1bfe9d, + 0x80000000, 0x3fc718bb, 0x0c603cc4, 0x3e36fdc9, 0x80000000, + 0x3fc6d064, 0x3728b8cf, 0xbe1e542e, 0x80000000, 0x3fc68829, + 0xc79a4067, 0x3e5c380f, 0x00000000, 0x3fc6400b, 0xf69eac69, + 0x3e550a84, 0x80000000, 0x3fc5f808, 0xb7a780a4, 0x3e5d9224, + 0x80000000, 0x3fc5b022, 0xad9dfb1e, 0xbe55242f, 0x00000000, + 0x3fc56858, 0x659b18be, 0xbe4bfda3, 0x80000000, 0x3fc520a9, + 0x66ee3631, 0xbe57d769, 0x80000000, 0x3fc4d916, 0x1ec62819, + 0x3e2427f7, 0x80000000, 0x3fc4919f, 0xdec25369, 0xbe435431, + 0x00000000, 0x3fc44a44, 0xa8acfc4b, 0xbe3c62e8, 0x00000000, + 0x3fc40304, 0xcf1d3eab, 0xbdfba29f, 0x80000000, 0x3fc3bbdf, + 0x79aba3ea, 0xbdf1b7c8, 0x80000000, 0x3fc374d6, 0xb8d186da, + 0xbe5130cf, 0x80000000, 0x3fc32de8, 0x9d74f152, 0x3e2285b6, + 0x00000000, 0x3fc2e716, 0x50ae7ca9, 0xbe503920, 0x80000000, + 0x3fc2a05e, 0x6caed92e, 0xbe533924, 0x00000000, 0x3fc259c2, + 0x9cb5034e, 0xbe510e31, 0x80000000, 0x3fc21340, 0x12c4d378, + 0xbe540b43, 0x80000000, 0x3fc1ccd9, 0xcc418706, 0x3e59887a, + 0x00000000, 0x3fc1868e, 0x921f4106, 0xbe528e67, 0x80000000, + 0x3fc1405c, 0x3969441e, 0x3e5d8051, 0x00000000, 0x3fc0fa46, + 0xd941ef5b, 0x3e5f9079, 0x80000000, 0x3fc0b44a, 0x5a3e81b2, + 0xbe567691, 0x00000000, 0x3fc06e69, 0x9d66afe7, 0xbe4d43fb, + 0x00000000, 0x3fc028a2, 0x0a92a162, 0xbe52f394, 0x00000000, + 0x3fbfc5ea, 0x209897e5, 0x3e529e37, 0x00000000, 0x3fbf3ac5, + 0x8458bd7b, 0x3e582831, 0x00000000, 0x3fbeafd5, 0xb8d8b4b8, + 0xbe486b4a, 0x00000000, 0x3fbe2518, 0xe0a3b7b6, 0x3e5bafd2, + 0x00000000, 0x3fbd9a90, 0x2bf2710e, 0x3e383b2b, 0x00000000, + 0x3fbd103c, 0x73eb6ab7, 0xbe56d78d, 0x00000000, 0x3fbc861b, + 0x32ceaff5, 0xbe32dc5a, 0x00000000, 0x3fbbfc2e, 0xbee04cb7, + 0xbe4a71a4, 0x00000000, 0x3fbb7274, 0x35ae9577, 0x3e38142f, + 0x00000000, 0x3fbae8ee, 0xcbaddab4, 0xbe5490f0, 0x00000000, + 0x3fba5f9a, 0x95ce1114, 0x3e597c71, 0x00000000, 0x3fb9d67a, + 0x6d7c0f78, 0x3e3abc2d, 0x00000000, 0x3fb94d8d, 0x2841a782, + 0xbe566cbc, 0x00000000, 0x3fb8c4d2, 0x6ed429c6, 0xbe3cfff9, + 0x00000000, 0x3fb83c4a, 0xe4a49fbb, 0xbe552964, 0x00000000, + 0x3fb7b3f4, 0x2193d81e, 0xbe42fa72, 0x00000000, 0x3fb72bd0, + 0xdd70c122, 0x3e527a8c, 0x00000000, 0x3fb6a3df, 0x03108a54, + 0xbe450393, 0x00000000, 0x3fb61c1f, 0x30ff7954, 0x3e565840, + 0x00000000, 0x3fb59492, 0xdedd460c, 0xbe5422b5, 0x00000000, + 0x3fb50d36, 0x950f9f45, 0xbe5313f6, 0x00000000, 0x3fb4860b, + 0x582cdcb1, 0x3e506d39, 0x00000000, 0x3fb3ff12, 0x7216d3a6, + 0x3e4aa719, 0x00000000, 0x3fb3784a, 0x57a423fd, 0x3e5a9b9f, + 0x00000000, 0x3fb2f1b4, 0x7a138b41, 0xbe50b418, 0x00000000, + 0x3fb26b4e, 0x2fbfd7ea, 0x3e23a53e, 0x00000000, 0x3fb1e519, + 0x18913ccb, 0x3e465fc1, 0x00000000, 0x3fb15f15, 0x7ea24e21, + 0x3e042843, 0x00000000, 0x3fb0d941, 0x7c6d9c77, 0x3e59f61e, + 0x00000000, 0x3fb0539e, 0x114efd44, 0x3e4ccab7, 0x00000000, + 0x3faf9c56, 0x1777f657, 0x3e552f65, 0x00000000, 0x3fae91d2, + 0xc317b86a, 0xbe5a61e0, 0x00000000, 0x3fad87ac, 0xb7664efb, + 0xbe41f64e, 0x00000000, 0x3fac7de6, 0x5d3d03a9, 0x3e0807a0, + 0x00000000, 0x3fab7480, 0x743c38eb, 0xbe3726e1, 0x00000000, + 0x3faa6b78, 0x06a253f1, 0x3e5ad636, 0x00000000, 0x3fa962d0, + 0xa35f541b, 0x3e5a187a, 0x00000000, 0x3fa85a88, 0x4b86e446, + 0xbe508150, 0x00000000, 0x3fa7529c, 0x2589cacf, 0x3e52938a, + 0x00000000, 0x3fa64b10, 0xaf6b11f2, 0xbe3454cd, 0x00000000, + 0x3fa543e2, 0x97506fef, 0xbe5fdec5, 0x00000000, 0x3fa43d10, + 0xe75f7dd9, 0xbe388dd3, 0x00000000, 0x3fa3369c, 0xa4139632, + 0xbdea5177, 0x00000000, 0x3fa23086, 0x352d6f1e, 0xbe565ad6, + 0x00000000, 0x3fa12acc, 0x77449eb7, 0xbe50d5c7, 0x00000000, + 0x3fa0256e, 0x7478da78, 0x3e404724, 0x00000000, 0x3f9e40dc, + 0xf59cef7f, 0xbe539d0a, 0x00000000, 0x3f9c3790, 0x1511d43c, + 0x3e53c2c8, 0x00000000, 0x3f9a2f00, 0x9b8bff3c, 0xbe43b3e1, + 0x00000000, 0x3f982724, 0xad1e22a5, 0x3e46f0bd, 0x00000000, + 0x3f962000, 0x130d9356, 0x3e475ba0, 0x00000000, 0x3f941994, + 0x8f86f883, 0xbe513d0b, 0x00000000, 0x3f9213dc, 0x914d0dc8, + 0xbe534335, 0x00000000, 0x3f900ed8, 0x2d73e5e7, 0xbe22ba75, + 0x00000000, 0x3f8c1510, 0xc5b7d70e, 0x3e599c5d, 0x00000000, + 0x3f880de0, 0x8a27857e, 0xbe3d28c8, 0x00000000, 0x3f840810, + 0xda767328, 0x3e531b3d, 0x00000000, 0x3f8003b0, 0x77bacaf3, + 0xbe5f04e3, 0x00000000, 0x3f780150, 0xdf4b0720, 0x3e5a8bff, + 0x00000000, 0x3f6ffc40, 0x34c48e71, 0xbe3fcd99, 0x00000000, + 0x3f5ff6c0, 0x1ad218af, 0xbe4c78a7, 0x00000000, 0x00000000, + 0x00000000, 0x80000000 + //@formatter:on + }); + + private ArrayDataPointerConstant log2 = pointerConstant(8, new int[]{ + //@formatter:off + 0xfefa39ef, 0x3fe62e42, 0xfefa39ef, 0xbfe62e42 + //@formatter:on + }); + + private ArrayDataPointerConstant double2 = pointerConstant(8, new int[]{ + //@formatter:off + 0x00000000, 0x40000000 + //@formatter:on + }); + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + // registers, + // input: xmm0, xmm1 + // scratch: xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 + // rax, rdx, rcx, r8, r11 + + // Code generated by Intel C compiler for LIBM library + Label block0 = new Label(); + Label block1 = new Label(); + Label block2 = new Label(); + Label block3 = new Label(); + Label block4 = new Label(); + Label block5 = new Label(); + Label block6 = new Label(); + Label block7 = new Label(); + Label block8 = new Label(); + Label block9 = new Label(); + Label block10 = new Label(); + Label block11 = new Label(); + Label block12 = new Label(); + Label block13 = new Label(); + Label block14 = new Label(); + Label block15 = new Label(); + Label block16 = new Label(); + Label block17 = new Label(); + Label block18 = new Label(); + Label block19 = new Label(); + Label block20 = new Label(); + Label block21 = new Label(); + Label block22 = new Label(); + Label block23 = new Label(); + Label block24 = new Label(); + Label block25 = new Label(); + Label block26 = new Label(); + Label block27 = new Label(); + Label block28 = new Label(); + Label block29 = new Label(); + Label block30 = new Label(); + Label block31 = new Label(); + Label block32 = new Label(); + Label block33 = new Label(); + Label block34 = new Label(); + Label block35 = new Label(); + Label block36 = new Label(); + Label block37 = new Label(); + Label block38 = new Label(); + Label block39 = new Label(); + Label block40 = new Label(); + Label block41 = new Label(); + Label block42 = new Label(); + Label block43 = new Label(); + Label block44 = new Label(); + Label block45 = new Label(); + Label block46 = new Label(); + Label block47 = new Label(); + Label block48 = new Label(); + Label block49 = new Label(); + Label block50 = new Label(); + Label block51 = new Label(); + Label block52 = new Label(); + Label block53 = new Label(); + Label block54 = new Label(); + Label block55 = new Label(); + Label block56 = new Label(); + Label block57 = new Label(); + + Register tmp1 = r8; + Register tmp2 = r9; + Register tmp3 = r10; + Register tmp4 = r11; + + masm.subq(rsp, 40); + masm.movsd(new AMD64Address(rsp, 8), xmm0); + masm.movsd(new AMD64Address(rsp, 16), xmm1); + + // Special case: pow(x, 2.0) => x * x + masm.movdq(tmp1, xmm1); + masm.cmpq(tmp1, recordExternalAddress(crb, double2)); + masm.jccb(AMD64Assembler.ConditionFlag.NotEqual, block57); + masm.mulsd(xmm0, xmm0); + masm.jmp(block56); + + masm.bind(block57); + masm.pextrw(rax, xmm0, 3); + masm.xorpd(xmm2, xmm2); + masm.movq(tmp2, 0x3ff0000000000000L); + masm.movdq(xmm2, tmp2); + masm.movl(tmp1, 1069088768); + masm.movdq(xmm7, tmp1); + masm.xorpd(xmm1, xmm1); + masm.movq(tmp3, 0x77f0000000000000L); + masm.movdq(xmm1, tmp3); + masm.movdqu(xmm3, xmm0); + masm.movl(rdx, 32752); + masm.andl(rdx, rax); + masm.subl(rdx, 16368); + masm.movl(rcx, rdx); + masm.sarl(rdx, 31); + masm.addl(rcx, rdx); + masm.xorl(rcx, rdx); + masm.por(xmm0, xmm2); + masm.movdqu(xmm6, recordExternalAddress(crb, highsigmask)); // 0x00000000, 0xfffff800, + // 0x00000000, 0xfffff800 + masm.psrlq(xmm0, 27); + masm.movq(xmm2, recordExternalAddress(crb, log2E)); // 0x00000000, 0x3ff72000, + // 0x161bb241, 0xbf5dabe1 + masm.psrld(xmm0, 2); + masm.addl(rcx, 16); + masm.bsrl(rcx, rcx); + masm.rcpps(xmm0, xmm0); + masm.psllq(xmm3, 12); + masm.movl(tmp4, 8192); + masm.movdq(xmm4, tmp4); + masm.psrlq(xmm3, 12); + masm.subl(rax, 16); + masm.cmpl(rax, 32736); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block0); + masm.movl(tmp1, 0); + + masm.bind(block1); + masm.mulss(xmm0, xmm7); + masm.movl(rdx, -1); + masm.subl(rcx, 4); + masm.shll(rdx); + masm.shlq(rdx, 32); + masm.movdq(xmm5, rdx); + masm.por(xmm3, xmm1); + masm.subl(rax, 16351); + masm.cmpl(rax, 1); + masm.jcc(AMD64Assembler.ConditionFlag.BelowEqual, block2); + masm.paddd(xmm0, xmm4); + masm.pand(xmm5, xmm3); + masm.movdl(rdx, xmm0); + masm.psllq(xmm0, 29); + + masm.bind(block3); + masm.subsd(xmm3, xmm5); + masm.pand(xmm0, xmm6); + masm.subl(rax, 1); + masm.sarl(rax, 4); + masm.cvtsi2sdl(xmm7, rax); + masm.mulpd(xmm5, xmm0); + + masm.bind(block4); + masm.mulsd(xmm3, xmm0); + masm.movdqu(xmm1, recordExternalAddress(crb, coeff)); // 0x6dc96112, 0xbf836578, + // 0xee241472, 0xbf9b0301 + masm.leaq(tmp4, recordExternalAddress(crb, lTbl)); + masm.subsd(xmm5, xmm2); + masm.movdqu(xmm4, recordExternalAddress(crb, coeff16)); // 0x9f95985a, 0xbfb528db, + // 0xb3841d2a, 0xbfd619b6 + masm.movl(rcx, rax); + masm.sarl(rax, 31); + masm.addl(rcx, rax); + masm.xorl(rax, rcx); + masm.addl(rax, 1); + masm.bsrl(rax, rax); + masm.unpcklpd(xmm5, xmm3); + masm.movdqu(xmm6, recordExternalAddress(crb, coeff32)); // 0x518775e3, 0x3f9004f2, + // 0xac8349bb, 0x3fa76c9b + masm.addsd(xmm3, xmm5); + masm.andl(rdx, 16760832); + masm.shrl(rdx, 10); + masm.addpd(xmm5, new AMD64Address(tmp4, rdx, AMD64Address.Scale.Times1, -3648)); + masm.movdqu(xmm0, recordExternalAddress(crb, coeff48)); // 0x486ececc, 0x3fc4635e, + // 0x161bb241, 0xbf5dabe1 + masm.pshufd(xmm2, xmm3, 68); + masm.mulsd(xmm3, xmm3); + masm.mulpd(xmm1, xmm2); + masm.mulpd(xmm4, xmm2); + masm.addsd(xmm5, xmm7); + masm.mulsd(xmm2, xmm3); + masm.addpd(xmm6, xmm1); + masm.mulsd(xmm3, xmm3); + masm.addpd(xmm0, xmm4); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movw(rcx, new AMD64Address(rsp, 22)); + masm.pshufd(xmm7, xmm5, 238); + masm.movq(xmm4, recordExternalAddress(crb, highmaskY)); // 0x00000000, 0xfffffff8, + // 0x00000000, 0xffffffff + masm.mulpd(xmm6, xmm2); + masm.pshufd(xmm3, xmm3, 68); + masm.mulpd(xmm0, xmm2); + masm.shll(rax, 4); + masm.subl(rax, 15872); + masm.andl(rcx, 32752); + masm.addl(rax, rcx); + masm.mulpd(xmm3, xmm6); + masm.cmpl(rax, 624); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block5); + masm.xorpd(xmm6, xmm6); + masm.movl(rdx, 17080); + masm.pinsrw(xmm6, rdx, 3); + masm.movdqu(xmm2, xmm1); + masm.pand(xmm4, xmm1); + masm.subsd(xmm1, xmm4); + masm.mulsd(xmm4, xmm5); + masm.addsd(xmm0, xmm7); + masm.mulsd(xmm1, xmm5); + masm.movdqu(xmm7, xmm6); + masm.addsd(xmm6, xmm4); + masm.leaq(tmp4, recordExternalAddress(crb, tExp)); + masm.addpd(xmm3, xmm0); + masm.movdl(rdx, xmm6); + masm.subsd(xmm6, xmm7); + masm.pshufd(xmm0, xmm3, 238); + masm.subsd(xmm4, xmm6); + masm.addsd(xmm0, xmm3); + masm.movl(rcx, rdx); + masm.andl(rdx, 255); + masm.addl(rdx, rdx); + masm.movdqu(xmm5, new AMD64Address(tmp4, rdx, AMD64Address.Scale.Times8, 0)); + masm.addsd(xmm4, xmm1); + masm.mulsd(xmm2, xmm0); + masm.movdqu(xmm7, recordExternalAddress(crb, eCoeff)); // 0xe78a6731, 0x3f55d87f, + // 0xd704a0c0, 0x3fac6b08 + masm.movdqu(xmm3, recordExternalAddress(crb, eCoeff16)); // 0x6fba4e77, 0x3f83b2ab, + // 0xff82c58f, 0x3fcebfbd + masm.shll(rcx, 12); + masm.xorl(rcx, tmp1); + masm.andl(rcx, -1048576); + masm.movdq(xmm6, rcx); + masm.addsd(xmm2, xmm4); + masm.movq(tmp2, 0x3fe62e42fefa39efL); + masm.movdq(xmm1, tmp2); + masm.pshufd(xmm0, xmm2, 68); + masm.pshufd(xmm4, xmm2, 68); + masm.mulsd(xmm1, xmm2); + masm.pshufd(xmm6, xmm6, 17); + masm.mulpd(xmm0, xmm0); + masm.mulpd(xmm7, xmm4); + masm.paddd(xmm5, xmm6); + masm.mulsd(xmm1, xmm5); + masm.pshufd(xmm6, xmm5, 238); + masm.mulsd(xmm0, xmm0); + masm.addpd(xmm3, xmm7); + masm.addsd(xmm1, xmm6); + masm.mulpd(xmm0, xmm3); + masm.pshufd(xmm3, xmm0, 238); + masm.mulsd(xmm0, xmm5); + masm.mulsd(xmm3, xmm5); + masm.addsd(xmm0, xmm1); + masm.addsd(xmm0, xmm3); + masm.addsd(xmm0, xmm5); + masm.jmp(block56); + + masm.bind(block0); + masm.addl(rax, 16); + masm.movl(rdx, 32752); + masm.andl(rdx, rax); + masm.cmpl(rdx, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block6); + masm.testl(rax, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block7); + + masm.bind(block8); + masm.movq(xmm0, new AMD64Address(rsp, 8)); + masm.movq(xmm3, new AMD64Address(rsp, 8)); + masm.movdl(rdx, xmm3); + masm.psrlq(xmm3, 32); + masm.movdl(rcx, xmm3); + masm.orl(rdx, rcx); + masm.cmpl(rdx, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block9); + masm.xorpd(xmm3, xmm3); + masm.movl(rax, 18416); + masm.pinsrw(xmm3, rax, 3); + masm.mulsd(xmm0, xmm3); + masm.xorpd(xmm2, xmm2); + masm.movl(rax, 16368); + masm.pinsrw(xmm2, rax, 3); + masm.movdqu(xmm3, xmm0); + masm.pextrw(rax, xmm0, 3); + masm.por(xmm0, xmm2); + masm.movl(rcx, 18416); + masm.psrlq(xmm0, 27); + masm.movq(xmm2, recordExternalAddress(crb, log2E)); // 0x00000000, 0x3ff72000, + // 0x161bb241, 0xbf5dabe1 + masm.psrld(xmm0, 2); + masm.rcpps(xmm0, xmm0); + masm.psllq(xmm3, 12); + masm.movdqu(xmm6, recordExternalAddress(crb, highsigmask)); // 0x00000000, 0xfffff800, + // 0x00000000, 0xfffff800 + masm.psrlq(xmm3, 12); + masm.mulss(xmm0, xmm7); + masm.movl(rdx, -1024); + masm.movdl(xmm5, rdx); + masm.por(xmm3, xmm1); + masm.paddd(xmm0, xmm4); + masm.psllq(xmm5, 32); + masm.movdl(rdx, xmm0); + masm.psllq(xmm0, 29); + masm.pand(xmm5, xmm3); + masm.movl(tmp1, 0); + masm.pand(xmm0, xmm6); + masm.subsd(xmm3, xmm5); + masm.andl(rax, 32752); + masm.subl(rax, 18416); + masm.sarl(rax, 4); + masm.cvtsi2sdl(xmm7, rax); + masm.mulpd(xmm5, xmm0); + masm.jmp(block4); + + masm.bind(block10); + masm.movq(xmm0, new AMD64Address(rsp, 8)); + masm.movq(xmm3, new AMD64Address(rsp, 8)); + masm.movdl(rdx, xmm3); + masm.psrlq(xmm3, 32); + masm.movdl(rcx, xmm3); + masm.orl(rdx, rcx); + masm.cmpl(rdx, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block9); + masm.xorpd(xmm3, xmm3); + masm.movl(rax, 18416); + masm.pinsrw(xmm3, rax, 3); + masm.mulsd(xmm0, xmm3); + masm.xorpd(xmm2, xmm2); + masm.movl(rax, 16368); + masm.pinsrw(xmm2, rax, 3); + masm.movdqu(xmm3, xmm0); + masm.pextrw(rax, xmm0, 3); + masm.por(xmm0, xmm2); + masm.movl(rcx, 18416); + masm.psrlq(xmm0, 27); + masm.movq(xmm2, recordExternalAddress(crb, log2E)); // 0x00000000, 0x3ff72000, + // 0x161bb241, 0xbf5dabe1 + masm.psrld(xmm0, 2); + masm.rcpps(xmm0, xmm0); + masm.psllq(xmm3, 12); + masm.movdqu(xmm6, recordExternalAddress(crb, highsigmask)); // 0x00000000, 0xfffff800, + // 0x00000000, 0xfffff800 + masm.psrlq(xmm3, 12); + masm.mulss(xmm0, xmm7); + masm.movl(rdx, -1024); + masm.movdl(xmm5, rdx); + masm.por(xmm3, xmm1); + masm.paddd(xmm0, xmm4); + masm.psllq(xmm5, 32); + masm.movdl(rdx, xmm0); + masm.psllq(xmm0, 29); + masm.pand(xmm5, xmm3); + masm.movl(tmp1, Integer.MIN_VALUE); + masm.pand(xmm0, xmm6); + masm.subsd(xmm3, xmm5); + masm.andl(rax, 32752); + masm.subl(rax, 18416); + masm.sarl(rax, 4); + masm.cvtsi2sdl(xmm7, rax); + masm.mulpd(xmm5, xmm0); + masm.jmp(block4); + + masm.bind(block5); + masm.cmpl(rax, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Less, block11); + masm.cmpl(rax, 752); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block12); + masm.addsd(xmm0, xmm7); + masm.movq(xmm2, recordExternalAddress(crb, halfmask)); // 0xf8000000, 0xffffffff, + // 0xf8000000, 0xffffffff + masm.addpd(xmm3, xmm0); + masm.xorpd(xmm6, xmm6); + masm.movl(rax, 17080); + masm.pinsrw(xmm6, rax, 3); + masm.pshufd(xmm0, xmm3, 238); + masm.addsd(xmm0, xmm3); + masm.movdqu(xmm3, xmm5); + masm.addsd(xmm5, xmm0); + masm.movdqu(xmm4, xmm2); + masm.subsd(xmm3, xmm5); + masm.movdqu(xmm7, xmm5); + masm.pand(xmm5, xmm2); + masm.movdqu(xmm2, xmm1); + masm.pand(xmm4, xmm1); + masm.subsd(xmm7, xmm5); + masm.addsd(xmm0, xmm3); + masm.subsd(xmm1, xmm4); + masm.mulsd(xmm4, xmm5); + masm.addsd(xmm0, xmm7); + masm.mulsd(xmm2, xmm0); + masm.movdqu(xmm7, xmm6); + masm.mulsd(xmm1, xmm5); + masm.addsd(xmm6, xmm4); + masm.movdl(rax, xmm6); + masm.subsd(xmm6, xmm7); + masm.leaq(tmp4, recordExternalAddress(crb, tExp)); + masm.addsd(xmm2, xmm1); + masm.movdqu(xmm7, recordExternalAddress(crb, eCoeff)); // 0xe78a6731, 0x3f55d87f, + // 0xd704a0c0, 0x3fac6b08 + masm.movdqu(xmm3, recordExternalAddress(crb, eCoeff16)); // 0x6fba4e77, 0x3f83b2ab, + // 0xff82c58f, 0x3fcebfbd + masm.subsd(xmm4, xmm6); + masm.pextrw(rdx, xmm6, 3); + masm.movl(rcx, rax); + masm.andl(rax, 255); + masm.addl(rax, rax); + masm.movdqu(xmm5, new AMD64Address(tmp4, rax, AMD64Address.Scale.Times8, 0)); + masm.addsd(xmm2, xmm4); + masm.sarl(rcx, 8); + masm.movl(rax, rcx); + masm.sarl(rcx, 1); + masm.subl(rax, rcx); + masm.shll(rcx, 20); + masm.xorl(rcx, tmp1); + masm.movdl(xmm6, rcx); + masm.movq(xmm1, recordExternalAddress(crb, eCoeff32)); // 0xfefa39ef, 0x3fe62e42, + // 0x00000000, 0x00000000 + masm.andl(rdx, 32767); + masm.cmpl(rdx, 16529); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block12); + masm.pshufd(xmm0, xmm2, 68); + masm.pshufd(xmm4, xmm2, 68); + masm.mulpd(xmm0, xmm0); + masm.mulpd(xmm7, xmm4); + masm.pshufd(xmm6, xmm6, 17); + masm.mulsd(xmm1, xmm2); + masm.mulsd(xmm0, xmm0); + masm.paddd(xmm5, xmm6); + masm.addpd(xmm3, xmm7); + masm.mulsd(xmm1, xmm5); + masm.pshufd(xmm6, xmm5, 238); + masm.mulpd(xmm0, xmm3); + masm.addsd(xmm1, xmm6); + masm.pshufd(xmm3, xmm0, 238); + masm.mulsd(xmm0, xmm5); + masm.mulsd(xmm3, xmm5); + masm.shll(rax, 4); + masm.xorpd(xmm4, xmm4); + masm.addl(rax, 16368); + masm.pinsrw(xmm4, rax, 3); + masm.addsd(xmm0, xmm1); + masm.addsd(xmm0, xmm3); + masm.movdqu(xmm1, xmm0); + masm.addsd(xmm0, xmm5); + masm.mulsd(xmm0, xmm4); + masm.pextrw(rax, xmm0, 3); + masm.andl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block13); + masm.cmpl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block14); + masm.jmp(block56); + + masm.bind(block6); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movq(xmm0, new AMD64Address(rsp, 8)); + masm.movdqu(xmm2, xmm0); + masm.movdl(rax, xmm2); + masm.psrlq(xmm2, 20); + masm.movdl(rdx, xmm2); + masm.orl(rax, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block15); + masm.movdl(rax, xmm1); + masm.psrlq(xmm1, 32); + masm.movdl(rdx, xmm1); + masm.movl(rcx, rdx); + masm.addl(rdx, rdx); + masm.orl(rax, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block16); + masm.addsd(xmm0, xmm0); + masm.jmp(block56); + + masm.bind(block16); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 16368); + masm.pinsrw(xmm0, rax, 3); + masm.movl(new AMD64Address(rsp, 0), 29); + masm.jmp(block17); + + masm.bind(block18); + masm.movq(xmm0, new AMD64Address(rsp, 16)); + masm.addpd(xmm0, xmm0); + masm.jmp(block56); + + masm.bind(block15); + masm.movdl(rax, xmm1); + masm.movdqu(xmm2, xmm1); + masm.psrlq(xmm1, 32); + masm.movdl(rdx, xmm1); + masm.movl(rcx, rdx); + masm.addl(rdx, rdx); + masm.orl(rax, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block19); + masm.pextrw(rax, xmm2, 3); + masm.andl(rax, 32752); + masm.cmpl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block20); + masm.movdl(rax, xmm2); + masm.psrlq(xmm2, 20); + masm.movdl(rdx, xmm2); + masm.orl(rax, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block18); + + masm.bind(block20); + masm.pextrw(rax, xmm0, 3); + masm.testl(rax, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block21); + masm.testl(rcx, Integer.MIN_VALUE); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block22); + masm.jmp(block56); + + masm.bind(block23); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movdl(rax, xmm1); + masm.testl(rax, 1); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block24); + masm.testl(rax, 2); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block25); + masm.jmp(block24); + + masm.bind(block21); + masm.shrl(rcx, 20); + masm.andl(rcx, 2047); + masm.cmpl(rcx, 1075); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block24); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block26); + masm.cmpl(rcx, 1074); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block23); + masm.cmpl(rcx, 1023); + masm.jcc(AMD64Assembler.ConditionFlag.Below, block24); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movl(rax, 17208); + masm.xorpd(xmm3, xmm3); + masm.pinsrw(xmm3, rax, 3); + masm.movdqu(xmm4, xmm3); + masm.addsd(xmm3, xmm1); + masm.subsd(xmm4, xmm3); + masm.addsd(xmm1, xmm4); + masm.pextrw(rax, xmm1, 3); + masm.andl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block24); + masm.movdl(rax, xmm3); + masm.andl(rax, 1); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block24); + + masm.bind(block25); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.pextrw(rax, xmm1, 3); + masm.andl(rax, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block27); + masm.jmp(block56); + + masm.bind(block27); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 32768); + masm.pinsrw(xmm0, rax, 3); + masm.jmp(block56); + + masm.bind(block24); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.pextrw(rax, xmm1, 3); + masm.andl(rax, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block22); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 32752); + masm.pinsrw(xmm0, rax, 3); + masm.jmp(block56); + + masm.bind(block26); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movdl(rax, xmm1); + masm.andl(rax, 1); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block24); + masm.jmp(block25); + + masm.bind(block28); + masm.movdl(rax, xmm1); + masm.psrlq(xmm1, 20); + masm.movdl(rdx, xmm1); + masm.orl(rax, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block29); + masm.movq(xmm0, new AMD64Address(rsp, 16)); + masm.addsd(xmm0, xmm0); + masm.jmp(block56); + + masm.bind(block29); + masm.movq(xmm0, new AMD64Address(rsp, 8)); + masm.pextrw(rax, xmm0, 3); + masm.cmpl(rax, 49136); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block30); + masm.movdl(rcx, xmm0); + masm.psrlq(xmm0, 20); + masm.movdl(rdx, xmm0); + masm.orl(rcx, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block30); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 32760); + masm.pinsrw(xmm0, rax, 3); + masm.jmp(block56); + + masm.bind(block30); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.andl(rax, 32752); + masm.subl(rax, 16368); + masm.pextrw(rdx, xmm1, 3); + masm.xorpd(xmm0, xmm0); + masm.xorl(rax, rdx); + masm.andl(rax, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block31); + masm.jmp(block56); + + masm.bind(block31); + masm.movl(rcx, 32752); + masm.pinsrw(xmm0, rcx, 3); + masm.jmp(block56); + + masm.bind(block32); + masm.movdl(rax, xmm1); + masm.cmpl(rdx, 17184); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block33); + masm.testl(rax, 1); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block34); + masm.testl(rax, 2); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block35); + masm.jmp(block36); + + masm.bind(block33); + masm.testl(rax, 1); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block35); + masm.jmp(block36); + + masm.bind(block7); + masm.movq(xmm2, new AMD64Address(rsp, 8)); + masm.movdl(rax, xmm2); + masm.psrlq(xmm2, 31); + masm.movdl(rcx, xmm2); + masm.orl(rax, rcx); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block9); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.pextrw(rdx, xmm1, 3); + masm.movdl(rax, xmm1); + masm.movdqu(xmm2, xmm1); + masm.psrlq(xmm2, 32); + masm.movdl(rcx, xmm2); + masm.addl(rcx, rcx); + masm.orl(rcx, rax); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block37); + masm.andl(rdx, 32752); + masm.cmpl(rdx, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block28); + masm.cmpl(rdx, 17200); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block35); + masm.cmpl(rdx, 17184); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block32); + masm.cmpl(rdx, 16368); + masm.jcc(AMD64Assembler.ConditionFlag.Below, block34); + masm.movl(rax, 17208); + masm.xorpd(xmm2, xmm2); + masm.pinsrw(xmm2, rax, 3); + masm.movdqu(xmm4, xmm2); + masm.addsd(xmm2, xmm1); + masm.subsd(xmm4, xmm2); + masm.addsd(xmm1, xmm4); + masm.pextrw(rax, xmm1, 3); + masm.andl(rax, 32767); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block34); + masm.movdl(rax, xmm2); + masm.andl(rax, 1); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block35); + + masm.bind(block36); + masm.xorpd(xmm1, xmm1); + masm.movl(rdx, 30704); + masm.pinsrw(xmm1, rdx, 3); + masm.movq(xmm2, recordExternalAddress(crb, log2E)); // 0x00000000, 0x3ff72000, + // 0x161bb241, 0xbf5dabe1 + masm.movq(xmm4, new AMD64Address(rsp, 8)); + masm.pextrw(rax, xmm4, 3); + masm.movl(rdx, 8192); + masm.movdl(xmm4, rdx); + masm.andl(rax, 32767); + masm.subl(rax, 16); + masm.jcc(AMD64Assembler.ConditionFlag.Less, block10); + masm.movl(rdx, rax); + masm.andl(rdx, 32752); + masm.subl(rdx, 16368); + masm.movl(rcx, rdx); + masm.sarl(rdx, 31); + masm.addl(rcx, rdx); + masm.xorl(rcx, rdx); + masm.addl(rcx, 16); + masm.bsrl(rcx, rcx); + masm.movl(tmp1, Integer.MIN_VALUE); + masm.jmp(block1); + + masm.bind(block34); + masm.xorpd(xmm1, xmm1); + masm.movl(rax, 32752); + masm.pinsrw(xmm1, rax, 3); + masm.xorpd(xmm0, xmm0); + masm.mulsd(xmm0, xmm1); + masm.movl(new AMD64Address(rsp, 0), 28); + masm.jmp(block17); + + masm.bind(block35); + masm.xorpd(xmm1, xmm1); + masm.movl(rdx, 30704); + masm.pinsrw(xmm1, rdx, 3); + masm.movq(xmm2, recordExternalAddress(crb, log2E)); // 0x00000000, 0x3ff72000, + // 0x161bb241, 0xbf5dabe1 + masm.movq(xmm4, new AMD64Address(rsp, 8)); + masm.pextrw(rax, xmm4, 3); + masm.movl(rdx, 8192); + masm.movdl(xmm4, rdx); + masm.andl(rax, 32767); + masm.subl(rax, 16); + masm.jcc(AMD64Assembler.ConditionFlag.Less, block8); + masm.movl(rdx, rax); + masm.andl(rdx, 32752); + masm.subl(rdx, 16368); + masm.movl(rcx, rdx); + masm.sarl(rdx, 31); + masm.addl(rcx, rdx); + masm.xorl(rcx, rdx); + masm.addl(rcx, 16); + masm.bsrl(rcx, rcx); + masm.movl(tmp1, 0); + masm.jmp(block1); + + masm.bind(block19); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 16368); + masm.pinsrw(xmm0, rax, 3); + masm.jmp(block56); + + masm.bind(block22); + masm.xorpd(xmm0, xmm0); + masm.jmp(block56); + + masm.bind(block11); + masm.addl(rax, 384); + masm.cmpl(rax, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Less, block38); + masm.mulsd(xmm5, xmm1); + masm.addsd(xmm0, xmm7); + masm.shrl(tmp1, 31); + masm.addpd(xmm3, xmm0); + masm.pshufd(xmm0, xmm3, 238); + masm.addsd(xmm3, xmm0); + masm.leaq(tmp4, recordExternalAddress(crb, log2)); // 0xfefa39ef, 0x3fe62e42, + // 0xfefa39ef, 0xbfe62e42 + masm.movq(xmm4, new AMD64Address(tmp4, tmp1, AMD64Address.Scale.Times8, 0)); + masm.mulsd(xmm1, xmm3); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 16368); + masm.shll(tmp1, 15); + masm.orl(rax, tmp1); + masm.pinsrw(xmm0, rax, 3); + masm.addsd(xmm5, xmm1); + masm.mulsd(xmm5, xmm4); + masm.addsd(xmm0, xmm5); + masm.jmp(block56); + + masm.bind(block38); + + masm.bind(block37); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 16368); + masm.pinsrw(xmm0, rax, 3); + masm.jmp(block56); + + masm.bind(block39); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 16368); + masm.pinsrw(xmm0, rax, 3); + masm.movl(new AMD64Address(rsp, 0), 26); + masm.jmp(block17); + + masm.bind(block9); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movdqu(xmm2, xmm1); + masm.pextrw(rax, xmm1, 3); + masm.andl(rax, 32752); + masm.cmpl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block40); + masm.movdl(rax, xmm2); + masm.psrlq(xmm2, 20); + masm.movdl(rdx, xmm2); + masm.orl(rax, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block18); + + masm.bind(block40); + masm.movdl(rax, xmm1); + masm.psrlq(xmm1, 32); + masm.movdl(rdx, xmm1); + masm.movl(rcx, rdx); + masm.addl(rdx, rdx); + masm.orl(rax, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block39); + masm.shrl(rdx, 21); + masm.cmpl(rdx, 1075); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block41); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block42); + masm.cmpl(rdx, 1023); + masm.jcc(AMD64Assembler.ConditionFlag.Below, block41); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movl(rax, 17208); + masm.xorpd(xmm3, xmm3); + masm.pinsrw(xmm3, rax, 3); + masm.movdqu(xmm4, xmm3); + masm.addsd(xmm3, xmm1); + masm.subsd(xmm4, xmm3); + masm.addsd(xmm1, xmm4); + masm.pextrw(rax, xmm1, 3); + masm.andl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block41); + masm.movdl(rax, xmm3); + masm.andl(rax, 1); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block41); + + masm.bind(block43); + masm.movq(xmm0, new AMD64Address(rsp, 8)); + masm.testl(rcx, Integer.MIN_VALUE); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block44); + masm.jmp(block56); + + masm.bind(block42); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movdl(rax, xmm1); + masm.testl(rax, 1); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block43); + + masm.bind(block41); + masm.testl(rcx, Integer.MIN_VALUE); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block22); + masm.xorpd(xmm0, xmm0); + + masm.bind(block44); + masm.movl(rax, 16368); + masm.xorpd(xmm1, xmm1); + masm.pinsrw(xmm1, rax, 3); + masm.divsd(xmm1, xmm0); + masm.movdqu(xmm0, xmm1); + masm.movl(new AMD64Address(rsp, 0), 27); + masm.jmp(block17); + + masm.bind(block12); + masm.movq(xmm2, new AMD64Address(rsp, 8)); + masm.movq(xmm6, new AMD64Address(rsp, 16)); + masm.pextrw(rax, xmm2, 3); + masm.pextrw(rdx, xmm6, 3); + masm.movl(rcx, 32752); + masm.andl(rcx, rdx); + masm.cmpl(rcx, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block45); + masm.andl(rax, 32752); + masm.subl(rax, 16368); + masm.xorl(rdx, rax); + masm.testl(rdx, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block46); + + masm.bind(block47); + masm.movl(rax, 32736); + masm.pinsrw(xmm0, rax, 3); + masm.shrl(tmp1, 16); + masm.orl(rax, tmp1); + masm.pinsrw(xmm1, rax, 3); + masm.mulsd(xmm0, xmm1); + + masm.bind(block14); + masm.movl(new AMD64Address(rsp, 0), 24); + masm.jmp(block17); + + masm.bind(block46); + masm.movl(rax, 16); + masm.pinsrw(xmm0, rax, 3); + masm.mulsd(xmm0, xmm0); + masm.testl(tmp1, Integer.MIN_VALUE); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block48); + masm.movq(tmp2, 0x8000000000000000L); + masm.movdq(xmm2, tmp2); + masm.xorpd(xmm0, xmm2); + + masm.bind(block48); + masm.movl(new AMD64Address(rsp, 0), 25); + masm.jmp(block17); + + masm.bind(block13); + masm.pextrw(rcx, xmm5, 3); + masm.pextrw(rdx, xmm4, 3); + masm.movl(rax, -1); + masm.andl(rcx, 32752); + masm.subl(rcx, 16368); + masm.andl(rdx, 32752); + masm.addl(rdx, rcx); + masm.movl(rcx, -31); + masm.sarl(rdx, 4); + masm.subl(rcx, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.LessEqual, block49); + masm.cmpl(rcx, 20); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block50); + masm.shll(rax); + + masm.bind(block49); + masm.movdl(xmm0, rax); + masm.psllq(xmm0, 32); + masm.pand(xmm0, xmm5); + masm.subsd(xmm5, xmm0); + masm.addsd(xmm5, xmm1); + masm.mulsd(xmm0, xmm4); + masm.mulsd(xmm5, xmm4); + masm.addsd(xmm0, xmm5); + + masm.bind(block50); + masm.jmp(block48); + + masm.bind(block2); + masm.movw(rcx, new AMD64Address(rsp, 22)); + masm.movl(rdx, Integer.MIN_VALUE); + masm.movdl(xmm1, rdx); + masm.xorpd(xmm7, xmm7); + masm.paddd(xmm0, xmm4); + masm.movdl(rdx, xmm0); + masm.psllq(xmm0, 29); + masm.paddq(xmm1, xmm3); + masm.pand(xmm5, xmm1); + masm.andl(rcx, 32752); + masm.cmpl(rcx, 16560); + masm.jcc(AMD64Assembler.ConditionFlag.Less, block3); + masm.pand(xmm0, xmm6); + masm.subsd(xmm3, xmm5); + masm.addl(rax, 16351); + masm.shrl(rax, 4); + masm.subl(rax, 1022); + masm.cvtsi2sdl(xmm7, rax); + masm.mulpd(xmm5, xmm0); + masm.leaq(r11, recordExternalAddress(crb, lTbl)); + masm.movq(xmm4, recordExternalAddress(crb, coeffH)); // 0x00000000, 0xbfd61a00, + // 0x00000000, 0xbf5dabe1 + masm.mulsd(xmm3, xmm0); + masm.movq(xmm6, recordExternalAddress(crb, coeffH)); // 0x00000000, 0xbfd61a00, + // 0x00000000, 0xbf5dabe1 + masm.subsd(xmm5, xmm2); + masm.movq(xmm1, recordExternalAddress(crb, coeffH8)); // 0x00000000, 0xbf5dabe1 + masm.pshufd(xmm2, xmm3, 68); + masm.unpcklpd(xmm5, xmm3); + masm.addsd(xmm3, xmm5); + masm.movq(xmm0, recordExternalAddress(crb, coeffH8)); // 0x00000000, 0xbf5dabe1 + masm.andl(rdx, 16760832); + masm.shrl(rdx, 10); + masm.addpd(xmm7, new AMD64Address(tmp4, rdx, AMD64Address.Scale.Times1, -3648)); + masm.mulsd(xmm4, xmm5); + masm.mulsd(xmm0, xmm5); + masm.mulsd(xmm6, xmm2); + masm.mulsd(xmm1, xmm2); + masm.movdqu(xmm2, xmm5); + masm.mulsd(xmm4, xmm5); + masm.addsd(xmm5, xmm0); + masm.movdqu(xmm0, xmm7); + masm.addsd(xmm2, xmm3); + masm.addsd(xmm7, xmm5); + masm.mulsd(xmm6, xmm2); + masm.subsd(xmm0, xmm7); + masm.movdqu(xmm2, xmm7); + masm.addsd(xmm7, xmm4); + masm.addsd(xmm0, xmm5); + masm.subsd(xmm2, xmm7); + masm.addsd(xmm4, xmm2); + masm.pshufd(xmm2, xmm5, 238); + masm.movdqu(xmm5, xmm7); + masm.addsd(xmm7, xmm2); + masm.addsd(xmm4, xmm0); + masm.movdqu(xmm0, recordExternalAddress(crb, coeff)); // 0x6dc96112, 0xbf836578, + // 0xee241472, 0xbf9b0301 + masm.subsd(xmm5, xmm7); + masm.addsd(xmm6, xmm4); + masm.movdqu(xmm4, xmm7); + masm.addsd(xmm5, xmm2); + masm.addsd(xmm7, xmm1); + masm.movdqu(xmm2, recordExternalAddress(crb, coeff64)); // 0x486ececc, 0x3fc4635e, + // 0x161bb241, 0xbf5dabe1 + masm.subsd(xmm4, xmm7); + masm.addsd(xmm6, xmm5); + masm.addsd(xmm4, xmm1); + masm.pshufd(xmm5, xmm7, 238); + masm.movapd(xmm1, xmm7); + masm.addsd(xmm7, xmm5); + masm.subsd(xmm1, xmm7); + masm.addsd(xmm1, xmm5); + masm.movdqu(xmm5, recordExternalAddress(crb, coeff80)); // 0x9f95985a, 0xbfb528db, + // 0xf8b5787d, 0x3ef2531e + masm.pshufd(xmm3, xmm3, 68); + masm.addsd(xmm6, xmm4); + masm.addsd(xmm6, xmm1); + masm.movdqu(xmm1, recordExternalAddress(crb, coeff32)); // 0x9f95985a, 0xbfb528db, + // 0xb3841d2a, 0xbfd619b6 + masm.mulpd(xmm0, xmm3); + masm.mulpd(xmm2, xmm3); + masm.pshufd(xmm4, xmm3, 68); + masm.mulpd(xmm3, xmm3); + masm.addpd(xmm0, xmm1); + masm.addpd(xmm5, xmm2); + masm.mulsd(xmm4, xmm3); + masm.movq(xmm2, recordExternalAddress(crb, highmaskLogX)); // 0xf8000000, 0xffffffff, + // 0x00000000, 0xfffff800 + masm.mulpd(xmm3, xmm3); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movw(rcx, new AMD64Address(rsp, 22)); + masm.mulpd(xmm0, xmm4); + masm.pextrw(rax, xmm7, 3); + masm.mulpd(xmm5, xmm4); + masm.mulpd(xmm0, xmm3); + masm.movq(xmm4, recordExternalAddress(crb, highmaskY8)); // 0x00000000, 0xffffffff + masm.pand(xmm2, xmm7); + masm.addsd(xmm5, xmm6); + masm.subsd(xmm7, xmm2); + masm.addpd(xmm5, xmm0); + masm.andl(rax, 32752); + masm.subl(rax, 16368); + masm.andl(rcx, 32752); + masm.cmpl(rcx, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block45); + masm.addl(rcx, rax); + masm.cmpl(rcx, 16576); + masm.jcc(AMD64Assembler.ConditionFlag.AboveEqual, block51); + masm.pshufd(xmm0, xmm5, 238); + masm.pand(xmm4, xmm1); + masm.movdqu(xmm3, xmm1); + masm.addsd(xmm5, xmm0); + masm.subsd(xmm1, xmm4); + masm.xorpd(xmm6, xmm6); + masm.movl(rdx, 17080); + masm.pinsrw(xmm6, rdx, 3); + masm.addsd(xmm7, xmm5); + masm.mulsd(xmm4, xmm2); + masm.mulsd(xmm1, xmm2); + masm.movdqu(xmm5, xmm6); + masm.mulsd(xmm3, xmm7); + masm.addsd(xmm6, xmm4); + masm.addsd(xmm1, xmm3); + masm.movdqu(xmm7, recordExternalAddress(crb, eCoeff)); // 0xe78a6731, 0x3f55d87f, + // 0xd704a0c0, 0x3fac6b08 + masm.movdl(rdx, xmm6); + masm.subsd(xmm6, xmm5); + masm.leaq(tmp4, recordExternalAddress(crb, tExp)); + masm.movdqu(xmm3, recordExternalAddress(crb, eCoeff16)); // 0x6fba4e77, 0x3f83b2ab, + // 0xff82c58f, 0x3fcebfbd + masm.movq(xmm2, recordExternalAddress(crb, eCoeff32)); // 0xfefa39ef, 0x3fe62e42, + // 0x00000000, 0x00000000 + masm.subsd(xmm4, xmm6); + masm.movl(rcx, rdx); + masm.andl(rdx, 255); + masm.addl(rdx, rdx); + masm.movdqu(xmm5, new AMD64Address(tmp4, rdx, AMD64Address.Scale.Times8, 0)); + masm.addsd(xmm4, xmm1); + masm.pextrw(rdx, xmm6, 3); + masm.shrl(rcx, 8); + masm.movl(rax, rcx); + masm.shrl(rcx, 1); + masm.subl(rax, rcx); + masm.shll(rcx, 20); + masm.movdl(xmm6, rcx); + masm.pshufd(xmm0, xmm4, 68); + masm.pshufd(xmm1, xmm4, 68); + masm.mulpd(xmm0, xmm0); + masm.mulpd(xmm7, xmm1); + masm.pshufd(xmm6, xmm6, 17); + masm.mulsd(xmm2, xmm4); + masm.andl(rdx, 32767); + masm.cmpl(rdx, 16529); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block12); + masm.mulsd(xmm0, xmm0); + masm.paddd(xmm5, xmm6); + masm.addpd(xmm3, xmm7); + masm.mulsd(xmm2, xmm5); + masm.pshufd(xmm6, xmm5, 238); + masm.mulpd(xmm0, xmm3); + masm.addsd(xmm2, xmm6); + masm.pshufd(xmm3, xmm0, 238); + masm.addl(rax, 1023); + masm.shll(rax, 20); + masm.orl(rax, tmp1); + masm.movdl(xmm4, rax); + masm.mulsd(xmm0, xmm5); + masm.mulsd(xmm3, xmm5); + masm.addsd(xmm0, xmm2); + masm.psllq(xmm4, 32); + masm.addsd(xmm0, xmm3); + masm.movdqu(xmm1, xmm0); + masm.addsd(xmm0, xmm5); + masm.mulsd(xmm0, xmm4); + masm.pextrw(rax, xmm0, 3); + masm.andl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block13); + masm.cmpl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block14); + + masm.bind(block52); + masm.jmp(block56); + + masm.bind(block45); + masm.movq(xmm0, new AMD64Address(rsp, 8)); + masm.xorpd(xmm2, xmm2); + masm.movl(rax, 49136); + masm.pinsrw(xmm2, rax, 3); + masm.addsd(xmm2, xmm0); + masm.pextrw(rax, xmm2, 3); + masm.cmpl(rax, 0); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block53); + masm.xorpd(xmm0, xmm0); + masm.movl(rax, 32760); + masm.pinsrw(xmm0, rax, 3); + masm.jmp(block56); + + masm.bind(block53); + masm.movq(xmm1, new AMD64Address(rsp, 16)); + masm.movdl(rdx, xmm1); + masm.movdqu(xmm3, xmm1); + masm.psrlq(xmm3, 20); + masm.movdl(rcx, xmm3); + masm.orl(rcx, rdx); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block54); + masm.addsd(xmm1, xmm1); + masm.movdqu(xmm0, xmm1); + masm.jmp(block56); + + masm.bind(block51); + masm.pextrw(rax, xmm1, 3); + masm.pextrw(rcx, xmm2, 3); + masm.xorl(rax, rcx); + masm.testl(rax, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block47); + masm.jmp(block46); + + masm.bind(block54); + masm.pextrw(rax, xmm0, 3); + masm.andl(rax, 32752); + masm.pextrw(rdx, xmm1, 3); + masm.xorpd(xmm0, xmm0); + masm.subl(rax, 16368); + masm.xorl(rax, rdx); + masm.testl(rax, 32768); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block55); + masm.jmp(block56); + + masm.bind(block55); + masm.movl(rdx, 32752); + masm.pinsrw(xmm0, rdx, 3); + masm.jmp(block56); + + masm.bind(block17); + masm.movq(new AMD64Address(rsp, 24), xmm0); + + masm.movq(xmm0, new AMD64Address(rsp, 24)); + + masm.bind(block56); + masm.addq(rsp, 40); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathSinOp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathSinOp.java new file mode 100644 index 00000000000..e40d54ff5ba --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathSinOp.java @@ -0,0 +1,902 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, Intel Corporation. All rights reserved. + * Intel Math Library (LIBM) Source Code + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.amd64; + +import static jdk.vm.ci.amd64.AMD64.r10; +import static jdk.vm.ci.amd64.AMD64.r11; +import static jdk.vm.ci.amd64.AMD64.r8; +import static jdk.vm.ci.amd64.AMD64.r9; +import static jdk.vm.ci.amd64.AMD64.rax; +import static jdk.vm.ci.amd64.AMD64.rbx; +import static jdk.vm.ci.amd64.AMD64.rcx; +import static jdk.vm.ci.amd64.AMD64.rdi; +import static jdk.vm.ci.amd64.AMD64.rdx; +import static jdk.vm.ci.amd64.AMD64.rsi; +import static jdk.vm.ci.amd64.AMD64.rsp; +import static jdk.vm.ci.amd64.AMD64.xmm0; +import static jdk.vm.ci.amd64.AMD64.xmm1; +import static jdk.vm.ci.amd64.AMD64.xmm2; +import static jdk.vm.ci.amd64.AMD64.xmm3; +import static jdk.vm.ci.amd64.AMD64.xmm4; +import static jdk.vm.ci.amd64.AMD64.xmm5; +import static jdk.vm.ci.amd64.AMD64.xmm6; +import static jdk.vm.ci.amd64.AMD64.xmm7; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.pointerConstant; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.recordExternalAddress; + +import org.graalvm.compiler.asm.Label; +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.asm.amd64.AMD64Assembler.ConditionFlag; +import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.lir.LIRInstructionClass; +import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; + +import jdk.vm.ci.amd64.AMD64; + +/** + * 
+ *                     ALGORITHM DESCRIPTION - SIN()
+ *                     ---------------------
+ *
+ *     1. RANGE REDUCTION
+ *
+ *     We perform an initial range reduction from X to r with
+ *
+ *          X =~= N * pi/32 + r
+ *
+ *     so that |r| <= pi/64 + epsilon. We restrict inputs to those
+ *     where |N| <= 932560. Beyond this, the range reduction is
+ *     insufficiently accurate. For extremely small inputs,
+ *     denormalization can occur internally, impacting performance.
+ *     This means that the main path is actually only taken for
+ *     2^-252 <= |X| < 90112.
+ *
+ *     To avoid branches, we perform the range reduction to full
+ *     accuracy each time.
+ *
+ *          X - N * (P_1 + P_2 + P_3)
+ *
+ *     where P_1 and P_2 are 32-bit numbers (so multiplication by N
+ *     is exact) and P_3 is a 53-bit number. Together, these
+ *     approximate pi well enough for all cases in the restricted
+ *     range.
+ *
+ *     The main reduction sequence is:
+ *
+ *             y = 32/pi * x
+ *             N = integer(y)
+ *     (computed by adding and subtracting off SHIFTER)
+ *
+ *             m_1 = N * P_1
+ *             m_2 = N * P_2
+ *             r_1 = x - m_1
+ *             r = r_1 - m_2
+ *     (this r can be used for most of the calculation)
+ *
+ *             c_1 = r_1 - r
+ *             m_3 = N * P_3
+ *             c_2 = c_1 - m_2
+ *             c = c_2 - m_3
+ *
+ *     2. MAIN ALGORITHM
+ *
+ *     The algorithm uses a table lookup based on B = M * pi / 32
+ *     where M = N mod 64. The stored values are:
+ *       sigma             closest power of 2 to cos(B)
+ *       C_hl              53-bit cos(B) - sigma
+ *       S_hi + S_lo       2 * 53-bit sin(B)
+ *
+ *     The computation is organized as follows:
+ *
+ *          sin(B + r + c) = [sin(B) + sigma * r] +
+ *                           r * (cos(B) - sigma) +
+ *                           sin(B) * [cos(r + c) - 1] +
+ *                           cos(B) * [sin(r + c) - r]
+ *
+ *     which is approximately:
+ *
+ *          [S_hi + sigma * r] +
+ *          C_hl * r +
+ *          S_lo + S_hi * [(cos(r) - 1) - r * c] +
+ *          (C_hl + sigma) * [(sin(r) - r) + c]
+ *
+ *     and this is what is actually computed. We separate this sum
+ *     into four parts:
+ *
+ *          hi + med + pols + corr
+ *
+ *     where
+ *
+ *          hi       = S_hi + sigma r
+ *          med      = C_hl * r
+ *          pols     = S_hi * (cos(r) - 1) + (C_hl + sigma) * (sin(r) - r)
+ *          corr     = S_lo + c * ((C_hl + sigma) - S_hi * r)
+ *
+ *     3. POLYNOMIAL
+ *
+ *     The polynomial S_hi * (cos(r) - 1) + (C_hl + sigma) *
+ *     (sin(r) - r) can be rearranged freely, since it is quite
+ *     small, so we exploit parallelism to the fullest.
+ *
+ *          psc4       =   SC_4 * r_1
+ *          msc4       =   psc4 * r
+ *          r2         =   r * r
+ *          msc2       =   SC_2 * r2
+ *          r4         =   r2 * r2
+ *          psc3       =   SC_3 + msc4
+ *          psc1       =   SC_1 + msc2
+ *          msc3       =   r4 * psc3
+ *          sincospols =   psc1 + msc3
+ *          pols       =   sincospols *
+ *                         
+ *
+ *     4. CORRECTION TERM
+ *
+ *     This is where the "c" component of the range reduction is
+ *     taken into account; recall that just "r" is used for most of
+ *     the calculation.
+ *
+ *          -c   = m_3 - c_2
+ *          -d   = S_hi * r - (C_hl + sigma)
+ *          corr = -c * -d + S_lo
+ *
+ *     5. COMPENSATED SUMMATIONS
+ *
+ *     The two successive compensated summations add up the high
+ *     and medium parts, leaving just the low parts to add up at
+ *     the end.
+ *
+ *          rs        =  sigma * r
+ *          res_int   =  S_hi + rs
+ *          k_0       =  S_hi - res_int
+ *          k_2       =  k_0 + rs
+ *          med       =  C_hl * r
+ *          res_hi    =  res_int + med
+ *          k_1       =  res_int - res_hi
+ *          k_3       =  k_1 + med
+ *
+ *     6. FINAL SUMMATION
+ *
+ *     We now add up all the small parts:
+ *
+ *          res_lo = pols(hi) + pols(lo) + corr + k_1 + k_3
+ *
+ *     Now the overall result is just:
+ *
+ *          res_hi + res_lo
+ *
+ *     7. SMALL ARGUMENTS
+ *
+ *     If |x| < SNN (SNN meaning the smallest normal number), we
+ *     simply perform 0.1111111 cdots 1111 * x. For SNN <= |x|, we
+ *     do 2^-55 * (2^55 * x - x).
+ *
+ * Special cases:
+ *  sin(NaN) = quiet NaN, and raise invalid exception
+ *  sin(INF) = NaN and raise invalid exception
+ *  sin(+/-0) = +/-0
+ *
+ */ +public final class AMD64MathSinOp extends AMD64MathIntrinsicUnaryOp { + + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64MathSinOp.class); + + public AMD64MathSinOp() { + super(TYPE, /* GPR */ rax, rcx, rdx, rbx, rsi, rdi, r8, r9, r10, r11, + /* XMM */ xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7); + } + + private ArrayDataPointerConstant onehalf = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x3fe00000, 0x00000000, 0x3fe00000 + // @formatter:on + }); + + private ArrayDataPointerConstant p2 = pointerConstant(16, new int[]{ + // @formatter:off + 0x1a600000, 0x3d90b461, 0x1a600000, 0x3d90b461 + // @formatter:on + }); + + private ArrayDataPointerConstant sc4 = pointerConstant(16, new int[]{ + // @formatter:off + 0xa556c734, 0x3ec71de3, 0x1a01a01a, 0x3efa01a0 + // @formatter:on + }); + + private ArrayDataPointerConstant ctable = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x3ff00000, 0x176d6d31, 0xbf73b92e, + 0xbc29b42c, 0x3fb917a6, 0xe0000000, 0xbc3e2718, 0x00000000, + 0x3ff00000, 0x011469fb, 0xbf93ad06, 0x3c69a60b, 0x3fc8f8b8, + 0xc0000000, 0xbc626d19, 0x00000000, 0x3ff00000, 0x939d225a, + 0xbfa60bea, 0x2ed59f06, 0x3fd29406, 0xa0000000, 0xbc75d28d, + 0x00000000, 0x3ff00000, 0x866b95cf, 0xbfb37ca1, 0xa6aea963, + 0x3fd87de2, 0xe0000000, 0xbc672ced, 0x00000000, 0x3ff00000, + 0x73fa1279, 0xbfbe3a68, 0x3806f63b, 0x3fde2b5d, 0x20000000, + 0x3c5e0d89, 0x00000000, 0x3ff00000, 0x5bc57974, 0xbfc59267, + 0x39ae68c8, 0x3fe1c73b, 0x20000000, 0x3c8b25dd, 0x00000000, + 0x3ff00000, 0x53aba2fd, 0xbfcd0dfe, 0x25091dd6, 0x3fe44cf3, + 0x20000000, 0x3c68076a, 0x00000000, 0x3ff00000, 0x99fcef32, + 0x3fca8279, 0x667f3bcd, 0x3fe6a09e, 0x20000000, 0xbc8bdd34, + 0x00000000, 0x3fe00000, 0x94247758, 0x3fc133cc, 0x6b151741, + 0x3fe8bc80, 0x20000000, 0xbc82c5e1, 0x00000000, 0x3fe00000, + 0x9ae68c87, 0x3fac73b3, 0x290ea1a3, 0x3fea9b66, 0xe0000000, + 0x3c39f630, 0x00000000, 0x3fe00000, 0x7f909c4e, 0xbf9d4a2c, + 0xf180bdb1, 0x3fec38b2, 0x80000000, 0xbc76e0b1, 0x00000000, + 0x3fe00000, 0x65455a75, 0xbfbe0875, 0xcf328d46, 0x3fed906b, + 0x20000000, 0x3c7457e6, 0x00000000, 0x3fe00000, 0x76acf82d, + 0x3fa4a031, 0x56c62dda, 0x3fee9f41, 0xe0000000, 0x3c8760b1, + 0x00000000, 0x3fd00000, 0x0e5967d5, 0xbfac1d1f, 0xcff75cb0, + 0x3fef6297, 0x20000000, 0x3c756217, 0x00000000, 0x3fd00000, + 0x0f592f50, 0xbf9ba165, 0xa3d12526, 0x3fefd88d, 0x40000000, + 0xbc887df6, 0x00000000, 0x3fc00000, 0x00000000, 0x00000000, + 0x00000000, 0x3ff00000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x0f592f50, 0x3f9ba165, 0xa3d12526, 0x3fefd88d, + 0x40000000, 0xbc887df6, 0x00000000, 0xbfc00000, 0x0e5967d5, + 0x3fac1d1f, 0xcff75cb0, 0x3fef6297, 0x20000000, 0x3c756217, + 0x00000000, 0xbfd00000, 0x76acf82d, 0xbfa4a031, 0x56c62dda, + 0x3fee9f41, 0xe0000000, 0x3c8760b1, 0x00000000, 0xbfd00000, + 0x65455a75, 0x3fbe0875, 0xcf328d46, 0x3fed906b, 0x20000000, + 0x3c7457e6, 0x00000000, 0xbfe00000, 0x7f909c4e, 0x3f9d4a2c, + 0xf180bdb1, 0x3fec38b2, 0x80000000, 0xbc76e0b1, 0x00000000, + 0xbfe00000, 0x9ae68c87, 0xbfac73b3, 0x290ea1a3, 0x3fea9b66, + 0xe0000000, 0x3c39f630, 0x00000000, 0xbfe00000, 0x94247758, + 0xbfc133cc, 0x6b151741, 0x3fe8bc80, 0x20000000, 0xbc82c5e1, + 0x00000000, 0xbfe00000, 0x99fcef32, 0xbfca8279, 0x667f3bcd, + 0x3fe6a09e, 0x20000000, 0xbc8bdd34, 0x00000000, 0xbfe00000, + 0x53aba2fd, 0x3fcd0dfe, 0x25091dd6, 0x3fe44cf3, 0x20000000, + 0x3c68076a, 0x00000000, 0xbff00000, 0x5bc57974, 0x3fc59267, + 0x39ae68c8, 0x3fe1c73b, 0x20000000, 0x3c8b25dd, 0x00000000, + 0xbff00000, 0x73fa1279, 0x3fbe3a68, 0x3806f63b, 0x3fde2b5d, + 0x20000000, 0x3c5e0d89, 0x00000000, 0xbff00000, 0x866b95cf, + 0x3fb37ca1, 0xa6aea963, 0x3fd87de2, 0xe0000000, 0xbc672ced, + 0x00000000, 0xbff00000, 0x939d225a, 0x3fa60bea, 0x2ed59f06, + 0x3fd29406, 0xa0000000, 0xbc75d28d, 0x00000000, 0xbff00000, + 0x011469fb, 0x3f93ad06, 0x3c69a60b, 0x3fc8f8b8, 0xc0000000, + 0xbc626d19, 0x00000000, 0xbff00000, 0x176d6d31, 0x3f73b92e, + 0xbc29b42c, 0x3fb917a6, 0xe0000000, 0xbc3e2718, 0x00000000, + 0xbff00000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0xbff00000, 0x176d6d31, + 0x3f73b92e, 0xbc29b42c, 0xbfb917a6, 0xe0000000, 0x3c3e2718, + 0x00000000, 0xbff00000, 0x011469fb, 0x3f93ad06, 0x3c69a60b, + 0xbfc8f8b8, 0xc0000000, 0x3c626d19, 0x00000000, 0xbff00000, + 0x939d225a, 0x3fa60bea, 0x2ed59f06, 0xbfd29406, 0xa0000000, + 0x3c75d28d, 0x00000000, 0xbff00000, 0x866b95cf, 0x3fb37ca1, + 0xa6aea963, 0xbfd87de2, 0xe0000000, 0x3c672ced, 0x00000000, + 0xbff00000, 0x73fa1279, 0x3fbe3a68, 0x3806f63b, 0xbfde2b5d, + 0x20000000, 0xbc5e0d89, 0x00000000, 0xbff00000, 0x5bc57974, + 0x3fc59267, 0x39ae68c8, 0xbfe1c73b, 0x20000000, 0xbc8b25dd, + 0x00000000, 0xbff00000, 0x53aba2fd, 0x3fcd0dfe, 0x25091dd6, + 0xbfe44cf3, 0x20000000, 0xbc68076a, 0x00000000, 0xbff00000, + 0x99fcef32, 0xbfca8279, 0x667f3bcd, 0xbfe6a09e, 0x20000000, + 0x3c8bdd34, 0x00000000, 0xbfe00000, 0x94247758, 0xbfc133cc, + 0x6b151741, 0xbfe8bc80, 0x20000000, 0x3c82c5e1, 0x00000000, + 0xbfe00000, 0x9ae68c87, 0xbfac73b3, 0x290ea1a3, 0xbfea9b66, + 0xe0000000, 0xbc39f630, 0x00000000, 0xbfe00000, 0x7f909c4e, + 0x3f9d4a2c, 0xf180bdb1, 0xbfec38b2, 0x80000000, 0x3c76e0b1, + 0x00000000, 0xbfe00000, 0x65455a75, 0x3fbe0875, 0xcf328d46, + 0xbfed906b, 0x20000000, 0xbc7457e6, 0x00000000, 0xbfe00000, + 0x76acf82d, 0xbfa4a031, 0x56c62dda, 0xbfee9f41, 0xe0000000, + 0xbc8760b1, 0x00000000, 0xbfd00000, 0x0e5967d5, 0x3fac1d1f, + 0xcff75cb0, 0xbfef6297, 0x20000000, 0xbc756217, 0x00000000, + 0xbfd00000, 0x0f592f50, 0x3f9ba165, 0xa3d12526, 0xbfefd88d, + 0x40000000, 0x3c887df6, 0x00000000, 0xbfc00000, 0x00000000, + 0x00000000, 0x00000000, 0xbff00000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x0f592f50, 0xbf9ba165, 0xa3d12526, + 0xbfefd88d, 0x40000000, 0x3c887df6, 0x00000000, 0x3fc00000, + 0x0e5967d5, 0xbfac1d1f, 0xcff75cb0, 0xbfef6297, 0x20000000, + 0xbc756217, 0x00000000, 0x3fd00000, 0x76acf82d, 0x3fa4a031, + 0x56c62dda, 0xbfee9f41, 0xe0000000, 0xbc8760b1, 0x00000000, + 0x3fd00000, 0x65455a75, 0xbfbe0875, 0xcf328d46, 0xbfed906b, + 0x20000000, 0xbc7457e6, 0x00000000, 0x3fe00000, 0x7f909c4e, + 0xbf9d4a2c, 0xf180bdb1, 0xbfec38b2, 0x80000000, 0x3c76e0b1, + 0x00000000, 0x3fe00000, 0x9ae68c87, 0x3fac73b3, 0x290ea1a3, + 0xbfea9b66, 0xe0000000, 0xbc39f630, 0x00000000, 0x3fe00000, + 0x94247758, 0x3fc133cc, 0x6b151741, 0xbfe8bc80, 0x20000000, + 0x3c82c5e1, 0x00000000, 0x3fe00000, 0x99fcef32, 0x3fca8279, + 0x667f3bcd, 0xbfe6a09e, 0x20000000, 0x3c8bdd34, 0x00000000, + 0x3fe00000, 0x53aba2fd, 0xbfcd0dfe, 0x25091dd6, 0xbfe44cf3, + 0x20000000, 0xbc68076a, 0x00000000, 0x3ff00000, 0x5bc57974, + 0xbfc59267, 0x39ae68c8, 0xbfe1c73b, 0x20000000, 0xbc8b25dd, + 0x00000000, 0x3ff00000, 0x73fa1279, 0xbfbe3a68, 0x3806f63b, + 0xbfde2b5d, 0x20000000, 0xbc5e0d89, 0x00000000, 0x3ff00000, + 0x866b95cf, 0xbfb37ca1, 0xa6aea963, 0xbfd87de2, 0xe0000000, + 0x3c672ced, 0x00000000, 0x3ff00000, 0x939d225a, 0xbfa60bea, + 0x2ed59f06, 0xbfd29406, 0xa0000000, 0x3c75d28d, 0x00000000, + 0x3ff00000, 0x011469fb, 0xbf93ad06, 0x3c69a60b, 0xbfc8f8b8, + 0xc0000000, 0x3c626d19, 0x00000000, 0x3ff00000, 0x176d6d31, + 0xbf73b92e, 0xbc29b42c, 0xbfb917a6, 0xe0000000, 0x3c3e2718, + 0x00000000, 0x3ff00000 + // @formatter:on + }); + + private ArrayDataPointerConstant sc2 = pointerConstant(16, new int[]{ + // @formatter:off + 0x11111111, 0x3f811111, 0x55555555, 0x3fa55555 + // @formatter:on + }); + + private ArrayDataPointerConstant sc3 = pointerConstant(16, new int[]{ + // @formatter:off + 0x1a01a01a, 0xbf2a01a0, 0x16c16c17, 0xbf56c16c + // @formatter:on + }); + + private ArrayDataPointerConstant sc1 = pointerConstant(16, new int[]{ + // @formatter:off + 0x55555555, 0xbfc55555, 0x00000000, 0xbfe00000 + // @formatter:on + }); + + private ArrayDataPointerConstant piInvTable = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x00000000, 0xa2f9836e, 0x4e441529, 0xfc2757d1, + 0xf534ddc0, 0xdb629599, 0x3c439041, 0xfe5163ab, 0xdebbc561, + 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c, 0xfe1deb1c, + 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41, + 0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, + 0xde05980f, 0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, + 0x4f463f66, 0x9e5fea2d, 0x7527bac7, 0xebe5f17b, 0x3d0739f7, + 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08, 0x56033046, 0xfc7b6bab, + 0xf0cfbc21 + // @formatter:on + }); + + private ArrayDataPointerConstant pi4 = pointerConstant(8, new int[]{ + // @formatter:off + 0x40000000, 0x3fe921fb, + }); + private ArrayDataPointerConstant pi48 = pointerConstant(8, new int[]{ + 0x18469899, 0x3e64442d + // @formatter:on + }); + + private ArrayDataPointerConstant pi32Inv = pointerConstant(8, new int[]{ + // @formatter:off + 0x6dc9c883, 0x40245f30 + // @formatter:on + }); + + private ArrayDataPointerConstant shifter = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x43380000 + // @formatter:on + }); + + private ArrayDataPointerConstant signMask = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x80000000 + // @formatter:on + }); + + private ArrayDataPointerConstant p3 = pointerConstant(8, new int[]{ + // @formatter:off + 0x2e037073, 0x3b63198a + // @formatter:on + }); + + private ArrayDataPointerConstant allOnes = pointerConstant(8, new int[]{ + // @formatter:off + 0xffffffff, 0x3fefffff + // @formatter:on + }); + + private ArrayDataPointerConstant twoPow55 = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x43600000 + // @formatter:on + }); + + private ArrayDataPointerConstant twoPowM55 = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x3c800000 + // @formatter:on + }); + + private ArrayDataPointerConstant p1 = pointerConstant(8, new int[]{ + // @formatter:off + 0x54400000, 0x3fb921fb + // @formatter:on + }); + + private ArrayDataPointerConstant negZero = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x80000000 + // @formatter:on + }); + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + Label block0 = new Label(); + Label block1 = new Label(); + Label block2 = new Label(); + Label block3 = new Label(); + Label block4 = new Label(); + Label block5 = new Label(); + Label block6 = new Label(); + Label block7 = new Label(); + Label block8 = new Label(); + Label block9 = new Label(); + Label block10 = new Label(); + Label block11 = new Label(); + Label block12 = new Label(); + Label block13 = new Label(); + Label block14 = new Label(); + + masm.push(AMD64.rbx); + masm.subq(rsp, 16); + masm.movsd(new AMD64Address(rsp, 8), xmm0); + masm.movl(rax, new AMD64Address(rsp, 12)); + masm.movq(xmm1, recordExternalAddress(crb, pi32Inv)); // 0x6dc9c883, 0x40245f30 + masm.movq(xmm2, recordExternalAddress(crb, shifter)); // 0x00000000, 0x43380000 + masm.andl(rax, 2147418112); + masm.subl(rax, 808452096); + masm.cmpl(rax, 281346048); + masm.jcc(ConditionFlag.Above, block0); + masm.mulsd(xmm1, xmm0); + masm.movdqu(xmm5, recordExternalAddress(crb, onehalf)); // 0x00000000, 0x3fe00000, + // 0x00000000, 0x3fe00000 + masm.movq(xmm4, recordExternalAddress(crb, signMask)); // 0x00000000, 0x80000000 + masm.pand(xmm4, xmm0); + masm.por(xmm5, xmm4); + masm.addpd(xmm1, xmm5); + masm.cvttsd2sil(rdx, xmm1); + masm.cvtsi2sdl(xmm1, rdx); + masm.movdqu(xmm6, recordExternalAddress(crb, p2)); // 0x1a600000, 0x3d90b461, + // 0x1a600000, 0x3d90b461 + masm.movq(r8, 0x3fb921fb54400000L); + masm.movdq(xmm3, r8); + masm.movdqu(xmm5, recordExternalAddress(crb, sc4)); // 0xa556c734, 0x3ec71de3, + // 0x1a01a01a, 0x3efa01a0 + masm.pshufd(xmm4, xmm0, 68); + masm.mulsd(xmm3, xmm1); + if (masm.supports(AMD64.CPUFeature.SSE3)) { + masm.movddup(xmm1, xmm1); + } else { + masm.movlhps(xmm1, xmm1); + } + masm.andl(rdx, 63); + masm.shll(rdx, 5); + masm.leaq(AMD64.rax, recordExternalAddress(crb, ctable)); + masm.addq(AMD64.rax, AMD64.rdx); + masm.mulpd(xmm6, xmm1); + masm.mulsd(xmm1, recordExternalAddress(crb, p3)); // 0x2e037073, 0x3b63198a + masm.subsd(xmm4, xmm3); + masm.movq(xmm7, new AMD64Address(AMD64.rax, 8)); + masm.subsd(xmm0, xmm3); + if (masm.supports(AMD64.CPUFeature.SSE3)) { + masm.movddup(xmm3, xmm4); + } else { + masm.movdqu(xmm3, xmm4); + masm.movlhps(xmm3, xmm3); + } + masm.subsd(xmm4, xmm6); + masm.pshufd(xmm0, xmm0, 68); + masm.movdqu(xmm2, new AMD64Address(AMD64.rax, 0)); + masm.mulpd(xmm5, xmm0); + masm.subpd(xmm0, xmm6); + masm.mulsd(xmm7, xmm4); + masm.subsd(xmm3, xmm4); + masm.mulpd(xmm5, xmm0); + masm.mulpd(xmm0, xmm0); + masm.subsd(xmm3, xmm6); + masm.movdqu(xmm6, recordExternalAddress(crb, sc2)); // 0x11111111, 0x3f811111, + // 0x55555555, 0x3fa55555 + masm.subsd(xmm1, xmm3); + masm.movq(xmm3, new AMD64Address(AMD64.rax, 24)); + masm.addsd(xmm2, xmm3); + masm.subsd(xmm7, xmm2); + masm.mulsd(xmm2, xmm4); + masm.mulpd(xmm6, xmm0); + masm.mulsd(xmm3, xmm4); + masm.mulpd(xmm2, xmm0); + masm.mulpd(xmm0, xmm0); + masm.addpd(xmm5, recordExternalAddress(crb, sc3)); // 0x1a01a01a, 0xbf2a01a0, + // 0x16c16c17, 0xbf56c16c + masm.mulsd(xmm4, new AMD64Address(AMD64.rax, 0)); + masm.addpd(xmm6, recordExternalAddress(crb, sc1)); // 0x55555555, 0xbfc55555, + // 0x00000000, 0xbfe00000 + masm.mulpd(xmm5, xmm0); + masm.movdqu(xmm0, xmm3); + masm.addsd(xmm3, new AMD64Address(AMD64.rax, 8)); + masm.mulpd(xmm1, xmm7); + masm.movdqu(xmm7, xmm4); + masm.addsd(xmm4, xmm3); + masm.addpd(xmm6, xmm5); + masm.movq(xmm5, new AMD64Address(AMD64.rax, 8)); + masm.subsd(xmm5, xmm3); + masm.subsd(xmm3, xmm4); + masm.addsd(xmm1, new AMD64Address(AMD64.rax, 16)); + masm.mulpd(xmm6, xmm2); + masm.addsd(xmm5, xmm0); + masm.addsd(xmm3, xmm7); + masm.addsd(xmm1, xmm5); + masm.addsd(xmm1, xmm3); + masm.addsd(xmm1, xmm6); + masm.unpckhpd(xmm6, xmm6); + masm.movdqu(xmm0, xmm4); + masm.addsd(xmm1, xmm6); + masm.addsd(xmm0, xmm1); + masm.jmp(block14); + + masm.bind(block0); + masm.jcc(ConditionFlag.Greater, block1); + masm.shrl(rax, 20); + masm.cmpl(rax, 3325); + masm.jcc(ConditionFlag.NotEqual, block2); + masm.mulsd(xmm0, recordExternalAddress(crb, allOnes)); // 0xffffffff, 0x3fefffff + masm.jmp(block14); + + masm.bind(block2); + masm.movq(xmm3, recordExternalAddress(crb, twoPow55)); // 0x00000000, 0x43600000 + masm.mulsd(xmm3, xmm0); + masm.subsd(xmm3, xmm0); + masm.mulsd(xmm3, recordExternalAddress(crb, twoPowM55)); // 0x00000000, 0x3c800000 + masm.jmp(block14); + + masm.bind(block1); + masm.pextrw(rax, xmm0, 3); + masm.andl(rax, 32752); + masm.cmpl(rax, 32752); + masm.jcc(ConditionFlag.Equal, block3); + masm.pextrw(rcx, xmm0, 3); + masm.andl(rcx, 32752); + masm.subl(rcx, 16224); + masm.shrl(rcx, 7); + masm.andl(rcx, 65532); + masm.leaq(r11, recordExternalAddress(crb, piInvTable)); + masm.addq(AMD64.rcx, r11); + masm.movdq(AMD64.rax, xmm0); + masm.movl(r10, new AMD64Address(AMD64.rcx, 20)); + masm.movl(r8, new AMD64Address(AMD64.rcx, 24)); + masm.movl(rdx, rax); + masm.shrq(AMD64.rax, 21); + masm.orl(rax, Integer.MIN_VALUE); + masm.shrl(rax, 11); + masm.movl(r9, r10); + masm.imulq(r10, AMD64.rdx); + masm.imulq(r9, AMD64.rax); + masm.imulq(r8, AMD64.rax); + masm.movl(rsi, new AMD64Address(AMD64.rcx, 16)); + masm.movl(rdi, new AMD64Address(AMD64.rcx, 12)); + masm.movl(r11, r10); + masm.shrq(r10, 32); + masm.addq(r9, r10); + masm.addq(r11, r8); + masm.movl(r8, r11); + masm.shrq(r11, 32); + masm.addq(r9, r11); + masm.movl(r10, rsi); + masm.imulq(rsi, AMD64.rdx); + masm.imulq(r10, AMD64.rax); + masm.movl(r11, rdi); + masm.imulq(rdi, AMD64.rdx); + masm.movl(rbx, rsi); + masm.shrq(rsi, 32); + masm.addq(r9, AMD64.rbx); + masm.movl(rbx, r9); + masm.shrq(r9, 32); + masm.addq(r10, rsi); + masm.addq(r10, r9); + masm.shlq(AMD64.rbx, 32); + masm.orq(r8, AMD64.rbx); + masm.imulq(r11, AMD64.rax); + masm.movl(r9, new AMD64Address(AMD64.rcx, 8)); + masm.movl(rsi, new AMD64Address(AMD64.rcx, 4)); + masm.movl(rbx, rdi); + masm.shrq(rdi, 32); + masm.addq(r10, AMD64.rbx); + masm.movl(rbx, r10); + masm.shrq(r10, 32); + masm.addq(r11, rdi); + masm.addq(r11, r10); + masm.movq(rdi, r9); + masm.imulq(r9, AMD64.rdx); + masm.imulq(rdi, AMD64.rax); + masm.movl(r10, r9); + masm.shrq(r9, 32); + masm.addq(r11, r10); + masm.movl(r10, r11); + masm.shrq(r11, 32); + masm.addq(rdi, r9); + masm.addq(rdi, r11); + masm.movq(r9, rsi); + masm.imulq(rsi, AMD64.rdx); + masm.imulq(r9, AMD64.rax); + masm.shlq(r10, 32); + masm.orq(r10, AMD64.rbx); + masm.movl(rax, new AMD64Address(AMD64.rcx, 0)); + masm.movl(r11, rsi); + masm.shrq(rsi, 32); + masm.addq(rdi, r11); + masm.movl(r11, rdi); + masm.shrq(rdi, 32); + masm.addq(r9, rsi); + masm.addq(r9, rdi); + masm.imulq(AMD64.rdx, AMD64.rax); + masm.pextrw(rbx, xmm0, 3); + masm.leaq(rdi, recordExternalAddress(crb, piInvTable)); + masm.subq(AMD64.rcx, rdi); + masm.addl(rcx, rcx); + masm.addl(rcx, rcx); + masm.addl(rcx, rcx); + masm.addl(rcx, 19); + masm.movl(rsi, 32768); + masm.andl(rsi, rbx); + masm.shrl(rbx, 4); + masm.andl(rbx, 2047); + masm.subl(rbx, 1023); + masm.subl(rcx, rbx); + masm.addq(r9, AMD64.rdx); + masm.movl(rdx, rcx); + masm.addl(rdx, 32); + masm.cmpl(rcx, 1); + masm.jcc(ConditionFlag.Less, block4); + masm.negl(rcx); + masm.addl(rcx, 29); + masm.shll(r9); + masm.movl(rdi, r9); + masm.andl(r9, 536870911); + masm.testl(r9, 268435456); + masm.jcc(ConditionFlag.NotEqual, block5); + masm.shrl(r9); + masm.movl(rbx, 0); + masm.shlq(r9, 32); + masm.orq(r9, r11); + + masm.bind(block6); + + masm.bind(block7); + + masm.cmpq(r9, 0); + masm.jcc(ConditionFlag.Equal, block8); + + masm.bind(block9); + masm.bsrq(r11, r9); + masm.movl(rcx, 29); + masm.subl(rcx, r11); + masm.jcc(ConditionFlag.LessEqual, block10); + masm.shlq(r9); + masm.movq(AMD64.rax, r10); + masm.shlq(r10); + masm.addl(rdx, rcx); + masm.negl(rcx); + masm.addl(rcx, 64); + masm.shrq(AMD64.rax); + masm.shrq(r8); + masm.orq(r9, AMD64.rax); + masm.orq(r10, r8); + + masm.bind(block11); + masm.cvtsi2sdq(xmm0, r9); + masm.shrq(r10, 1); + masm.cvtsi2sdq(xmm3, r10); + masm.xorpd(xmm4, xmm4); + masm.shll(rdx, 4); + masm.negl(rdx); + masm.addl(rdx, 16368); + masm.orl(rdx, rsi); + masm.xorl(rdx, rbx); + masm.pinsrw(xmm4, rdx, 3); + masm.movq(xmm2, recordExternalAddress(crb, pi4)); // 0x40000000, 0x3fe921fb, + // 0x18469899, 0x3e64442d + masm.movq(xmm6, recordExternalAddress(crb, pi48)); // 0x3fe921fb, 0x18469899, + // 0x3e64442d + masm.xorpd(xmm5, xmm5); + masm.subl(rdx, 1008); + masm.pinsrw(xmm5, rdx, 3); + masm.mulsd(xmm0, xmm4); + masm.shll(rsi, 16); + masm.sarl(rsi, 31); + masm.mulsd(xmm3, xmm5); + masm.movdqu(xmm1, xmm0); + masm.mulsd(xmm0, xmm2); + masm.shrl(rdi, 29); + masm.addsd(xmm1, xmm3); + masm.mulsd(xmm3, xmm2); + masm.addl(rdi, rsi); + masm.xorl(rdi, rsi); + masm.mulsd(xmm6, xmm1); + masm.movl(rax, rdi); + masm.addsd(xmm6, xmm3); + masm.movdqu(xmm2, xmm0); + masm.addsd(xmm0, xmm6); + masm.subsd(xmm2, xmm0); + masm.addsd(xmm6, xmm2); + + masm.bind(block12); + masm.movq(xmm1, recordExternalAddress(crb, pi32Inv)); // 0x6dc9c883, 0x40245f30 + masm.mulsd(xmm1, xmm0); + masm.movq(xmm5, recordExternalAddress(crb, onehalf)); // 0x00000000, 0x3fe00000, + // 0x00000000, 0x3fe00000 + masm.movq(xmm4, recordExternalAddress(crb, signMask)); // 0x00000000, 0x80000000 + masm.pand(xmm4, xmm0); + masm.por(xmm5, xmm4); + masm.addpd(xmm1, xmm5); + masm.cvttsd2sil(rdx, xmm1); + masm.cvtsi2sdl(xmm1, rdx); + masm.movq(xmm3, recordExternalAddress(crb, p1)); // 0x54400000, 0x3fb921fb + masm.movdqu(xmm2, recordExternalAddress(crb, p2)); // 0x1a600000, 0x3d90b461, + // 0x1a600000, 0x3d90b461 + masm.mulsd(xmm3, xmm1); + masm.unpcklpd(xmm1, xmm1); + masm.shll(rax, 3); + masm.addl(rdx, 1865216); + masm.movdqu(xmm4, xmm0); + masm.addl(rdx, rax); + masm.andl(rdx, 63); + masm.movdqu(xmm5, recordExternalAddress(crb, sc4)); // 0x54400000, 0x3fb921fb + masm.leaq(AMD64.rax, recordExternalAddress(crb, ctable)); + masm.shll(rdx, 5); + masm.addq(AMD64.rax, AMD64.rdx); + masm.mulpd(xmm2, xmm1); + masm.subsd(xmm0, xmm3); + masm.mulsd(xmm1, recordExternalAddress(crb, p3)); // 0x2e037073, 0x3b63198a + masm.subsd(xmm4, xmm3); + masm.movq(xmm7, new AMD64Address(AMD64.rax, 8)); + masm.unpcklpd(xmm0, xmm0); + masm.movdqu(xmm3, xmm4); + masm.subsd(xmm4, xmm2); + masm.mulpd(xmm5, xmm0); + masm.subpd(xmm0, xmm2); + masm.mulsd(xmm7, xmm4); + masm.subsd(xmm3, xmm4); + masm.mulpd(xmm5, xmm0); + masm.mulpd(xmm0, xmm0); + masm.subsd(xmm3, xmm2); + masm.movdqu(xmm2, new AMD64Address(AMD64.rax, 0)); + masm.subsd(xmm1, xmm3); + masm.movq(xmm3, new AMD64Address(AMD64.rax, 24)); + masm.addsd(xmm2, xmm3); + masm.subsd(xmm7, xmm2); + masm.subsd(xmm1, xmm6); + masm.movdqu(xmm6, recordExternalAddress(crb, sc2)); // 0x11111111, 0x3f811111, + // 0x55555555, 0x3fa55555 + masm.mulsd(xmm2, xmm4); + masm.mulpd(xmm6, xmm0); + masm.mulsd(xmm3, xmm4); + masm.mulpd(xmm2, xmm0); + masm.mulpd(xmm0, xmm0); + masm.addpd(xmm5, recordExternalAddress(crb, sc3)); // 0x1a01a01a, 0xbf2a01a0, + // 0x16c16c17, 0xbf56c16c + masm.mulsd(xmm4, new AMD64Address(AMD64.rax, 0)); + masm.addpd(xmm6, recordExternalAddress(crb, sc1)); // 0x55555555, 0xbfc55555, + // 0x00000000, 0xbfe00000 + masm.mulpd(xmm5, xmm0); + masm.movdqu(xmm0, xmm3); + masm.addsd(xmm3, new AMD64Address(AMD64.rax, 8)); + masm.mulpd(xmm1, xmm7); + masm.movdqu(xmm7, xmm4); + masm.addsd(xmm4, xmm3); + masm.addpd(xmm6, xmm5); + masm.movq(xmm5, new AMD64Address(AMD64.rax, 8)); + masm.subsd(xmm5, xmm3); + masm.subsd(xmm3, xmm4); + masm.addsd(xmm1, new AMD64Address(AMD64.rax, 16)); + masm.mulpd(xmm6, xmm2); + masm.addsd(xmm5, xmm0); + masm.addsd(xmm3, xmm7); + masm.addsd(xmm1, xmm5); + masm.addsd(xmm1, xmm3); + masm.addsd(xmm1, xmm6); + masm.unpckhpd(xmm6, xmm6); + masm.movdqu(xmm0, xmm4); + masm.addsd(xmm1, xmm6); + masm.addsd(xmm0, xmm1); + masm.jmp(block14); + + masm.bind(block8); + masm.addl(rdx, 64); + masm.movq(r9, r10); + masm.movq(r10, r8); + masm.movl(r8, 0); + masm.cmpq(r9, 0); + masm.jcc(ConditionFlag.NotEqual, block9); + masm.addl(rdx, 64); + masm.movq(r9, r10); + masm.movq(r10, r8); + masm.cmpq(r9, 0); + masm.jcc(ConditionFlag.NotEqual, block9); + masm.xorpd(xmm0, xmm0); + masm.xorpd(xmm6, xmm6); + masm.jmp(block12); + + masm.bind(block10); + masm.jcc(ConditionFlag.Equal, block11); + masm.negl(rcx); + masm.shrq(r10); + masm.movq(AMD64.rax, r9); + masm.shrq(r9); + masm.subl(rdx, rcx); + masm.negl(rcx); + masm.addl(rcx, 64); + masm.shlq(AMD64.rax); + masm.orq(r10, AMD64.rax); + masm.jmp(block11); + + masm.bind(block4); + masm.negl(rcx); + masm.shlq(r9, 32); + masm.orq(r9, r11); + masm.shlq(r9); + masm.movq(rdi, r9); + masm.testl(r9, Integer.MIN_VALUE); + masm.jcc(ConditionFlag.NotEqual, block13); + masm.shrl(r9); + masm.movl(rbx, 0); + masm.shrq(rdi, 3); + masm.jmp(block7); + + masm.bind(block5); + masm.shrl(r9); + masm.movl(rbx, 536870912); + masm.shrl(rbx); + masm.shlq(r9, 32); + masm.orq(r9, r11); + masm.shlq(AMD64.rbx, 32); + masm.addl(rdi, 536870912); + masm.movl(AMD64.rcx, 0); + masm.movl(r11, 0); + masm.subq(AMD64.rcx, r8); + masm.sbbq(r11, r10); + masm.sbbq(AMD64.rbx, r9); + masm.movq(r8, AMD64.rcx); + masm.movq(r10, r11); + masm.movq(r9, AMD64.rbx); + masm.movl(rbx, 32768); + masm.jmp(block6); + + masm.bind(block13); + masm.shrl(r9); + masm.movq(AMD64.rbx, 0x100000000L); + masm.shrq(AMD64.rbx); + masm.movl(AMD64.rcx, 0); + masm.movl(r11, 0); + masm.subq(AMD64.rcx, r8); + masm.sbbq(r11, r10); + masm.sbbq(AMD64.rbx, r9); + masm.movq(r8, AMD64.rcx); + masm.movq(r10, r11); + masm.movq(r9, AMD64.rbx); + masm.movl(rbx, 32768); + masm.shrq(rdi, 3); + masm.addl(rdi, 536870912); + masm.jmp(block7); + + masm.bind(block3); + masm.movq(xmm0, new AMD64Address(rsp, 8)); + masm.mulsd(xmm0, recordExternalAddress(crb, negZero)); // 0x00000000, 0x80000000 + masm.movq(new AMD64Address(rsp, 0), xmm0); + + masm.bind(block14); + masm.addq(rsp, 16); + masm.pop(AMD64.rbx); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathTanOp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathTanOp.java new file mode 100644 index 00000000000..0e067c8644c --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64MathTanOp.java @@ -0,0 +1,1119 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, Intel Corporation. All rights reserved. + * Intel Math Library (LIBM) Source Code + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.amd64; + +import static jdk.vm.ci.amd64.AMD64.r10; +import static jdk.vm.ci.amd64.AMD64.r11; +import static jdk.vm.ci.amd64.AMD64.r8; +import static jdk.vm.ci.amd64.AMD64.r9; +import static jdk.vm.ci.amd64.AMD64.rax; +import static jdk.vm.ci.amd64.AMD64.rbx; +import static jdk.vm.ci.amd64.AMD64.rcx; +import static jdk.vm.ci.amd64.AMD64.rdi; +import static jdk.vm.ci.amd64.AMD64.rdx; +import static jdk.vm.ci.amd64.AMD64.rsi; +import static jdk.vm.ci.amd64.AMD64.rsp; +import static jdk.vm.ci.amd64.AMD64.xmm0; +import static jdk.vm.ci.amd64.AMD64.xmm1; +import static jdk.vm.ci.amd64.AMD64.xmm2; +import static jdk.vm.ci.amd64.AMD64.xmm3; +import static jdk.vm.ci.amd64.AMD64.xmm4; +import static jdk.vm.ci.amd64.AMD64.xmm5; +import static jdk.vm.ci.amd64.AMD64.xmm6; +import static jdk.vm.ci.amd64.AMD64.xmm7; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.pointerConstant; +import static org.graalvm.compiler.lir.amd64.AMD64HotSpotHelper.recordExternalAddress; + +import org.graalvm.compiler.asm.Label; +import org.graalvm.compiler.asm.amd64.AMD64Address; +import org.graalvm.compiler.asm.amd64.AMD64Assembler; +import org.graalvm.compiler.asm.amd64.AMD64MacroAssembler; +import org.graalvm.compiler.lir.LIRInstructionClass; +import org.graalvm.compiler.lir.asm.ArrayDataPointerConstant; +import org.graalvm.compiler.lir.asm.CompilationResultBuilder; + +import jdk.vm.ci.amd64.AMD64; + +/** + * 
+ *                     ALGORITHM DESCRIPTION - TAN()
+ *                     ---------------------
+ *
+ * Polynomials coefficients and other constants.
+ *
+ * Note that in this algorithm, there is a different polynomial for
+ * each breakpoint, so there are 32 sets of polynomial coefficients
+ * as well as 32 instances of the other constants.
+ *
+ * The polynomial coefficients and constants are offset from the start
+ * of the main block as follows:
+ *
+ *   0:  c8 | c0
+ *  16:  c9 | c1
+ *  32: c10 | c2
+ *  48: c11 | c3
+ *  64: c12 | c4
+ *  80: c13 | c5
+ *  96: c14 | c6
+ * 112: c15 | c7
+ * 128: T_hi
+ * 136: T_lo
+ * 144: Sigma
+ * 152: T_hl
+ * 160: Tau
+ * 168: Mask
+ * 176: (end of block)
+ *
+ * The total table size is therefore 5632 bytes.
+ *
+ * Note that c0 and c1 are always zero. We could try storing
+ * other constants here, and just loading the low part of the
+ * SIMD register in these cases, after ensuring the high part
+ * is zero.
+ *
+ * The higher terms of the polynomial are computed in the *low*
+ * part of the SIMD register. This is so we can overlap the
+ * multiplication by r^8 and the unpacking of the other part.
+ *
+ * The constants are:
+ * T_hi + T_lo = accurate constant term in power series
+ * Sigma + T_hl = accurate coefficient of r in power series (Sigma=1 bit)
+ * Tau = multiplier for the reciprocal, always -1 or 0
+ *
+ * The basic reconstruction formula using these constants is:
+ *
+ * High = tau * recip_hi + t_hi
+ * Med = (sgn * r + t_hl * r)_hi
+ * Low = (sgn * r + t_hl * r)_lo +
+ *       tau * recip_lo + T_lo + (T_hl + sigma) * c + pol
+ *
+ * where pol = c0 + c1 * r + c2 * r^2 + ... + c15 * r^15
+ *
+ * (c0 = c1 = 0, but using them keeps SIMD regularity)
+ *
+ * We then do a compensated sum High + Med, add the low parts together
+ * and then do the final sum.
+ *
+ * Here recip_hi + recip_lo is an accurate reciprocal of the remainder
+ * modulo pi/2
+ *
+ * Special cases:
+ *  tan(NaN) = quiet NaN, and raise invalid exception
+ *  tan(INF) = NaN and raise invalid exception
+ *  tan(+/-0) = +/-0
+ *
+ */ +public final class AMD64MathTanOp extends AMD64MathIntrinsicUnaryOp { + + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64MathTanOp.class); + + public AMD64MathTanOp() { + super(TYPE, /* GPR */ rax, rcx, rdx, rbx, rsi, rdi, r8, r9, r10, r11, + /* XMM */ xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7); + } + + private ArrayDataPointerConstant onehalf = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x3fe00000, 0x00000000, 0x3fe00000 + // @formatter:on + }); + + private ArrayDataPointerConstant mul16 = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x40300000, 0x00000000, 0x3ff00000 + // @formatter:on + }); + + private ArrayDataPointerConstant signMask = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x80000000, 0x00000000, 0x80000000 + // @formatter:on + }); + + private ArrayDataPointerConstant pi32Inv = pointerConstant(16, new int[]{ + // @formatter:off + 0x6dc9c883, 0x3fe45f30, 0x6dc9c883, 0x40245f30 + // @formatter:on + }); + + private ArrayDataPointerConstant p1 = pointerConstant(16, new int[]{ + // @formatter:off + 0x54444000, 0x3fb921fb, 0x54440000, 0x3fb921fb + // @formatter:on + }); + + private ArrayDataPointerConstant p2 = pointerConstant(16, new int[]{ + // @formatter:off + 0x67674000, 0xbd32e7b9, 0x4c4c0000, 0x3d468c23 + // @formatter:on + }); + + private ArrayDataPointerConstant p3 = pointerConstant(16, new int[]{ + // @formatter:off + 0x3707344a, 0x3aa8a2e0, 0x03707345, 0x3ae98a2e + // @formatter:on + }); + + private ArrayDataPointerConstant ctable = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x882c10fa, + 0x3f9664f4, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x55e6c23d, 0x3f8226e3, 0x55555555, + 0x3fd55555, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x0e157de0, 0x3f6d6d3d, 0x11111111, 0x3fc11111, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x452b75e3, 0x3f57da36, + 0x1ba1ba1c, 0x3faba1ba, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x3ff00000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x4e435f9b, + 0x3f953f83, 0x00000000, 0x00000000, 0x3c6e8e46, 0x3f9b74ea, + 0x00000000, 0x00000000, 0xda5b7511, 0x3f85ad63, 0xdc230b9b, + 0x3fb97558, 0x26cb3788, 0x3f881308, 0x76fc4985, 0x3fd62ac9, + 0x77bb08ba, 0x3f757c85, 0xb6247521, 0x3fb1381e, 0x5922170c, + 0x3f754e95, 0x8746482d, 0x3fc27f83, 0x11055b30, 0x3f64e391, + 0x3e666320, 0x3fa3e609, 0x0de9dae3, 0x3f6301df, 0x1f1dca06, + 0x3fafa8ae, 0x8c5b2da2, 0x3fb936bb, 0x4e88f7a5, 0x3c587d05, + 0x00000000, 0x3ff00000, 0xa8935dd9, 0x3f83dde2, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x5a279ea3, 0x3faa3407, + 0x00000000, 0x00000000, 0x432d65fa, 0x3fa70153, 0x00000000, + 0x00000000, 0x891a4602, 0x3f9d03ef, 0xd62ca5f8, 0x3fca77d9, + 0xb35f4628, 0x3f97a265, 0x433258fa, 0x3fd8cf51, 0xb58fd909, + 0x3f8f88e3, 0x01771cea, 0x3fc2b154, 0xf3562f8e, 0x3f888f57, + 0xc028a723, 0x3fc7370f, 0x20b7f9f0, 0x3f80f44c, 0x214368e9, + 0x3fb6dfaa, 0x28891863, 0x3f79b4b6, 0x172dbbf0, 0x3fb6cb8e, + 0xe0553158, 0x3fc975f5, 0x593fe814, 0x3c2ef5d3, 0x00000000, + 0x3ff00000, 0x03dec550, 0x3fa44203, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x9314533e, 0x3fbb8ec5, 0x00000000, + 0x00000000, 0x09aa36d0, 0x3fb6d3f4, 0x00000000, 0x00000000, + 0xdcb427fd, 0x3fb13950, 0xd87ab0bb, 0x3fd5335e, 0xce0ae8a5, + 0x3fabb382, 0x79143126, 0x3fddba41, 0x5f2b28d4, 0x3fa552f1, + 0x59f21a6d, 0x3fd015ab, 0x22c27d95, 0x3fa0e984, 0xe19fc6aa, + 0x3fd0576c, 0x8f2c2950, 0x3f9a4898, 0xc0b3f22c, 0x3fc59462, + 0x1883a4b8, 0x3f94b61c, 0x3f838640, 0x3fc30eb8, 0x355c63dc, + 0x3fd36a08, 0x1dce993d, 0xbc6d704d, 0x00000000, 0x3ff00000, + 0x2b82ab63, 0x3fb78e92, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x56f37042, 0x3fccfc56, 0x00000000, 0x00000000, + 0xaa563951, 0x3fc90125, 0x00000000, 0x00000000, 0x3d0e7c5d, + 0x3fc50533, 0x9bed9b2e, 0x3fdf0ed9, 0x5fe7c47c, 0x3fc1f250, + 0x96c125e5, 0x3fe2edd9, 0x5a02bbd8, 0x3fbe5c71, 0x86362c20, + 0x3fda08b7, 0x4b4435ed, 0x3fb9d342, 0x4b494091, 0x3fd911bd, + 0xb56658be, 0x3fb5e4c7, 0x93a2fd76, 0x3fd3c092, 0xda271794, + 0x3fb29910, 0x3303df2b, 0x3fd189be, 0x99fcef32, 0x3fda8279, + 0xb68c1467, 0x3c708b2f, 0x00000000, 0x3ff00000, 0x980c4337, + 0x3fc5f619, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0xcc03e501, 0x3fdff10f, 0x00000000, 0x00000000, 0x44a4e845, + 0x3fddb63b, 0x00000000, 0x00000000, 0x3768ad9f, 0x3fdb72a4, + 0x3dd01cca, 0x3fe5fdb9, 0xa61d2811, 0x3fd972b2, 0x5645ad0b, + 0x3fe977f9, 0xd013b3ab, 0x3fd78ca3, 0xbf0bf914, 0x3fe4f192, + 0x4d53e730, 0x3fd5d060, 0x3f8b9000, 0x3fe49933, 0xe2b82f08, + 0x3fd4322a, 0x5936a835, 0x3fe27ae1, 0xb1c61c9b, 0x3fd2b3fb, + 0xef478605, 0x3fe1659e, 0x190834ec, 0x3fe11ab7, 0xcdb625ea, + 0xbc8e564b, 0x00000000, 0x3ff00000, 0xb07217e3, 0x3fd248f1, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x2b2c49d0, + 0x3ff2de9c, 0x00000000, 0x00000000, 0x2655bc98, 0x3ff33e58, + 0x00000000, 0x00000000, 0xff691fa2, 0x3ff3972e, 0xe93463bd, + 0x3feeed87, 0x070e10a0, 0x3ff3f5b2, 0xf4d790a4, 0x3ff20c10, + 0xa04e8ea3, 0x3ff4541a, 0x386accd3, 0x3ff1369e, 0x222a66dd, + 0x3ff4b521, 0x22a9777e, 0x3ff20817, 0x52a04a6e, 0x3ff5178f, + 0xddaa0031, 0x3ff22137, 0x4447d47c, 0x3ff57c01, 0x1e9c7f1d, + 0x3ff29311, 0x2ab7f990, 0x3fe561b8, 0x209c7df1, 0x3c87a8c5, + 0x00000000, 0x3ff00000, 0x4170bcc6, 0x3fdc92d8, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0xc7ab4d5a, 0x40085e24, + 0x00000000, 0x00000000, 0xe93ea75d, 0x400b963d, 0x00000000, + 0x00000000, 0x94a7f25a, 0x400f37e2, 0x4b6261cb, 0x3ff5f984, + 0x5a9dd812, 0x4011aab0, 0x74c30018, 0x3ffaf5a5, 0x7f2ce8e3, + 0x4013fe8b, 0xfe8e54fa, 0x3ffd7334, 0x670d618d, 0x4016a10c, + 0x4db97058, 0x4000e012, 0x24df44dd, 0x40199c5f, 0x697d6ece, + 0x4003006e, 0x83298b82, 0x401cfc4d, 0x19d490d6, 0x40058c19, + 0x2ae42850, 0x3fea4300, 0x118e20e6, 0xbc7a6db8, 0x00000000, + 0x40000000, 0xe33345b8, 0xbfd4e526, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x65965966, 0x40219659, 0x00000000, + 0x00000000, 0x882c10fa, 0x402664f4, 0x00000000, 0x00000000, + 0x83cd3723, 0x402c8342, 0x00000000, 0x40000000, 0x55e6c23d, + 0x403226e3, 0x55555555, 0x40055555, 0x34451939, 0x40371c96, + 0xaaaaaaab, 0x400aaaaa, 0x0e157de0, 0x403d6d3d, 0x11111111, + 0x40111111, 0xa738201f, 0x4042bbce, 0x05b05b06, 0x4015b05b, + 0x452b75e3, 0x4047da36, 0x1ba1ba1c, 0x401ba1ba, 0x00000000, + 0x3ff00000, 0x00000000, 0x00000000, 0x00000000, 0x40000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x4f48b8d3, 0xbf33eaf9, 0x00000000, 0x00000000, + 0x0cf7586f, 0x3f20b8ea, 0x00000000, 0x00000000, 0xd0258911, + 0xbf0abaf3, 0x23e49fe9, 0xbfab5a8c, 0x2d53222e, 0x3ef60d15, + 0x21169451, 0x3fa172b2, 0xbb254dbc, 0xbee1d3b5, 0xdbf93b8e, + 0xbf84c7db, 0x05b4630b, 0x3ecd3364, 0xee9aada7, 0x3f743924, + 0x794a8297, 0xbeb7b7b9, 0xe015f797, 0xbf5d41f5, 0xe41a4a56, + 0x3ea35dfb, 0xe4c2a251, 0x3f49a2ab, 0x5af9e000, 0xbfce49ce, + 0x8c743719, 0x3d1eb860, 0x00000000, 0x00000000, 0x1b4863cf, + 0x3fd78294, 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, + 0x535ad890, 0xbf2b9320, 0x00000000, 0x00000000, 0x018fdf1f, + 0x3f16d61d, 0x00000000, 0x00000000, 0x0359f1be, 0xbf0139e4, + 0xa4317c6d, 0xbfa67e17, 0x82672d0f, 0x3eebb405, 0x2f1b621e, + 0x3f9f455b, 0x51ccf238, 0xbed55317, 0xf437b9ac, 0xbf804bee, + 0xc791a2b5, 0x3ec0e993, 0x919a1db2, 0x3f7080c2, 0x336a5b0e, + 0xbeaa48a2, 0x0a268358, 0xbf55a443, 0xdfd978e4, 0x3e94b61f, + 0xd7767a58, 0x3f431806, 0x2aea0000, 0xbfc9bbe8, 0x7723ea61, + 0xbd3a2369, 0x00000000, 0x00000000, 0xdf7796ff, 0x3fd6e642, + 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, 0xb9ff07ce, + 0xbf231c78, 0x00000000, 0x00000000, 0xa5517182, 0x3f0ff0e0, + 0x00000000, 0x00000000, 0x790b4cbc, 0xbef66191, 0x848a46c6, + 0xbfa21ac0, 0xb16435fa, 0x3ee1d3ec, 0x2a1aa832, 0x3f9c71ea, + 0xfdd299ef, 0xbec9dd1a, 0x3f8dbaaf, 0xbf793363, 0x309fc6ea, + 0x3eb415d6, 0xbee60471, 0x3f6b83ba, 0x94a0a697, 0xbe9dae11, + 0x3e5c67b3, 0xbf4fd07b, 0x9a8f3e3e, 0x3e86bd75, 0xa4beb7a4, + 0x3f3d1eb1, 0x29cfc000, 0xbfc549ce, 0xbf159358, 0xbd397b33, + 0x00000000, 0x00000000, 0x871fee6c, 0x3fd666f0, 0x00000000, + 0x3ff00000, 0x00000000, 0xfffffff8, 0x7d98a556, 0xbf1a3958, + 0x00000000, 0x00000000, 0x9d88dc01, 0x3f0704c2, 0x00000000, + 0x00000000, 0x73742a2b, 0xbeed054a, 0x58844587, 0xbf9c2a13, + 0x55688a79, 0x3ed7a326, 0xee33f1d6, 0x3f9a48f4, 0xa8dc9888, + 0xbebf8939, 0xaad4b5b8, 0xbf72f746, 0x9102efa1, 0x3ea88f82, + 0xdabc29cf, 0x3f678228, 0x9289afb8, 0xbe90f456, 0x741fb4ed, + 0xbf46f3a3, 0xa97f6663, 0x3e79b4bf, 0xca89ff3f, 0x3f36db70, + 0xa8a2a000, 0xbfc0ee13, 0x3da24be1, 0xbd338b9f, 0x00000000, + 0x00000000, 0x11cd6c69, 0x3fd601fd, 0x00000000, 0x3ff00000, + 0x00000000, 0xfffffff8, 0x1a154b97, 0xbf116b01, 0x00000000, + 0x00000000, 0x2d427630, 0x3f0147bf, 0x00000000, 0x00000000, + 0xb93820c8, 0xbee264d4, 0xbb6cbb18, 0xbf94ab8c, 0x888d4d92, + 0x3ed0568b, 0x60730f7c, 0x3f98b19b, 0xe4b1fb11, 0xbeb2f950, + 0x22cf9f74, 0xbf6b21cd, 0x4a3ff0a6, 0x3e9f499e, 0xfd2b83ce, + 0x3f64aad7, 0x637b73af, 0xbe83487c, 0xe522591a, 0xbf3fc092, + 0xa158e8bc, 0x3e6e3aae, 0xe5e82ffa, 0x3f329d2f, 0xd636a000, + 0xbfb9477f, 0xc2c2d2bc, 0xbd135ef9, 0x00000000, 0x00000000, + 0xf2fdb123, 0x3fd5b566, 0x00000000, 0x3ff00000, 0x00000000, + 0xfffffff8, 0xc41acb64, 0xbf05448d, 0x00000000, 0x00000000, + 0xdbb03d6f, 0x3efb7ad2, 0x00000000, 0x00000000, 0x9e42962d, + 0xbed5aea5, 0x2579f8ef, 0xbf8b2398, 0x288a1ed9, 0x3ec81441, + 0xb0198dc5, 0x3f979a3a, 0x2fdfe253, 0xbea57cd3, 0x5766336f, + 0xbf617caa, 0x600944c3, 0x3e954ed6, 0xa4e0aaf8, 0x3f62c646, + 0x6b8fb29c, 0xbe74e3a3, 0xdc4c0409, 0xbf33f952, 0x9bffe365, + 0x3e6301ec, 0xb8869e44, 0x3f2fc566, 0xe1e04000, 0xbfb0cc62, + 0x016b907f, 0xbd119cbc, 0x00000000, 0x00000000, 0xe6b9d8fa, + 0x3fd57fb3, 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, + 0x5daf22a6, 0xbef429d7, 0x00000000, 0x00000000, 0x06bca545, + 0x3ef7a27d, 0x00000000, 0x00000000, 0x7211c19a, 0xbec41c3e, + 0x956ed53e, 0xbf7ae3f4, 0xee750e72, 0x3ec3901b, 0x91d443f5, + 0x3f96f713, 0x36661e6c, 0xbe936e09, 0x506f9381, 0xbf5122e8, + 0xcb6dd43f, 0x3e9041b9, 0x6698b2ff, 0x3f61b0c7, 0x576bf12b, + 0xbe625a8a, 0xe5a0e9dc, 0xbf23499d, 0x110384dd, 0x3e5b1c2c, + 0x68d43db6, 0x3f2cb899, 0x6ecac000, 0xbfa0c414, 0xcd7dd58c, + 0x3d13500f, 0x00000000, 0x00000000, 0x85a2c8fb, 0x3fd55fe0, + 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x2bf70ebe, 0x3ef66a8f, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0xd644267f, 0x3ec22805, 0x16c16c17, 0x3f96c16c, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xc4e09162, + 0x3e8d6db2, 0xbc011567, 0x3f61566a, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x1f79955c, 0x3e57da4e, 0x9334ef0b, + 0x3f2bbd77, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x55555555, 0x3fd55555, 0x00000000, + 0x3ff00000, 0x00000000, 0xfffffff8, 0x5daf22a6, 0x3ef429d7, + 0x00000000, 0x00000000, 0x06bca545, 0x3ef7a27d, 0x00000000, + 0x00000000, 0x7211c19a, 0x3ec41c3e, 0x956ed53e, 0x3f7ae3f4, + 0xee750e72, 0x3ec3901b, 0x91d443f5, 0x3f96f713, 0x36661e6c, + 0x3e936e09, 0x506f9381, 0x3f5122e8, 0xcb6dd43f, 0x3e9041b9, + 0x6698b2ff, 0x3f61b0c7, 0x576bf12b, 0x3e625a8a, 0xe5a0e9dc, + 0x3f23499d, 0x110384dd, 0x3e5b1c2c, 0x68d43db6, 0x3f2cb899, + 0x6ecac000, 0x3fa0c414, 0xcd7dd58c, 0xbd13500f, 0x00000000, + 0x00000000, 0x85a2c8fb, 0x3fd55fe0, 0x00000000, 0x3ff00000, + 0x00000000, 0xfffffff8, 0xc41acb64, 0x3f05448d, 0x00000000, + 0x00000000, 0xdbb03d6f, 0x3efb7ad2, 0x00000000, 0x00000000, + 0x9e42962d, 0x3ed5aea5, 0x2579f8ef, 0x3f8b2398, 0x288a1ed9, + 0x3ec81441, 0xb0198dc5, 0x3f979a3a, 0x2fdfe253, 0x3ea57cd3, + 0x5766336f, 0x3f617caa, 0x600944c3, 0x3e954ed6, 0xa4e0aaf8, + 0x3f62c646, 0x6b8fb29c, 0x3e74e3a3, 0xdc4c0409, 0x3f33f952, + 0x9bffe365, 0x3e6301ec, 0xb8869e44, 0x3f2fc566, 0xe1e04000, + 0x3fb0cc62, 0x016b907f, 0x3d119cbc, 0x00000000, 0x00000000, + 0xe6b9d8fa, 0x3fd57fb3, 0x00000000, 0x3ff00000, 0x00000000, + 0xfffffff8, 0x1a154b97, 0x3f116b01, 0x00000000, 0x00000000, + 0x2d427630, 0x3f0147bf, 0x00000000, 0x00000000, 0xb93820c8, + 0x3ee264d4, 0xbb6cbb18, 0x3f94ab8c, 0x888d4d92, 0x3ed0568b, + 0x60730f7c, 0x3f98b19b, 0xe4b1fb11, 0x3eb2f950, 0x22cf9f74, + 0x3f6b21cd, 0x4a3ff0a6, 0x3e9f499e, 0xfd2b83ce, 0x3f64aad7, + 0x637b73af, 0x3e83487c, 0xe522591a, 0x3f3fc092, 0xa158e8bc, + 0x3e6e3aae, 0xe5e82ffa, 0x3f329d2f, 0xd636a000, 0x3fb9477f, + 0xc2c2d2bc, 0x3d135ef9, 0x00000000, 0x00000000, 0xf2fdb123, + 0x3fd5b566, 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, + 0x7d98a556, 0x3f1a3958, 0x00000000, 0x00000000, 0x9d88dc01, + 0x3f0704c2, 0x00000000, 0x00000000, 0x73742a2b, 0x3eed054a, + 0x58844587, 0x3f9c2a13, 0x55688a79, 0x3ed7a326, 0xee33f1d6, + 0x3f9a48f4, 0xa8dc9888, 0x3ebf8939, 0xaad4b5b8, 0x3f72f746, + 0x9102efa1, 0x3ea88f82, 0xdabc29cf, 0x3f678228, 0x9289afb8, + 0x3e90f456, 0x741fb4ed, 0x3f46f3a3, 0xa97f6663, 0x3e79b4bf, + 0xca89ff3f, 0x3f36db70, 0xa8a2a000, 0x3fc0ee13, 0x3da24be1, + 0x3d338b9f, 0x00000000, 0x00000000, 0x11cd6c69, 0x3fd601fd, + 0x00000000, 0x3ff00000, 0x00000000, 0xfffffff8, 0xb9ff07ce, + 0x3f231c78, 0x00000000, 0x00000000, 0xa5517182, 0x3f0ff0e0, + 0x00000000, 0x00000000, 0x790b4cbc, 0x3ef66191, 0x848a46c6, + 0x3fa21ac0, 0xb16435fa, 0x3ee1d3ec, 0x2a1aa832, 0x3f9c71ea, + 0xfdd299ef, 0x3ec9dd1a, 0x3f8dbaaf, 0x3f793363, 0x309fc6ea, + 0x3eb415d6, 0xbee60471, 0x3f6b83ba, 0x94a0a697, 0x3e9dae11, + 0x3e5c67b3, 0x3f4fd07b, 0x9a8f3e3e, 0x3e86bd75, 0xa4beb7a4, + 0x3f3d1eb1, 0x29cfc000, 0x3fc549ce, 0xbf159358, 0x3d397b33, + 0x00000000, 0x00000000, 0x871fee6c, 0x3fd666f0, 0x00000000, + 0x3ff00000, 0x00000000, 0xfffffff8, 0x535ad890, 0x3f2b9320, + 0x00000000, 0x00000000, 0x018fdf1f, 0x3f16d61d, 0x00000000, + 0x00000000, 0x0359f1be, 0x3f0139e4, 0xa4317c6d, 0x3fa67e17, + 0x82672d0f, 0x3eebb405, 0x2f1b621e, 0x3f9f455b, 0x51ccf238, + 0x3ed55317, 0xf437b9ac, 0x3f804bee, 0xc791a2b5, 0x3ec0e993, + 0x919a1db2, 0x3f7080c2, 0x336a5b0e, 0x3eaa48a2, 0x0a268358, + 0x3f55a443, 0xdfd978e4, 0x3e94b61f, 0xd7767a58, 0x3f431806, + 0x2aea0000, 0x3fc9bbe8, 0x7723ea61, 0x3d3a2369, 0x00000000, + 0x00000000, 0xdf7796ff, 0x3fd6e642, 0x00000000, 0x3ff00000, + 0x00000000, 0xfffffff8, 0x4f48b8d3, 0x3f33eaf9, 0x00000000, + 0x00000000, 0x0cf7586f, 0x3f20b8ea, 0x00000000, 0x00000000, + 0xd0258911, 0x3f0abaf3, 0x23e49fe9, 0x3fab5a8c, 0x2d53222e, + 0x3ef60d15, 0x21169451, 0x3fa172b2, 0xbb254dbc, 0x3ee1d3b5, + 0xdbf93b8e, 0x3f84c7db, 0x05b4630b, 0x3ecd3364, 0xee9aada7, + 0x3f743924, 0x794a8297, 0x3eb7b7b9, 0xe015f797, 0x3f5d41f5, + 0xe41a4a56, 0x3ea35dfb, 0xe4c2a251, 0x3f49a2ab, 0x5af9e000, + 0x3fce49ce, 0x8c743719, 0xbd1eb860, 0x00000000, 0x00000000, + 0x1b4863cf, 0x3fd78294, 0x00000000, 0x3ff00000, 0x00000000, + 0xfffffff8, 0x65965966, 0xc0219659, 0x00000000, 0x00000000, + 0x882c10fa, 0x402664f4, 0x00000000, 0x00000000, 0x83cd3723, + 0xc02c8342, 0x00000000, 0xc0000000, 0x55e6c23d, 0x403226e3, + 0x55555555, 0x40055555, 0x34451939, 0xc0371c96, 0xaaaaaaab, + 0xc00aaaaa, 0x0e157de0, 0x403d6d3d, 0x11111111, 0x40111111, + 0xa738201f, 0xc042bbce, 0x05b05b06, 0xc015b05b, 0x452b75e3, + 0x4047da36, 0x1ba1ba1c, 0x401ba1ba, 0x00000000, 0xbff00000, + 0x00000000, 0x00000000, 0x00000000, 0x40000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0xc7ab4d5a, 0xc0085e24, 0x00000000, 0x00000000, 0xe93ea75d, + 0x400b963d, 0x00000000, 0x00000000, 0x94a7f25a, 0xc00f37e2, + 0x4b6261cb, 0xbff5f984, 0x5a9dd812, 0x4011aab0, 0x74c30018, + 0x3ffaf5a5, 0x7f2ce8e3, 0xc013fe8b, 0xfe8e54fa, 0xbffd7334, + 0x670d618d, 0x4016a10c, 0x4db97058, 0x4000e012, 0x24df44dd, + 0xc0199c5f, 0x697d6ece, 0xc003006e, 0x83298b82, 0x401cfc4d, + 0x19d490d6, 0x40058c19, 0x2ae42850, 0xbfea4300, 0x118e20e6, + 0x3c7a6db8, 0x00000000, 0x40000000, 0xe33345b8, 0xbfd4e526, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x2b2c49d0, + 0xbff2de9c, 0x00000000, 0x00000000, 0x2655bc98, 0x3ff33e58, + 0x00000000, 0x00000000, 0xff691fa2, 0xbff3972e, 0xe93463bd, + 0xbfeeed87, 0x070e10a0, 0x3ff3f5b2, 0xf4d790a4, 0x3ff20c10, + 0xa04e8ea3, 0xbff4541a, 0x386accd3, 0xbff1369e, 0x222a66dd, + 0x3ff4b521, 0x22a9777e, 0x3ff20817, 0x52a04a6e, 0xbff5178f, + 0xddaa0031, 0xbff22137, 0x4447d47c, 0x3ff57c01, 0x1e9c7f1d, + 0x3ff29311, 0x2ab7f990, 0xbfe561b8, 0x209c7df1, 0xbc87a8c5, + 0x00000000, 0x3ff00000, 0x4170bcc6, 0x3fdc92d8, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0xcc03e501, 0xbfdff10f, + 0x00000000, 0x00000000, 0x44a4e845, 0x3fddb63b, 0x00000000, + 0x00000000, 0x3768ad9f, 0xbfdb72a4, 0x3dd01cca, 0xbfe5fdb9, + 0xa61d2811, 0x3fd972b2, 0x5645ad0b, 0x3fe977f9, 0xd013b3ab, + 0xbfd78ca3, 0xbf0bf914, 0xbfe4f192, 0x4d53e730, 0x3fd5d060, + 0x3f8b9000, 0x3fe49933, 0xe2b82f08, 0xbfd4322a, 0x5936a835, + 0xbfe27ae1, 0xb1c61c9b, 0x3fd2b3fb, 0xef478605, 0x3fe1659e, + 0x190834ec, 0xbfe11ab7, 0xcdb625ea, 0x3c8e564b, 0x00000000, + 0x3ff00000, 0xb07217e3, 0x3fd248f1, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x56f37042, 0xbfccfc56, 0x00000000, + 0x00000000, 0xaa563951, 0x3fc90125, 0x00000000, 0x00000000, + 0x3d0e7c5d, 0xbfc50533, 0x9bed9b2e, 0xbfdf0ed9, 0x5fe7c47c, + 0x3fc1f250, 0x96c125e5, 0x3fe2edd9, 0x5a02bbd8, 0xbfbe5c71, + 0x86362c20, 0xbfda08b7, 0x4b4435ed, 0x3fb9d342, 0x4b494091, + 0x3fd911bd, 0xb56658be, 0xbfb5e4c7, 0x93a2fd76, 0xbfd3c092, + 0xda271794, 0x3fb29910, 0x3303df2b, 0x3fd189be, 0x99fcef32, + 0xbfda8279, 0xb68c1467, 0xbc708b2f, 0x00000000, 0x3ff00000, + 0x980c4337, 0x3fc5f619, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x9314533e, 0xbfbb8ec5, 0x00000000, 0x00000000, + 0x09aa36d0, 0x3fb6d3f4, 0x00000000, 0x00000000, 0xdcb427fd, + 0xbfb13950, 0xd87ab0bb, 0xbfd5335e, 0xce0ae8a5, 0x3fabb382, + 0x79143126, 0x3fddba41, 0x5f2b28d4, 0xbfa552f1, 0x59f21a6d, + 0xbfd015ab, 0x22c27d95, 0x3fa0e984, 0xe19fc6aa, 0x3fd0576c, + 0x8f2c2950, 0xbf9a4898, 0xc0b3f22c, 0xbfc59462, 0x1883a4b8, + 0x3f94b61c, 0x3f838640, 0x3fc30eb8, 0x355c63dc, 0xbfd36a08, + 0x1dce993d, 0x3c6d704d, 0x00000000, 0x3ff00000, 0x2b82ab63, + 0x3fb78e92, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x5a279ea3, 0xbfaa3407, 0x00000000, 0x00000000, 0x432d65fa, + 0x3fa70153, 0x00000000, 0x00000000, 0x891a4602, 0xbf9d03ef, + 0xd62ca5f8, 0xbfca77d9, 0xb35f4628, 0x3f97a265, 0x433258fa, + 0x3fd8cf51, 0xb58fd909, 0xbf8f88e3, 0x01771cea, 0xbfc2b154, + 0xf3562f8e, 0x3f888f57, 0xc028a723, 0x3fc7370f, 0x20b7f9f0, + 0xbf80f44c, 0x214368e9, 0xbfb6dfaa, 0x28891863, 0x3f79b4b6, + 0x172dbbf0, 0x3fb6cb8e, 0xe0553158, 0xbfc975f5, 0x593fe814, + 0xbc2ef5d3, 0x00000000, 0x3ff00000, 0x03dec550, 0x3fa44203, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x4e435f9b, + 0xbf953f83, 0x00000000, 0x00000000, 0x3c6e8e46, 0x3f9b74ea, + 0x00000000, 0x00000000, 0xda5b7511, 0xbf85ad63, 0xdc230b9b, + 0xbfb97558, 0x26cb3788, 0x3f881308, 0x76fc4985, 0x3fd62ac9, + 0x77bb08ba, 0xbf757c85, 0xb6247521, 0xbfb1381e, 0x5922170c, + 0x3f754e95, 0x8746482d, 0x3fc27f83, 0x11055b30, 0xbf64e391, + 0x3e666320, 0xbfa3e609, 0x0de9dae3, 0x3f6301df, 0x1f1dca06, + 0x3fafa8ae, 0x8c5b2da2, 0xbfb936bb, 0x4e88f7a5, 0xbc587d05, + 0x00000000, 0x3ff00000, 0xa8935dd9, 0x3f83dde2, 0x00000000, + 0x00000000, 0x00000000, 0x00000000 + // @formatter:on + }); + + private ArrayDataPointerConstant mask35 = pointerConstant(16, new int[]{ + // @formatter:off + 0xfffc0000, 0xffffffff, 0x00000000, 0x00000000 + // @formatter:on + }); + + private ArrayDataPointerConstant q11 = pointerConstant(16, new int[]{ + // @formatter:off + 0xb8fe4d77, 0x3f82609a + // @formatter:on + }); + + private ArrayDataPointerConstant q9 = pointerConstant(16, new int[]{ + // @formatter:off + 0xbf847a43, 0x3f9664a0 + // @formatter:on + }); + + private ArrayDataPointerConstant q7 = pointerConstant(16, new int[]{ + // @formatter:off + 0x52c4c8ab, 0x3faba1ba + // @formatter:on + }); + + private ArrayDataPointerConstant q5 = pointerConstant(16, new int[]{ + // @formatter:off + 0x11092746, 0x3fc11111 + // @formatter:on + }); + + private ArrayDataPointerConstant q3 = pointerConstant(16, new int[]{ + // @formatter:off + 0x55555612, 0x3fd55555 + // @formatter:on + }); + + private ArrayDataPointerConstant piInvTable = pointerConstant(16, new int[]{ + // @formatter:off + 0x00000000, 0x00000000, 0xa2f9836e, 0x4e441529, 0xfc2757d1, + 0xf534ddc0, 0xdb629599, 0x3c439041, 0xfe5163ab, 0xdebbc561, + 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c, 0xfe1deb1c, + 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41, + 0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, + 0xde05980f, 0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, + 0x4f463f66, 0x9e5fea2d, 0x7527bac7, 0xebe5f17b, 0x3d0739f7, + 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08, 0x56033046, 0xfc7b6bab, + 0xf0cfbc21 + // @formatter:on + }); + + private ArrayDataPointerConstant pi4 = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x3fe921fb, + }); + private ArrayDataPointerConstant pi48 = pointerConstant(8, new int[]{ + 0x4611a626, 0x3e85110b + // @formatter:on + }); + + private ArrayDataPointerConstant qq2 = pointerConstant(8, new int[]{ + // @formatter:off + 0x676733af, 0x3d32e7b9 + // @formatter:on + }); + + private ArrayDataPointerConstant one = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x3ff00000 + // @formatter:on + }); + + private ArrayDataPointerConstant twoPow55 = pointerConstant(8, new int[]{ + // @formatter:off + 0x00000000, 0x43600000 + // @formatter:on + }); + + private ArrayDataPointerConstant twoPowM55 = pointerConstant(4, new int[]{ + // @formatter:off + 0x00000000, 0x3c800000 + // @formatter:on + }); + + private ArrayDataPointerConstant negZero = pointerConstant(4, new int[]{ + // @formatter:off + 0x00000000, 0x80000000 + // @formatter:on + }); + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + Label block0 = new Label(); + Label block1 = new Label(); + Label block2 = new Label(); + Label block3 = new Label(); + Label block4 = new Label(); + Label block5 = new Label(); + Label block6 = new Label(); + Label block7 = new Label(); + Label block8 = new Label(); + Label block9 = new Label(); + Label block10 = new Label(); + Label block11 = new Label(); + Label block12 = new Label(); + Label block13 = new Label(); + Label block14 = new Label(); + + masm.push(rbx); + masm.subq(rsp, 16); + masm.movsd(new AMD64Address(rsp, 8), xmm0); + + masm.pextrw(rax, xmm0, 3); + masm.andl(rax, 32767); + masm.subl(rax, 16314); + masm.cmpl(rax, 270); + masm.jcc(AMD64Assembler.ConditionFlag.Above, block0); + masm.movdqu(xmm5, recordExternalAddress(crb, onehalf)); // 0x00000000, 0x3fe00000, + // 0x00000000, 0x3fe00000 + masm.movdqu(xmm6, recordExternalAddress(crb, mul16)); // 0x00000000, 0x40300000, + // 0x00000000, 0x3ff00000 + masm.unpcklpd(xmm0, xmm0); + masm.movdqu(xmm4, recordExternalAddress(crb, signMask)); // 0x00000000, 0x80000000, + // 0x00000000, 0x80000000 + masm.andpd(xmm4, xmm0); + masm.movdqu(xmm1, recordExternalAddress(crb, pi32Inv)); // 0x6dc9c883, 0x3fe45f30, + // 0x6dc9c883, 0x40245f30 + masm.mulpd(xmm1, xmm0); + masm.por(xmm5, xmm4); + masm.addpd(xmm1, xmm5); + masm.movdqu(xmm7, xmm1); + masm.unpckhpd(xmm7, xmm7); + masm.cvttsd2sil(rdx, xmm7); + masm.cvttpd2dq(xmm1, xmm1); + masm.cvtdq2pd(xmm1, xmm1); + masm.mulpd(xmm1, xmm6); + masm.movdqu(xmm3, recordExternalAddress(crb, p1)); // 0x54444000, 0x3fb921fb, + // 0x54440000, 0x3fb921fb + masm.movq(xmm5, recordExternalAddress(crb, qq2)); // 0x676733af, 0x3d32e7b9 + masm.addq(rdx, 469248); + masm.movdqu(xmm4, recordExternalAddress(crb, p2)); // 0x67674000, 0xbd32e7b9, + // 0x4c4c0000, 0x3d468c23 + masm.mulpd(xmm3, xmm1); + masm.andq(rdx, 31); + masm.mulsd(xmm5, xmm1); + masm.movq(rcx, rdx); + masm.mulpd(xmm4, xmm1); + masm.shlq(rcx, 1); + masm.subpd(xmm0, xmm3); + masm.mulpd(xmm1, recordExternalAddress(crb, p3)); // 0x3707344a, 0x3aa8a2e0, + // 0x03707345, 0x3ae98a2e + masm.addq(rdx, rcx); + masm.shlq(rcx, 2); + masm.addq(rdx, rcx); + masm.addsd(xmm5, xmm0); + masm.movdqu(xmm2, xmm0); + masm.subpd(xmm0, xmm4); + masm.movq(xmm6, recordExternalAddress(crb, one)); // 0x00000000, 0x3ff00000 + masm.shlq(rdx, 4); + masm.leaq(rax, recordExternalAddress(crb, ctable)); + masm.andpd(xmm5, recordExternalAddress(crb, mask35)); // 0xfffc0000, 0xffffffff, + // 0x00000000, 0x00000000 + masm.movdqu(xmm3, xmm0); + masm.addq(rax, rdx); + masm.subpd(xmm2, xmm0); + masm.unpckhpd(xmm0, xmm0); + masm.divsd(xmm6, xmm5); + masm.subpd(xmm2, xmm4); + masm.movdqu(xmm7, new AMD64Address(rax, 16)); + masm.subsd(xmm3, xmm5); + masm.mulpd(xmm7, xmm0); + masm.subpd(xmm2, xmm1); + masm.movdqu(xmm1, new AMD64Address(rax, 48)); + masm.mulpd(xmm1, xmm0); + masm.movdqu(xmm4, new AMD64Address(rax, 96)); + masm.mulpd(xmm4, xmm0); + masm.addsd(xmm2, xmm3); + masm.movdqu(xmm3, xmm0); + masm.mulpd(xmm0, xmm0); + masm.addpd(xmm7, new AMD64Address(rax, 0)); + masm.addpd(xmm1, new AMD64Address(rax, 32)); + masm.mulpd(xmm1, xmm0); + masm.addpd(xmm4, new AMD64Address(rax, 80)); + masm.addpd(xmm7, xmm1); + masm.movdqu(xmm1, new AMD64Address(rax, 112)); + masm.mulpd(xmm1, xmm0); + masm.mulpd(xmm0, xmm0); + masm.addpd(xmm4, xmm1); + masm.movdqu(xmm1, new AMD64Address(rax, 64)); + masm.mulpd(xmm1, xmm0); + masm.addpd(xmm7, xmm1); + masm.movdqu(xmm1, xmm3); + masm.mulpd(xmm3, xmm0); + masm.mulsd(xmm0, xmm0); + masm.mulpd(xmm1, new AMD64Address(rax, 144)); + masm.mulpd(xmm4, xmm3); + masm.movdqu(xmm3, xmm1); + masm.addpd(xmm7, xmm4); + masm.movdqu(xmm4, xmm1); + masm.mulsd(xmm0, xmm7); + masm.unpckhpd(xmm7, xmm7); + masm.addsd(xmm0, xmm7); + masm.unpckhpd(xmm1, xmm1); + masm.addsd(xmm3, xmm1); + masm.subsd(xmm4, xmm3); + masm.addsd(xmm1, xmm4); + masm.movdqu(xmm4, xmm2); + masm.movq(xmm7, new AMD64Address(rax, 144)); + masm.unpckhpd(xmm2, xmm2); + masm.addsd(xmm7, new AMD64Address(rax, 152)); + masm.mulsd(xmm7, xmm2); + masm.addsd(xmm7, new AMD64Address(rax, 136)); + masm.addsd(xmm7, xmm1); + masm.addsd(xmm0, xmm7); + masm.movq(xmm7, recordExternalAddress(crb, one)); // 0x00000000, 0x3ff00000 + masm.mulsd(xmm4, xmm6); + masm.movq(xmm2, new AMD64Address(rax, 168)); + masm.andpd(xmm2, xmm6); + masm.mulsd(xmm5, xmm2); + masm.mulsd(xmm6, new AMD64Address(rax, 160)); + masm.subsd(xmm7, xmm5); + masm.subsd(xmm2, new AMD64Address(rax, 128)); + masm.subsd(xmm7, xmm4); + masm.mulsd(xmm7, xmm6); + masm.movdqu(xmm4, xmm3); + masm.subsd(xmm3, xmm2); + masm.addsd(xmm2, xmm3); + masm.subsd(xmm4, xmm2); + masm.addsd(xmm0, xmm4); + masm.subsd(xmm0, xmm7); + masm.addsd(xmm0, xmm3); + masm.jmp(block14); + + masm.bind(block0); + masm.jcc(AMD64Assembler.ConditionFlag.Greater, block1); + masm.pextrw(rax, xmm0, 3); + masm.movl(rdx, rax); + masm.andl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block2); + masm.andl(rdx, 32767); + masm.cmpl(rdx, 15904); + masm.jcc(AMD64Assembler.ConditionFlag.Below, block3); + masm.movdqu(xmm2, xmm0); + masm.movdqu(xmm3, xmm0); + masm.movq(xmm1, recordExternalAddress(crb, q11)); // 0xb8fe4d77, 0x3f82609a + masm.mulsd(xmm2, xmm0); + masm.mulsd(xmm3, xmm2); + masm.mulsd(xmm1, xmm2); + masm.addsd(xmm1, recordExternalAddress(crb, q9)); // 0xbf847a43, 0x3f9664a0 + masm.mulsd(xmm1, xmm2); + masm.addsd(xmm1, recordExternalAddress(crb, q7)); // 0x52c4c8ab, 0x3faba1ba + masm.mulsd(xmm1, xmm2); + masm.addsd(xmm1, recordExternalAddress(crb, q5)); // 0x11092746, 0x3fc11111 + masm.mulsd(xmm1, xmm2); + masm.addsd(xmm1, recordExternalAddress(crb, q3)); // 0x55555612, 0x3fd55555 + masm.mulsd(xmm1, xmm3); + masm.addsd(xmm0, xmm1); + masm.jmp(block14); + + masm.bind(block3); + masm.movq(xmm3, recordExternalAddress(crb, twoPow55)); // 0x00000000, 0x43600000 + masm.mulsd(xmm3, xmm0); + masm.addsd(xmm0, xmm3); + masm.mulsd(xmm0, recordExternalAddress(crb, twoPowM55)); // 0x00000000, 0x3c800000 + masm.jmp(block14); + + masm.bind(block2); + masm.movdqu(xmm1, xmm0); + masm.mulsd(xmm1, xmm1); + masm.jmp(block14); + + masm.bind(block1); + masm.pextrw(rax, xmm0, 3); + masm.andl(rax, 32752); + masm.cmpl(rax, 32752); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block4); + masm.pextrw(rcx, xmm0, 3); + masm.andl(rcx, 32752); + masm.subl(rcx, 16224); + masm.shrl(rcx, 7); + masm.andl(rcx, 65532); + masm.leaq(r11, recordExternalAddress(crb, piInvTable)); + masm.addq(rcx, r11); + masm.movdq(rax, xmm0); + masm.movl(r10, new AMD64Address(rcx, 20)); + masm.movl(r8, new AMD64Address(rcx, 24)); + masm.movl(rdx, rax); + masm.shrq(rax, 21); + masm.orl(rax, Integer.MIN_VALUE); + masm.shrl(rax, 11); + masm.movl(r9, r10); + masm.imulq(r10, rdx); + masm.imulq(r9, rax); + masm.imulq(r8, rax); + masm.movl(rsi, new AMD64Address(rcx, 16)); + masm.movl(rdi, new AMD64Address(rcx, 12)); + masm.movl(r11, r10); + masm.shrq(r10, 32); + masm.addq(r9, r10); + masm.addq(r11, r8); + masm.movl(r8, r11); + masm.shrq(r11, 32); + masm.addq(r9, r11); + masm.movl(r10, rsi); + masm.imulq(rsi, rdx); + masm.imulq(r10, rax); + masm.movl(r11, rdi); + masm.imulq(rdi, rdx); + masm.movl(rbx, rsi); + masm.shrq(rsi, 32); + masm.addq(r9, rbx); + masm.movl(rbx, r9); + masm.shrq(r9, 32); + masm.addq(r10, rsi); + masm.addq(r10, r9); + masm.shlq(rbx, 32); + masm.orq(r8, rbx); + masm.imulq(r11, rax); + masm.movl(r9, new AMD64Address(rcx, 8)); + masm.movl(rsi, new AMD64Address(rcx, 4)); + masm.movl(rbx, rdi); + masm.shrq(rdi, 32); + masm.addq(r10, rbx); + masm.movl(rbx, r10); + masm.shrq(r10, 32); + masm.addq(r11, rdi); + masm.addq(r11, r10); + masm.movq(rdi, r9); + masm.imulq(r9, rdx); + masm.imulq(rdi, rax); + masm.movl(r10, r9); + masm.shrq(r9, 32); + masm.addq(r11, r10); + masm.movl(r10, r11); + masm.shrq(r11, 32); + masm.addq(rdi, r9); + masm.addq(rdi, r11); + masm.movq(r9, rsi); + masm.imulq(rsi, rdx); + masm.imulq(r9, rax); + masm.shlq(r10, 32); + masm.orq(r10, rbx); + masm.movl(rax, new AMD64Address(rcx, 0)); + masm.movl(r11, rsi); + masm.shrq(rsi, 32); + masm.addq(rdi, r11); + masm.movl(r11, rdi); + masm.shrq(rdi, 32); + masm.addq(r9, rsi); + masm.addq(r9, rdi); + masm.imulq(rdx, rax); + masm.pextrw(rbx, xmm0, 3); + masm.leaq(rdi, recordExternalAddress(crb, piInvTable)); + masm.subq(rcx, rdi); + masm.addl(rcx, rcx); + masm.addl(rcx, rcx); + masm.addl(rcx, rcx); + masm.addl(rcx, 19); + masm.movl(rsi, 32768); + masm.andl(rsi, rbx); + masm.shrl(rbx, 4); + masm.andl(rbx, 2047); + masm.subl(rbx, 1023); + masm.subl(rcx, rbx); + masm.addq(r9, rdx); + masm.movl(rdx, rcx); + masm.addl(rdx, 32); + masm.cmpl(rcx, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Less, block5); + masm.negl(rcx); + masm.addl(rcx, 29); + masm.shll(r9); + masm.movl(rdi, r9); + masm.andl(r9, 1073741823); + masm.testl(r9, 536870912); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block6); + masm.shrl(r9); + masm.movl(rbx, 0); + masm.shlq(r9, 32); + masm.orq(r9, r11); + + masm.bind(block7); + + masm.bind(block8); + masm.cmpq(r9, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block9); + + masm.bind(block10); + masm.bsrq(r11, r9); + masm.movl(rcx, 29); + masm.subl(rcx, r11); + masm.jcc(AMD64Assembler.ConditionFlag.LessEqual, block11); + masm.shlq(r9); + masm.movq(rax, r10); + masm.shlq(r10); + masm.addl(rdx, rcx); + masm.negl(rcx); + masm.addl(rcx, 64); + masm.shrq(rax); + masm.shrq(r8); + masm.orq(r9, rax); + masm.orq(r10, r8); + + masm.bind(block12); + masm.cvtsi2sdq(xmm0, r9); + masm.shrq(r10, 1); + masm.cvtsi2sdq(xmm3, r10); + masm.xorpd(xmm4, xmm4); + masm.shll(rdx, 4); + masm.negl(rdx); + masm.addl(rdx, 16368); + masm.orl(rdx, rsi); + masm.xorl(rdx, rbx); + masm.pinsrw(xmm4, rdx, 3); + masm.movq(xmm2, recordExternalAddress(crb, pi4)); // 0x00000000, 0x3fe921fb, + // 0x4611a626, 0x3e85110b + masm.movq(xmm7, recordExternalAddress(crb, pi48)); // 0x3fe921fb, 0x4611a626, + // 0x3e85110b + masm.xorpd(xmm5, xmm5); + masm.subl(rdx, 1008); + masm.pinsrw(xmm5, rdx, 3); + masm.mulsd(xmm0, xmm4); + masm.shll(rsi, 16); + masm.sarl(rsi, 31); + masm.mulsd(xmm3, xmm5); + masm.movdqu(xmm1, xmm0); + masm.mulsd(xmm0, xmm2); + masm.shrl(rdi, 30); + masm.addsd(xmm1, xmm3); + masm.mulsd(xmm3, xmm2); + masm.addl(rdi, rsi); + masm.xorl(rdi, rsi); + masm.mulsd(xmm7, xmm1); + masm.movl(rax, rdi); + masm.addsd(xmm7, xmm3); + masm.movdqu(xmm2, xmm0); + masm.addsd(xmm0, xmm7); + masm.subsd(xmm2, xmm0); + masm.addsd(xmm7, xmm2); + masm.movdqu(xmm1, recordExternalAddress(crb, pi32Inv)); // 0x6dc9c883, 0x3fe45f30, + // 0x6dc9c883, 0x40245f30 + if (masm.supports(AMD64.CPUFeature.SSE3)) { + masm.movddup(xmm0, xmm0); + } else { + masm.movlhps(xmm0, xmm0); + } + masm.movdqu(xmm4, recordExternalAddress(crb, signMask)); // 0x00000000, 0x80000000, + // 0x00000000, 0x80000000 + masm.andpd(xmm4, xmm0); + masm.mulpd(xmm1, xmm0); + if (masm.supports(AMD64.CPUFeature.SSE3)) { + masm.movddup(xmm7, xmm7); + } else { + masm.movlhps(xmm7, xmm7); + } + masm.movdqu(xmm5, recordExternalAddress(crb, onehalf)); // 0x00000000, 0x3fe00000, + // 0x00000000, 0x3fe00000 + masm.movdqu(xmm6, recordExternalAddress(crb, mul16)); // 0x00000000, 0x40300000, + // 0x00000000, 0x3ff00000 + masm.por(xmm5, xmm4); + masm.addpd(xmm1, xmm5); + masm.movdqu(xmm5, xmm1); + masm.unpckhpd(xmm5, xmm5); + masm.cvttsd2sil(rdx, xmm5); + masm.cvttpd2dq(xmm1, xmm1); + masm.cvtdq2pd(xmm1, xmm1); + masm.mulpd(xmm1, xmm6); + masm.movdqu(xmm3, recordExternalAddress(crb, p1)); // 0x54444000, 0x3fb921fb, + // 0x54440000, 0x3fb921fb + masm.movq(xmm5, recordExternalAddress(crb, qq2)); // 0x676733af, 0x3d32e7b9 + masm.shll(rax, 4); + masm.addl(rdx, 469248); + masm.movdqu(xmm4, recordExternalAddress(crb, p2)); // 0x67674000, 0xbd32e7b9, + // 0x4c4c0000, 0x3d468c23 + masm.mulpd(xmm3, xmm1); + masm.addl(rdx, rax); + masm.andl(rdx, 31); + masm.mulsd(xmm5, xmm1); + masm.movl(rcx, rdx); + masm.mulpd(xmm4, xmm1); + masm.shll(rcx, 1); + masm.subpd(xmm0, xmm3); + masm.mulpd(xmm1, recordExternalAddress(crb, p3)); // 0x3707344a, 0x3aa8a2e0, + // 0x03707345, 0x3ae98a2e + masm.addl(rdx, rcx); + masm.shll(rcx, 2); + masm.addl(rdx, rcx); + masm.addsd(xmm5, xmm0); + masm.movdqu(xmm2, xmm0); + masm.subpd(xmm0, xmm4); + masm.movq(xmm6, recordExternalAddress(crb, one)); // 0x00000000, 0x3ff00000 + masm.shll(rdx, 4); + masm.leaq(rax, recordExternalAddress(crb, ctable)); + masm.andpd(xmm5, recordExternalAddress(crb, mask35)); // 0xfffc0000, 0xffffffff, + // 0x00000000, 0x00000000 + masm.movdqu(xmm3, xmm0); + masm.addq(rax, rdx); + masm.subpd(xmm2, xmm0); + masm.unpckhpd(xmm0, xmm0); + masm.divsd(xmm6, xmm5); + masm.subpd(xmm2, xmm4); + masm.subsd(xmm3, xmm5); + masm.subpd(xmm2, xmm1); + masm.movdqu(xmm1, new AMD64Address(rax, 48)); + masm.addpd(xmm2, xmm7); + masm.movdqu(xmm7, new AMD64Address(rax, 16)); + masm.mulpd(xmm7, xmm0); + masm.movdqu(xmm4, new AMD64Address(rax, 96)); + masm.mulpd(xmm1, xmm0); + masm.mulpd(xmm4, xmm0); + masm.addsd(xmm2, xmm3); + masm.movdqu(xmm3, xmm0); + masm.mulpd(xmm0, xmm0); + masm.addpd(xmm7, new AMD64Address(rax, 0)); + masm.addpd(xmm1, new AMD64Address(rax, 32)); + masm.mulpd(xmm1, xmm0); + masm.addpd(xmm4, new AMD64Address(rax, 80)); + masm.addpd(xmm7, xmm1); + masm.movdqu(xmm1, new AMD64Address(rax, 112)); + masm.mulpd(xmm1, xmm0); + masm.mulpd(xmm0, xmm0); + masm.addpd(xmm4, xmm1); + masm.movdqu(xmm1, new AMD64Address(rax, 64)); + masm.mulpd(xmm1, xmm0); + masm.addpd(xmm7, xmm1); + masm.movdqu(xmm1, xmm3); + masm.mulpd(xmm3, xmm0); + masm.mulsd(xmm0, xmm0); + masm.mulpd(xmm1, new AMD64Address(rax, 144)); + masm.mulpd(xmm4, xmm3); + masm.movdqu(xmm3, xmm1); + masm.addpd(xmm7, xmm4); + masm.movdqu(xmm4, xmm1); + masm.mulsd(xmm0, xmm7); + masm.unpckhpd(xmm7, xmm7); + masm.addsd(xmm0, xmm7); + masm.unpckhpd(xmm1, xmm1); + masm.addsd(xmm3, xmm1); + masm.subsd(xmm4, xmm3); + masm.addsd(xmm1, xmm4); + masm.movdqu(xmm4, xmm2); + masm.movq(xmm7, new AMD64Address(rax, 144)); + masm.unpckhpd(xmm2, xmm2); + masm.addsd(xmm7, new AMD64Address(rax, 152)); + masm.mulsd(xmm7, xmm2); + masm.addsd(xmm7, new AMD64Address(rax, 136)); + masm.addsd(xmm7, xmm1); + masm.addsd(xmm0, xmm7); + masm.movq(xmm7, recordExternalAddress(crb, one)); // 0x00000000, 0x3ff00000 + masm.mulsd(xmm4, xmm6); + masm.movq(xmm2, new AMD64Address(rax, 168)); + masm.andpd(xmm2, xmm6); + masm.mulsd(xmm5, xmm2); + masm.mulsd(xmm6, new AMD64Address(rax, 160)); + masm.subsd(xmm7, xmm5); + masm.subsd(xmm2, new AMD64Address(rax, 128)); + masm.subsd(xmm7, xmm4); + masm.mulsd(xmm7, xmm6); + masm.movdqu(xmm4, xmm3); + masm.subsd(xmm3, xmm2); + masm.addsd(xmm2, xmm3); + masm.subsd(xmm4, xmm2); + masm.addsd(xmm0, xmm4); + masm.subsd(xmm0, xmm7); + masm.addsd(xmm0, xmm3); + masm.jmp(block14); + + masm.bind(block9); + masm.addl(rdx, 64); + masm.movq(r9, r10); + masm.movq(r10, r8); + masm.movl(r8, 0); + masm.cmpq(r9, 0); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block10); + masm.addl(rdx, 64); + masm.movq(r9, r10); + masm.movq(r10, r8); + masm.cmpq(r9, 0); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block10); + masm.jmp(block12); + + masm.bind(block11); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, block12); + masm.negl(rcx); + masm.shrq(r10); + masm.movq(rax, r9); + masm.shrq(r9); + masm.subl(rdx, rcx); + masm.negl(rcx); + masm.addl(rcx, 64); + masm.shlq(rax); + masm.orq(r10, rax); + masm.jmp(block12); + + masm.bind(block5); + masm.notl(rcx); + masm.shlq(r9, 32); + masm.orq(r9, r11); + masm.shlq(r9); + masm.movq(rdi, r9); + masm.testl(r9, Integer.MIN_VALUE); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, block13); + masm.shrl(r9); + masm.movl(rbx, 0); + masm.shrq(rdi, 2); + masm.jmp(block8); + + masm.bind(block6); + masm.shrl(r9); + masm.movl(rbx, 1073741824); + masm.shrl(rbx); + masm.shlq(r9, 32); + masm.orq(r9, r11); + masm.shlq(rbx, 32); + masm.addl(rdi, 1073741824); + masm.movl(rcx, 0); + masm.movl(r11, 0); + masm.subq(rcx, r8); + masm.sbbq(r11, r10); + masm.sbbq(rbx, r9); + masm.movq(r8, rcx); + masm.movq(r10, r11); + masm.movq(r9, rbx); + masm.movl(rbx, 32768); + masm.jmp(block7); + + masm.bind(block13); + masm.shrl(r9); + masm.movq(rbx, 0x100000000L); + masm.shrq(rbx); + masm.movl(rcx, 0); + masm.movl(r11, 0); + masm.subq(rcx, r8); + masm.sbbq(r11, r10); + masm.sbbq(rbx, r9); + masm.movq(r8, rcx); + masm.movq(r10, r11); + masm.movq(r9, rbx); + masm.movl(rbx, 32768); + masm.shrq(rdi, 2); + masm.addl(rdi, 1073741824); + masm.jmp(block8); + + masm.bind(block4); + masm.movq(xmm0, new AMD64Address(rsp, 8)); + masm.mulsd(xmm0, recordExternalAddress(crb, negZero)); // 0x00000000, 0x80000000 + masm.movq(new AMD64Address(rsp, 0), xmm0); + + masm.bind(block14); + masm.addq(rsp, 16); + masm.pop(rbx); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64Move.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64Move.java index 3cfb4475115..829c5c1f184 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64Move.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.amd64/src/org/graalvm/compiler/lir/amd64/AMD64Move.java @@ -698,7 +698,7 @@ public class AMD64Move { if (crb.mustReplaceWithNullRegister(input)) { masm.movq(result, crb.nullRegister); } else { - masm.movq(result, 0x0L); + masm.movslq(result, 0); } } else if (crb.target.inlineObjects) { crb.recordInlineDataInCode(input); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.jtt/src/org/graalvm/compiler/lir/jtt/SPARCBranchBailoutTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.jtt/src/org/graalvm/compiler/lir/jtt/SPARCBranchBailoutTest.java index d06035777dc..43a00a9819b 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.jtt/src/org/graalvm/compiler/lir/jtt/SPARCBranchBailoutTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir.jtt/src/org/graalvm/compiler/lir/jtt/SPARCBranchBailoutTest.java @@ -25,17 +25,15 @@ package org.graalvm.compiler.lir.jtt; import org.graalvm.compiler.api.directives.GraalDirectives; -import org.graalvm.compiler.core.common.PermanentBailoutException; +import org.graalvm.compiler.asm.BranchTargetOutOfBoundsException; import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.debug.DebugContext.Scope; -import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.lir.LIRInstruction; import org.graalvm.compiler.lir.LIRInstructionClass; import org.graalvm.compiler.lir.asm.CompilationResultBuilder; import org.graalvm.compiler.lir.gen.LIRGeneratorTool; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions; -import org.junit.Assert; import org.junit.Assume; import org.junit.Test; @@ -82,18 +80,14 @@ public class SPARCBranchBailoutTest extends LIRTest { } @SuppressWarnings("try") - @Test + @Test(expected = BranchTargetOutOfBoundsException.class) public void testBailoutOnBranchOverflow() throws Throwable { Assume.assumeTrue(getBackend().getTarget().arch instanceof SPARC); ResolvedJavaMethod m = getResolvedJavaMethod("testBranch"); DebugContext debug = getDebugContext(); - try { - try (Scope s = debug.disable()) { - StructuredGraph graph = parseEager(m, AllowAssumptions.YES, debug); - compile(m, graph); - } - } catch (GraalError e) { - Assert.assertEquals(PermanentBailoutException.class, e.getCause().getClass()); + try (Scope s = debug.disable()) { + StructuredGraph graph = parseEager(m, AllowAssumptions.YES, debug); + compile(m, graph); } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/LIR.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/LIR.java index bc91f6382b7..1d533c72777 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/LIR.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/LIR.java @@ -28,12 +28,14 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.core.common.cfg.AbstractBlockBase; import org.graalvm.compiler.core.common.cfg.AbstractControlFlowGraph; import org.graalvm.compiler.core.common.cfg.BlockMap; import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.lir.StandardOp.BlockEndOp; import org.graalvm.compiler.lir.StandardOp.LabelOp; +import org.graalvm.compiler.lir.StandardOp.LabelHoldingOp; import org.graalvm.compiler.lir.gen.LIRGenerator; import org.graalvm.compiler.options.OptionValues; @@ -233,8 +235,11 @@ public final class LIR extends LIRGenerator.VariableProvider { continue; } for (LIRInstruction inst : lirInstructions.get(block)) { - if (inst instanceof LabelOp) { - ((LabelOp) inst).getLabel().reset(); + if (inst instanceof LabelHoldingOp) { + Label label = ((LabelHoldingOp) inst).getLabel(); + if (label != null) { + label.reset(); + } } } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/LIRValueUtil.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/LIRValueUtil.java index 37412750257..0560641e912 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/LIRValueUtil.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/LIRValueUtil.java @@ -71,6 +71,11 @@ public final class LIRValueUtil { return asConstantValue(value).getJavaConstant(); } + public static boolean isNullConstant(Value value) { + assert value != null; + return isJavaConstant(value) && asJavaConstant(value).isNull(); + } + public static boolean isIntConstant(Value value, long expected) { if (isJavaConstant(value)) { JavaConstant javaConstant = asJavaConstant(value); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/StandardOp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/StandardOp.java index 5b08f595f9d..eca0669e42e 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/StandardOp.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/StandardOp.java @@ -70,10 +70,14 @@ public class StandardOp { boolean makeNullCheckFor(Value value, LIRFrameState nullCheckState, int implicitNullCheckLimit); } + public interface LabelHoldingOp { + Label getLabel(); + } + /** * LIR operation that defines the position of a label. */ - public static final class LabelOp extends LIRInstruction { + public static final class LabelOp extends LIRInstruction implements LabelHoldingOp { public static final LIRInstructionClass TYPE = LIRInstructionClass.create(LabelOp.class); public static final EnumSet incomingFlags = EnumSet.of(REG, STACK); @@ -155,6 +159,7 @@ public class StandardOp { crb.asm.bind(label); } + @Override public Label getLabel() { return label; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/Variable.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/Variable.java index 4bfbeb2ce40..99eb0494416 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/Variable.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/Variable.java @@ -33,7 +33,7 @@ import jdk.vm.ci.meta.ValueKind; * Represents a value that is yet to be bound to a machine location (such as a {@link RegisterValue} * or {@link StackSlot}) by a register allocator. */ -public final class Variable extends AllocatableValue { +public class Variable extends AllocatableValue { /** * The identifier of the variable. This is a non-zero index in a contiguous 0-based name space. diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/alloc/lsra/LinearScanLifetimeAnalysisPhase.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/alloc/lsra/LinearScanLifetimeAnalysisPhase.java index d01fb2b40a1..318d855ee4d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/alloc/lsra/LinearScanLifetimeAnalysisPhase.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/alloc/lsra/LinearScanLifetimeAnalysisPhase.java @@ -176,7 +176,7 @@ public class LinearScanLifetimeAnalysisPhase extends LinearScanAllocationPhase { ValueConsumer useConsumer = (operand, mode, flags) -> { if (isVariable(operand)) { - int operandNum = allocator.operandNumber(operand); + int operandNum = getOperandNumber(operand); if (!liveKillScratch.get(operandNum)) { liveGenScratch.set(operandNum); if (debug.isLogEnabled()) { @@ -194,7 +194,7 @@ public class LinearScanLifetimeAnalysisPhase extends LinearScanAllocationPhase { }; ValueConsumer stateConsumer = (operand, mode, flags) -> { if (LinearScan.isVariableOrRegister(operand)) { - int operandNum = allocator.operandNumber(operand); + int operandNum = getOperandNumber(operand); if (!liveKillScratch.get(operandNum)) { liveGenScratch.set(operandNum); if (debug.isLogEnabled()) { @@ -205,7 +205,7 @@ public class LinearScanLifetimeAnalysisPhase extends LinearScanAllocationPhase { }; ValueConsumer defConsumer = (operand, mode, flags) -> { if (isVariable(operand)) { - int varNum = allocator.operandNumber(operand); + int varNum = getOperandNumber(operand); liveKillScratch.set(varNum); if (debug.isLogEnabled()) { debug.log("liveKill for operand %d(%s)", varNum, operand); @@ -268,7 +268,7 @@ public class LinearScanLifetimeAnalysisPhase extends LinearScanAllocationPhase { */ if (isRegister(operand)) { if (allocator.isProcessed(operand)) { - liveKill.set(allocator.operandNumber(operand)); + liveKill.set(getOperandNumber(operand)); } } } @@ -281,11 +281,15 @@ public class LinearScanLifetimeAnalysisPhase extends LinearScanAllocationPhase { */ if (isRegister(operand) && block != allocator.getLIR().getControlFlowGraph().getStartBlock()) { if (allocator.isProcessed(operand)) { - assert liveKill.get(allocator.operandNumber(operand)) : "using fixed register " + asRegister(operand) + " that is not defined in this block " + block; + assert liveKill.get(getOperandNumber(operand)) : "using fixed register " + asRegister(operand) + " that is not defined in this block " + block; } } } + protected int getOperandNumber(Value operand) { + return allocator.operandNumber(operand); + } + /** * Performs a backward dataflow analysis to compute global live sets (i.e. * {@link BlockData#liveIn} and {@link BlockData#liveOut}) for each block. diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/alloc/lsra/ssa/SSALinearScanLifetimeAnalysisPhase.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/alloc/lsra/ssa/SSALinearScanLifetimeAnalysisPhase.java index ab8d6a1715b..42fa81681cc 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/alloc/lsra/ssa/SSALinearScanLifetimeAnalysisPhase.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/alloc/lsra/ssa/SSALinearScanLifetimeAnalysisPhase.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,14 +24,17 @@ package org.graalvm.compiler.lir.alloc.lsra.ssa; +import java.util.BitSet; import java.util.EnumSet; +import org.graalvm.compiler.core.common.cfg.AbstractBlockBase; import org.graalvm.compiler.debug.DebugContext; +import org.graalvm.compiler.lir.LIR; import org.graalvm.compiler.lir.LIRInstruction; import org.graalvm.compiler.lir.LIRInstruction.OperandFlag; import org.graalvm.compiler.lir.LIRInstruction.OperandMode; +import org.graalvm.compiler.lir.StandardOp; import org.graalvm.compiler.lir.StandardOp.LabelOp; -import org.graalvm.compiler.lir.ValueConsumer; import org.graalvm.compiler.lir.alloc.lsra.Interval; import org.graalvm.compiler.lir.alloc.lsra.Interval.RegisterPriority; import org.graalvm.compiler.lir.alloc.lsra.LinearScan; @@ -53,17 +56,39 @@ public class SSALinearScanLifetimeAnalysisPhase extends LinearScanLifetimeAnalys if (hintAtDef && op instanceof LabelOp) { LabelOp label = (LabelOp) op; + if (!label.isPhiIn()) { + return; + } Interval to = allocator.getOrCreateInterval((AllocatableValue) targetValue); - SSAUtil.forEachPhiRegisterHint(allocator.getLIR(), allocator.blockForId(label.id()), label, targetValue, mode, (ValueConsumer) (registerHint, valueMode, valueFlags) -> { - if (LinearScan.isVariableOrRegister(registerHint)) { - Interval from = allocator.getOrCreateInterval((AllocatableValue) registerHint); + LIR lir = allocator.getLIR(); + AbstractBlockBase block = allocator.blockForId(label.id()); + assert mode == OperandMode.DEF : "Wrong operand mode: " + mode; + assert lir.getLIRforBlock(block).get(0).equals(label) : String.format("Block %s and Label %s do not match!", block, label); - setHint(debug, op, to, from); - setHint(debug, op, from, to); + int idx = SSAUtil.indexOfValue(label, targetValue); + assert idx >= 0 : String.format("Value %s not in label %s", targetValue, label); + + BitSet blockLiveIn = allocator.getBlockData(block).liveIn; + + AbstractBlockBase selectedPredecessor = null; + AllocatableValue selectedSource = null; + for (AbstractBlockBase pred : block.getPredecessors()) { + if (selectedPredecessor == null || pred.getRelativeFrequency() > selectedPredecessor.getRelativeFrequency()) { + StandardOp.JumpOp jump = SSAUtil.phiOut(lir, pred); + Value sourceValue = jump.getOutgoingValue(idx); + if (LinearScan.isVariableOrRegister(sourceValue) && !blockLiveIn.get(getOperandNumber(sourceValue))) { + selectedSource = (AllocatableValue) sourceValue; + selectedPredecessor = pred; + } } - }); + } + if (selectedSource != null) { + Interval from = allocator.getOrCreateInterval(selectedSource); + setHint(debug, op, to, from); + setHint(debug, op, from, to); + } } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/asm/CompilationResultBuilder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/asm/CompilationResultBuilder.java index 3978704d812..08e6fe6b060 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/asm/CompilationResultBuilder.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/asm/CompilationResultBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -38,6 +38,7 @@ import jdk.internal.vm.compiler.collections.EconomicMap; import jdk.internal.vm.compiler.collections.Equivalence; import org.graalvm.compiler.asm.AbstractAddress; import org.graalvm.compiler.asm.Assembler; +import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.code.CompilationResult; import org.graalvm.compiler.code.CompilationResult.CodeAnnotation; import org.graalvm.compiler.code.DataSection.Data; @@ -46,7 +47,6 @@ import org.graalvm.compiler.core.common.NumUtil; import org.graalvm.compiler.core.common.cfg.AbstractBlockBase; import org.graalvm.compiler.core.common.spi.ForeignCallsProvider; import org.graalvm.compiler.core.common.type.DataPointerConstant; -import org.graalvm.compiler.debug.Assertions; import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.NodeSourcePosition; @@ -54,12 +54,14 @@ import org.graalvm.compiler.lir.LIR; import org.graalvm.compiler.lir.LIRFrameState; import org.graalvm.compiler.lir.LIRInstruction; import org.graalvm.compiler.lir.LabelRef; +import org.graalvm.compiler.lir.StandardOp.LabelHoldingOp; import org.graalvm.compiler.lir.framemap.FrameMap; import org.graalvm.compiler.options.Option; import org.graalvm.compiler.options.OptionKey; import org.graalvm.compiler.options.OptionType; import org.graalvm.compiler.options.OptionValues; +import jdk.vm.ci.code.BailoutException; import jdk.vm.ci.code.CodeCacheProvider; import jdk.vm.ci.code.DebugInfo; import jdk.vm.ci.code.Register; @@ -155,6 +157,20 @@ public class CompilationResultBuilder { private Consumer beforeOp; private Consumer afterOp; + /** + * These position maps are used for estimating offsets of forward branches. Used for + * architectures where certain branch instructions have limited displacement such as ARM tbz or + * SPARC cbcond. + */ + private EconomicMap labelBindLirPositions; + private EconomicMap lirPositions; + /** + * This flag is for setting the + * {@link CompilationResultBuilder#labelWithinRange(LIRInstruction, Label, int)} into a + * conservative mode and always answering false. + */ + private boolean conservativeLabelOffsets = false; + public final boolean mustReplaceWithNullRegister(JavaConstant nullConstant) { return !nullRegister.equals(Register.None) && JavaConstant.NULL_POINTER.equals(nullConstant); } @@ -179,14 +195,6 @@ public class CompilationResultBuilder { this.debug = debug; assert frameContext != null; this.dataCache = dataCache; - - if (dataBuilder.needDetailedPatchingInformation() || Assertions.assertionsEnabled()) { - /* - * Always enabled in debug mode, even when the VM does not request detailed information, - * to increase test coverage. - */ - asm.setCodePatchingAnnotationConsumer(assemblerCodeAnnotation -> compilationResult.addAnnotation(new AssemblerAnnotation(assemblerCodeAnnotation))); - } } public void setTotalFrameSize(int frameSize) { @@ -543,6 +551,8 @@ public class CompilationResultBuilder { if (op.getPosition() != null) { crb.recordSourceMapping(start, crb.asm.position(), op.getPosition()); } + } catch (BailoutException e) { + throw e; } catch (AssertionError t) { throw new GraalError(t); } catch (RuntimeException t) { @@ -559,6 +569,8 @@ public class CompilationResultBuilder { if (dataCache != null) { dataCache.clear(); } + lir = null; + currentBlockIndex = 0; } public void setOpCallback(Consumer beforeOp, Consumer afterOp) { @@ -570,4 +582,57 @@ public class CompilationResultBuilder { return options; } + /** + * Builds up a map for label and LIR instruction positions where labels are or labels pointing + * to. + */ + public void buildLabelOffsets(LIR generatedLIR) { + labelBindLirPositions = EconomicMap.create(Equivalence.IDENTITY); + lirPositions = EconomicMap.create(Equivalence.IDENTITY); + int instructionPosition = 0; + for (AbstractBlockBase block : generatedLIR.codeEmittingOrder()) { + if (block != null) { + for (LIRInstruction op : generatedLIR.getLIRforBlock(block)) { + if (op instanceof LabelHoldingOp) { + Label label = ((LabelHoldingOp) op).getLabel(); + if (label != null) { + labelBindLirPositions.put(label, instructionPosition); + } + lirPositions.put(op, instructionPosition); + } + instructionPosition++; + } + } + } + } + + /** + * Answers the code generator whether the jump from instruction to label is within disp LIR + * instructions. + * + * @param disp Maximum number of LIR instructions between label and instruction + */ + public boolean labelWithinRange(LIRInstruction instruction, Label label, int disp) { + if (conservativeLabelOffsets) { + return false; + } + Integer labelPosition = labelBindLirPositions.get(label); + Integer instructionPosition = lirPositions.get(instruction); + boolean result; + if (labelPosition != null && instructionPosition != null) { + result = Math.abs(labelPosition - instructionPosition) < disp; + } else { + result = false; + } + return result; + } + + /** + * Sets this CompilationResultBuilder into conservative mode. If set, + * {@link CompilationResultBuilder#labelWithinRange(LIRInstruction, Label, int)} always returns + * false. + */ + public void setConservativeLabelRanges() { + this.conservativeLabelOffsets = true; + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/asm/DataBuilder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/asm/DataBuilder.java index dde6e263418..b7e99242c77 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/asm/DataBuilder.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/asm/DataBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,13 +29,5 @@ import org.graalvm.compiler.code.DataSection.Data; import jdk.vm.ci.meta.Constant; public abstract class DataBuilder { - - /** - * When the method returns true, then Graal must produce detailed information that allows code - * patching without decoding instructions, i.e., Graal must produce annotations for the machine - * code that describe the exact locations of operands within instructions. - */ - public abstract boolean needDetailedPatchingInformation(); - public abstract Data createDataItem(Constant c); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/ArithmeticLIRGenerator.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/ArithmeticLIRGenerator.java index ac8fcc99e91..36441722023 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/ArithmeticLIRGenerator.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/ArithmeticLIRGenerator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2009, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -110,4 +110,9 @@ public abstract class ArithmeticLIRGenerator implements ArithmeticLIRGeneratorTo return isAdd ? emitAdd(resultKind, a, b, setFlags) : emitSub(resultKind, a, b, setFlags); } + public Value emitRor(Value value, Value distance) { + // (value >>> distance) | (value << -distance) + return emitOr(emitUShr(value, distance), emitShl(value, emitNegate(distance))); + } + } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/LIRGenerator.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/LIRGenerator.java index 7dc8f9c6fa6..0e5bf09f1c0 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/LIRGenerator.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/LIRGenerator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2009, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -35,8 +35,8 @@ import static org.graalvm.compiler.lir.LIRValueUtil.isVirtualStackSlot; import java.util.ArrayList; import java.util.List; +import java.util.Optional; -import jdk.vm.ci.code.RegisterConfig; import org.graalvm.compiler.asm.Label; import org.graalvm.compiler.core.common.LIRKind; import org.graalvm.compiler.core.common.calc.Condition; @@ -59,6 +59,7 @@ import org.graalvm.compiler.lir.StandardOp; import org.graalvm.compiler.lir.StandardOp.BlockEndOp; import org.graalvm.compiler.lir.StandardOp.LabelOp; import org.graalvm.compiler.lir.StandardOp.SaveRegistersOp; +import org.graalvm.compiler.lir.hashing.Hasher; import org.graalvm.compiler.lir.SwitchStrategy; import org.graalvm.compiler.lir.Variable; import org.graalvm.compiler.options.Option; @@ -70,6 +71,7 @@ import jdk.vm.ci.code.CallingConvention; import jdk.vm.ci.code.CodeCacheProvider; import jdk.vm.ci.code.Register; import jdk.vm.ci.code.RegisterAttributes; +import jdk.vm.ci.code.RegisterConfig; import jdk.vm.ci.code.StackSlot; import jdk.vm.ci.code.TargetDescription; import jdk.vm.ci.meta.AllocatableValue; @@ -449,29 +451,51 @@ public abstract class LIRGenerator implements LIRGeneratorTool { @Override public void emitStrategySwitch(JavaConstant[] keyConstants, double[] keyProbabilities, LabelRef[] keyTargets, LabelRef defaultTarget, Variable value) { - int keyCount = keyConstants.length; SwitchStrategy strategy = SwitchStrategy.getBestStrategy(keyProbabilities, keyConstants, keyTargets); + + int keyCount = keyConstants.length; + double minDensity = 1 / Math.sqrt(strategy.getAverageEffort()); + Optional hasher = hasherFor(keyConstants, minDensity); + double hashTableSwitchDensity = hasher.map(h -> keyCount / (double) h.cardinality()).orElse(0d); long valueRange = keyConstants[keyCount - 1].asLong() - keyConstants[0].asLong() + 1; double tableSwitchDensity = keyCount / (double) valueRange; + /* * This heuristic tries to find a compromise between the effort for the best switch strategy * and the density of a tableswitch. If the effort for the strategy is at least 4, then a * tableswitch is preferred if better than a certain value that starts at 0.5 and lowers * gradually with additional effort. */ - if (strategy.getAverageEffort() < 4 || tableSwitchDensity < (1 / Math.sqrt(strategy.getAverageEffort()))) { + if (strategy.getAverageEffort() < 4d || (tableSwitchDensity < minDensity && hashTableSwitchDensity < minDensity)) { emitStrategySwitch(strategy, value, keyTargets, defaultTarget); } else { - int minValue = keyConstants[0].asInt(); - assert valueRange < Integer.MAX_VALUE; - LabelRef[] targets = new LabelRef[(int) valueRange]; - for (int i = 0; i < valueRange; i++) { - targets[i] = defaultTarget; + if (hashTableSwitchDensity > tableSwitchDensity) { + Hasher h = hasher.get(); + int cardinality = h.cardinality(); + LabelRef[] targets = new LabelRef[cardinality]; + JavaConstant[] keys = new JavaConstant[cardinality]; + for (int i = 0; i < cardinality; i++) { + keys[i] = JavaConstant.INT_0; + targets[i] = defaultTarget; + } + for (int i = 0; i < keyCount; i++) { + int idx = h.hash(keyConstants[i].asLong()); + keys[idx] = keyConstants[i]; + targets[idx] = keyTargets[i]; + } + emitHashTableSwitch(h, keys, defaultTarget, targets, value); + } else { + int minValue = keyConstants[0].asInt(); + assert valueRange < Integer.MAX_VALUE; + LabelRef[] targets = new LabelRef[(int) valueRange]; + for (int i = 0; i < valueRange; i++) { + targets[i] = defaultTarget; + } + for (int i = 0; i < keyCount; i++) { + targets[keyConstants[i].asInt() - minValue] = keyTargets[i]; + } + emitTableSwitch(minValue, defaultTarget, targets, value); } - for (int i = 0; i < keyCount; i++) { - targets[keyConstants[i].asInt() - minValue] = keyTargets[i]; - } - emitTableSwitch(minValue, defaultTarget, targets, value); } } @@ -480,6 +504,16 @@ public abstract class LIRGenerator implements LIRGeneratorTool { protected abstract void emitTableSwitch(int lowKey, LabelRef defaultTarget, LabelRef[] targets, Value key); + @SuppressWarnings("unused") + protected Optional hasherFor(JavaConstant[] keyConstants, double minDensity) { + return Optional.empty(); + } + + @SuppressWarnings("unused") + protected void emitHashTableSwitch(Hasher hasher, JavaConstant[] keys, LabelRef defaultTarget, LabelRef[] targets, Value value) { + throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support hash table switches"); + } + @Override public void beforeRegisterAllocation() { } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/LIRGeneratorTool.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/LIRGeneratorTool.java index a5e1343b576..7296aa8ada9 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/LIRGeneratorTool.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/gen/LIRGeneratorTool.java @@ -24,6 +24,9 @@ package org.graalvm.compiler.lir.gen; +import java.util.BitSet; +import java.util.List; + import org.graalvm.compiler.core.common.CompressEncoding; import org.graalvm.compiler.core.common.LIRKind; import org.graalvm.compiler.core.common.calc.Condition; @@ -39,11 +42,13 @@ import org.graalvm.compiler.lir.LIRInstruction; import org.graalvm.compiler.lir.LabelRef; import org.graalvm.compiler.lir.SwitchStrategy; import org.graalvm.compiler.lir.Variable; +import org.graalvm.compiler.lir.VirtualStackSlot; import jdk.vm.ci.code.CodeCacheProvider; import jdk.vm.ci.code.Register; import jdk.vm.ci.code.RegisterAttributes; import jdk.vm.ci.code.RegisterConfig; +import jdk.vm.ci.code.StackSlot; import jdk.vm.ci.code.TargetDescription; import jdk.vm.ci.code.ValueKindFactory; import jdk.vm.ci.meta.AllocatableValue; @@ -263,6 +268,11 @@ public interface LIRGeneratorTool extends DiagnosticLIRGeneratorTool, ValueKindF Variable emitArrayEquals(JavaKind kind, Value array1, Value array2, Value length, int constantLength, boolean directPointers); + @SuppressWarnings("unused") + default Variable emitArrayEquals(JavaKind kind1, JavaKind kind2, Value array1, Value array2, Value length, int constantLength, boolean directPointers) { + throw GraalError.unimplemented("Array.equals with different types substitution is not implemented on this architecture"); + } + @SuppressWarnings("unused") default Variable emitArrayIndexOf(JavaKind kind, boolean findTwoConsecutive, Value sourcePointer, Value sourceCount, Value... searchValues) { throw GraalError.unimplemented("String.indexOf substitution is not implemented on this architecture"); @@ -313,4 +323,21 @@ public interface LIRGeneratorTool extends DiagnosticLIRGeneratorTool, ValueKindF * after this fence will execute until all previous instructions have retired. */ void emitSpeculationFence(); + + default VirtualStackSlot allocateStackSlots(int slots, BitSet objects, List outObjectStackSlots) { + return getResult().getFrameMapBuilder().allocateStackSlots(slots, objects, outObjectStackSlots); + } + + default Value emitReadCallerStackPointer(Stamp wordStamp) { + /* + * We do not know the frame size yet. So we load the address of the first spill slot + * relative to the beginning of the frame, which is equivalent to the stack pointer of the + * caller. + */ + return emitAddress(StackSlot.get(getLIRKind(wordStamp), 0, true)); + } + + default Value emitReadReturnAddress(Stamp wordStamp, int returnAddressSize) { + return emitMove(StackSlot.get(getLIRKind(wordStamp), -returnAddressSize, true)); + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/hashing/HashFunction.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/hashing/HashFunction.java new file mode 100644 index 00000000000..cf109a2cf6c --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/hashing/HashFunction.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.hashing; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.function.BiFunction; +import java.util.function.Function; + +import org.graalvm.compiler.lir.gen.ArithmeticLIRGenerator; + +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.Value; + +/** + * This class provides a set of cheap imperfect hash functions based on the paper "Improving Switch + * Statement Performance with Hashing Optimized at Compile Time". + * (http://programming.sirrida.de/hashsuper.pdf) + */ +public abstract class HashFunction { + + /** + * Applies the hash function. + * + * @param value the value to be hashed + * @param min {@code value} is guaranteed to be greater or equal to this minimum + * @return the hash value within int range + */ + public abstract int apply(long value, long min); + + /** + * Generates LIR that implements the hash function in terms of value and min. + * + * @param value the value to be hashed + * @param min the lowest key + * @param gen the lir generator + * @return new lir value with the hash function applied + */ + public abstract Value gen(Value value, Value min, ArithmeticLIRGenerator gen); + + /** + * Returns an estimate of number of CPU cycles necessary to apply the hash function. + */ + public abstract int effort(); + + /** + * @return a list of all available hash functions + */ + public static final List instances() { + return Collections.unmodifiableList(instances); + } + + private static List instances = new ArrayList<>(); + + private static int[] mersennePrimes = {3, 7, 31, 127, 8191, 131071, 524287, 2147483647}; + + static { + //@formatter:off + + add("val", 0, + (val, min) -> val, + gen -> (val, min) -> val); + + add("val - min", 1, + (val, min) -> val - min, + gen -> (val, min) -> gen.emitSub(val, min, false)); + + add("val >> min", 1, + (val, min) -> val >> min, + gen -> (val, min) -> gen.emitShr(val, min)); + + add("val >> (val & min)", 2, + (val, min) -> val >> (val & min), + gen -> (val, min) -> gen.emitShr(val, gen.emitAnd(val, min))); + + add("(val >> min) ^ val", 2, + (val, min) -> (val >> min) ^ val, + gen -> (val, min) -> gen.emitXor(gen.emitShr(val, min), val)); + + add("(val >> min) * val", 3, + (val, min) -> (val >> min) * val, + gen -> (val, min) -> gen.emitMul(gen.emitShr(val, min), val, false)); + + addWithPrimes("(val * prime) >> min", 3, + prime -> (val, min) -> (val * prime) >> min, + (gen, prime) -> (val, min) -> gen.emitShr(gen.emitMul(val, prime, false), min)); + + addWithPrimes("rotateRight(val, prime)", 3, + prime -> (val, min) -> Long.rotateRight(val, prime), + (gen, prime) -> (val, min) -> gen.emitRor(val, prime)); + + addWithPrimes("rotateRight(val, prime) + val", 4, + prime -> (val, min) -> Long.rotateRight(val, prime) + val, + (gen, prime) -> (val, min) -> gen.emitAdd(gen.emitRor(val, prime), val, false)); + + addWithPrimes("rotateRight(val, prime) ^ val", 4, + prime -> (val, min) -> Long.rotateRight(val, prime) ^ val, + (gen, prime) -> (val, min) -> gen.emitXor(gen.emitRor(val, prime), val)); + //@formatter:on + } + + private static void add(String toString, int effort, BiFunction f, Function> gen) { + instances.add(new HashFunction() { + + @Override + public int apply(long value, long min) { + return f.apply(value, min).intValue(); + } + + @Override + public int effort() { + return effort; + } + + @Override + public String toString() { + return toString; + } + + @Override + public Value gen(Value val, Value min, ArithmeticLIRGenerator t) { + return gen.apply(t).apply(t.emitNarrow(val, 32), t.emitNarrow(min, 32)); + } + }); + } + + private static void addWithPrimes(String toString, int effort, Function> f, + BiFunction> gen) { + for (int p : mersennePrimes) { + add(toString, effort, f.apply(p), g -> gen.apply(g, g.getLIRGen().emitJavaConstant(JavaConstant.forInt(p)))); + } + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/hashing/Hasher.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/hashing/Hasher.java new file mode 100644 index 00000000000..c363951e9fe --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/hashing/Hasher.java @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.lir.hashing; + +import java.util.Comparator; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; + +import org.graalvm.compiler.lir.gen.ArithmeticLIRGenerator; + +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.Value; + +/** + * This class holds a hash function at a specific cardinality and min value (lowest key). The + * cardinality is the required size of the hash table to make the hasher injective for the provided + * keys. + */ +public final class Hasher { + + /** + * Tries to find a hash function without conflicts for the provided keys. + * + * @param keys the keys + * @param minDensity the minimum density of the switch table. Used to determine the maximum + * cardinality of the hash function + * @return an optional hasher + */ + public static Optional forKeys(JavaConstant[] keys, double minDensity) { + if (keys.length <= 2) { + return Optional.empty(); + } else { + int maxCardinality = (int) Math.round(keys.length / minDensity); + assertSorted(keys); + TreeSet candidates = new TreeSet<>(new Comparator() { + @Override + public int compare(Hasher o1, Hasher o2) { + int d = o1.cardinality - o2.cardinality; + if (d != 0) { + return d; + } else { + return o1.effort() - o2.effort(); + } + } + }); + long min = keys[0].asLong(); + for (HashFunction f : HashFunction.instances()) { + for (int cardinality = keys.length; cardinality < maxCardinality; cardinality++) { + if (isValid(keys, min, f, cardinality)) { + candidates.add(new Hasher(f, cardinality, min)); + break; + } + } + } + if (candidates.isEmpty()) { + return Optional.empty(); + } else { + return Optional.of(candidates.first()); + } + } + } + + private static void assertSorted(JavaConstant[] keys) { + for (int i = 1; i < keys.length; i++) { + assert keys[i - 1].asLong() < keys[i].asLong(); + } + } + + private static boolean isValid(JavaConstant[] keys, long min, HashFunction function, int cardinality) { + Set seen = new HashSet<>(keys.length); + for (JavaConstant key : keys) { + int hash = function.apply(key.asLong(), min) & (cardinality - 1); + if (!seen.add(hash)) { + return false; + } + } + return true; + } + + private final HashFunction function; + private final int cardinality; + private final long min; + + private Hasher(HashFunction function, int cardinality, long min) { + this.function = function; + this.cardinality = cardinality; + this.min = min; + } + + /** + * Applies the hash function. + * + * @param value the value to be hashed + * @return the hash value + */ + public int hash(long value) { + return function.apply(value, min) & (cardinality - 1); + } + + /** + * Applies the hash function to a lir value. + * + * @param value the value to be hashed + * @param gen the lir generator + * @return the hashed lir value + */ + public Value hash(Value value, ArithmeticLIRGenerator gen) { + Value h = function.gen(value, gen.getLIRGen().emitJavaConstant(JavaConstant.forLong(min)), gen); + return gen.emitAnd(h, gen.getLIRGen().emitJavaConstant(JavaConstant.forInt(cardinality - 1))); + } + + /** + * @return the hashing effort + */ + public int effort() { + return function.effort() + 1; + } + + /** + * @return the cardinality of the hash function that should match the size of the table switch. + */ + public int cardinality() { + return cardinality; + } + + /** + * @return the hash function + */ + public HashFunction function() { + return function; + } + + @Override + public String toString() { + return "Hasher[function=" + function + ", effort=" + effort() + ", cardinality=" + cardinality + "]"; + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/ssa/SSAUtil.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/ssa/SSAUtil.java index 5cddac5c814..741bf057799 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/ssa/SSAUtil.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.lir/src/org/graalvm/compiler/lir/ssa/SSAUtil.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,11 +31,9 @@ import org.graalvm.compiler.core.common.LIRKind; import org.graalvm.compiler.core.common.cfg.AbstractBlockBase; import org.graalvm.compiler.lir.LIR; import org.graalvm.compiler.lir.LIRInstruction; -import org.graalvm.compiler.lir.LIRInstruction.OperandMode; import org.graalvm.compiler.lir.StandardOp.BlockEndOp; import org.graalvm.compiler.lir.StandardOp.JumpOp; import org.graalvm.compiler.lir.StandardOp.LabelOp; -import org.graalvm.compiler.lir.ValueConsumer; import jdk.vm.ci.meta.Value; @@ -114,13 +112,6 @@ public final class SSAUtil { return (JumpOp) op; } - public static JumpOp phiOutOrNull(LIR lir, AbstractBlockBase block) { - if (block.getSuccessorCount() != 1) { - return null; - } - return phiOut(lir, block); - } - public static int phiOutIndex(LIR lir, AbstractBlockBase block) { assert block.getSuccessorCount() == 1; ArrayList instructions = lir.getLIRforBlock(block); @@ -149,10 +140,6 @@ public final class SSAUtil { return new SSAVerifier(lir).verify(); } - public static boolean isMerge(AbstractBlockBase block) { - return block.getPredecessorCount() > 1; - } - public static void verifyPhi(LIR lir, AbstractBlockBase merge) { assert merge.getPredecessorCount() > 1; for (AbstractBlockBase pred : merge.getPredecessors()) { @@ -163,25 +150,7 @@ public final class SSAUtil { } } - public static void forEachPhiRegisterHint(LIR lir, AbstractBlockBase block, LabelOp label, Value targetValue, OperandMode mode, ValueConsumer valueConsumer) { - assert mode == OperandMode.DEF : "Wrong operand mode: " + mode; - assert lir.getLIRforBlock(block).get(0).equals(label) : String.format("Block %s and Label %s do not match!", block, label); - - if (!label.isPhiIn()) { - return; - } - int idx = indexOfValue(label, targetValue); - assert idx >= 0 : String.format("Value %s not in label %s", targetValue, label); - - for (AbstractBlockBase pred : block.getPredecessors()) { - JumpOp jump = phiOut(lir, pred); - Value sourceValue = jump.getOutgoingValue(idx); - valueConsumer.visitValue(jump, sourceValue, null, null); - } - - } - - private static int indexOfValue(LabelOp label, Value value) { + public static int indexOfValue(LabelOp label, Value value) { for (int i = 0; i < label.getIncomingSize(); i++) { if (label.getIncomingValue(i).equals(value)) { return i; @@ -189,20 +158,4 @@ public final class SSAUtil { } return -1; } - - public static int numPhiOut(LIR lir, AbstractBlockBase block) { - if (block.getSuccessorCount() != 1) { - // cannot be a phi_out block - return 0; - } - return numPhiIn(lir, block.getSuccessors()[0]); - } - - private static int numPhiIn(LIR lir, AbstractBlockBase block) { - if (!isMerge(block)) { - return 0; - } - return phiIn(lir, block).getPhiSize(); - } - } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/ConvertDeoptimizeToGuardPhase.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop.phases/src/org/graalvm/compiler/loop/phases/ConvertDeoptimizeToGuardPhase.java similarity index 64% rename from src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/ConvertDeoptimizeToGuardPhase.java rename to src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop.phases/src/org/graalvm/compiler/loop/phases/ConvertDeoptimizeToGuardPhase.java index e075da33750..3b48cba416d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/ConvertDeoptimizeToGuardPhase.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop.phases/src/org/graalvm/compiler/loop/phases/ConvertDeoptimizeToGuardPhase.java @@ -22,17 +22,20 @@ */ -package org.graalvm.compiler.phases.common; +package org.graalvm.compiler.loop.phases; import static org.graalvm.compiler.phases.common.DeadCodeEliminationPhase.Optionality.Optional; import java.util.List; import org.graalvm.compiler.core.common.GraalOptions; +import org.graalvm.compiler.core.common.cfg.Loop; import org.graalvm.compiler.debug.DebugCloseable; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeSourcePosition; import org.graalvm.compiler.graph.spi.SimplifierTool; +import org.graalvm.compiler.loop.LoopEx; +import org.graalvm.compiler.loop.LoopsData; import org.graalvm.compiler.nodeinfo.InputType; import org.graalvm.compiler.nodes.AbstractBeginNode; import org.graalvm.compiler.nodes.AbstractEndNode; @@ -55,9 +58,12 @@ import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.ValuePhiNode; import org.graalvm.compiler.nodes.calc.CompareNode; +import org.graalvm.compiler.nodes.cfg.Block; import org.graalvm.compiler.nodes.spi.LoweringProvider; import org.graalvm.compiler.nodes.util.GraphUtil; import org.graalvm.compiler.phases.BasePhase; +import org.graalvm.compiler.phases.common.DeadCodeEliminationPhase; +import org.graalvm.compiler.phases.common.LazyValue; import org.graalvm.compiler.phases.tiers.PhaseContext; import jdk.vm.ci.meta.Constant; @@ -81,6 +87,7 @@ public class ConvertDeoptimizeToGuardPhase extends BasePhase { protected void run(final StructuredGraph graph, PhaseContext context) { assert graph.hasValueProxies() : "ConvertDeoptimizeToGuardPhase always creates proxies"; assert !graph.getGuardsStage().areFrameStatesAtDeopts() : graph.getGuardsStage(); + LazyValue lazyLoops = new LazyValue<>(() -> new LoopsData(graph)); for (DeoptimizeNode d : graph.getNodes(DeoptimizeNode.TYPE)) { assert d.isAlive(); @@ -88,14 +95,14 @@ public class ConvertDeoptimizeToGuardPhase extends BasePhase { continue; } try (DebugCloseable closable = d.withNodeSourcePosition()) { - propagateFixed(d, d, context != null ? context.getLowerer() : null); + propagateFixed(d, d, context != null ? context.getLowerer() : null, lazyLoops); } } if (context != null) { for (FixedGuardNode fixedGuard : graph.getNodes(FixedGuardNode.TYPE)) { try (DebugCloseable closable = fixedGuard.withNodeSourcePosition()) { - trySplitFixedGuard(fixedGuard, context); + trySplitFixedGuard(fixedGuard, context, lazyLoops); } } } @@ -103,7 +110,7 @@ public class ConvertDeoptimizeToGuardPhase extends BasePhase { new DeadCodeEliminationPhase(Optional).apply(graph); } - private void trySplitFixedGuard(FixedGuardNode fixedGuard, PhaseContext context) { + private static void trySplitFixedGuard(FixedGuardNode fixedGuard, PhaseContext context, LazyValue lazyLoops) { LogicNode condition = fixedGuard.condition(); if (condition instanceof CompareNode) { CompareNode compare = (CompareNode) condition; @@ -113,13 +120,14 @@ public class ConvertDeoptimizeToGuardPhase extends BasePhase { ValueNode y = compare.getY(); ValuePhiNode yPhi = (y instanceof ValuePhiNode) ? (ValuePhiNode) y : null; if (y instanceof ConstantNode || yPhi != null) { - processFixedGuardAndPhis(fixedGuard, context, compare, x, xPhi, y, yPhi); + processFixedGuardAndPhis(fixedGuard, context, compare, x, xPhi, y, yPhi, lazyLoops); } } } } - private void processFixedGuardAndPhis(FixedGuardNode fixedGuard, PhaseContext context, CompareNode compare, ValueNode x, ValuePhiNode xPhi, ValueNode y, ValuePhiNode yPhi) { + private static void processFixedGuardAndPhis(FixedGuardNode fixedGuard, PhaseContext context, CompareNode compare, ValueNode x, ValuePhiNode xPhi, ValueNode y, ValuePhiNode yPhi, + LazyValue lazyLoops) { AbstractBeginNode pred = AbstractBeginNode.prevBegin(fixedGuard); if (pred instanceof AbstractMergeNode) { AbstractMergeNode merge = (AbstractMergeNode) pred; @@ -130,16 +138,15 @@ public class ConvertDeoptimizeToGuardPhase extends BasePhase { return; } - processFixedGuardAndMerge(fixedGuard, context, compare, x, xPhi, y, yPhi, merge); + processFixedGuardAndMerge(fixedGuard, context, compare, x, xPhi, y, yPhi, merge, lazyLoops); } } @SuppressWarnings("try") - private void processFixedGuardAndMerge(FixedGuardNode fixedGuard, PhaseContext context, CompareNode compare, ValueNode x, ValuePhiNode xPhi, ValueNode y, ValuePhiNode yPhi, - AbstractMergeNode merge) { + private static void processFixedGuardAndMerge(FixedGuardNode fixedGuard, PhaseContext context, CompareNode compare, ValueNode x, ValuePhiNode xPhi, ValueNode y, ValuePhiNode yPhi, + AbstractMergeNode merge, LazyValue lazyLoops) { List mergePredecessors = merge.cfgPredecessors().snapshot(); - for (int i = 0; i < mergePredecessors.size(); ++i) { - AbstractEndNode mergePredecessor = mergePredecessors.get(i); + for (AbstractEndNode mergePredecessor : mergePredecessors) { if (!mergePredecessor.isAlive()) { break; } @@ -157,14 +164,14 @@ public class ConvertDeoptimizeToGuardPhase extends BasePhase { } if (xs != null && ys != null && compare.condition().foldCondition(xs, ys, context.getConstantReflection(), compare.unorderedIsTrue()) == fixedGuard.isNegated()) { try (DebugCloseable position = fixedGuard.withNodeSourcePosition()) { - propagateFixed(mergePredecessor, fixedGuard, context.getLowerer()); + propagateFixed(mergePredecessor, fixedGuard, context.getLowerer(), lazyLoops); } } } } @SuppressWarnings("try") - private void propagateFixed(FixedNode from, StaticDeoptimizingNode deopt, LoweringProvider loweringProvider) { + private static void propagateFixed(FixedNode from, StaticDeoptimizingNode deopt, LoweringProvider loweringProvider, LazyValue lazyLoops) { Node current = from; while (current != null) { if (GraalOptions.GuardPriorities.getValue(from.getOptions()) && current instanceof FixedGuardNode) { @@ -179,46 +186,50 @@ public class ConvertDeoptimizeToGuardPhase extends BasePhase { FixedNode next = mergeNode.next(); while (mergeNode.isAlive()) { AbstractEndNode end = mergeNode.forwardEnds().first(); - propagateFixed(end, deopt, loweringProvider); + propagateFixed(end, deopt, loweringProvider, lazyLoops); } if (next.isAlive()) { - propagateFixed(next, deopt, loweringProvider); + propagateFixed(next, deopt, loweringProvider, lazyLoops); } return; } else if (current.predecessor() instanceof IfNode) { + AbstractBeginNode begin = (AbstractBeginNode) current; IfNode ifNode = (IfNode) current.predecessor(); - // Prioritize the source position of the IfNode - try (DebugCloseable closable = ifNode.withNodeSourcePosition()) { - StructuredGraph graph = ifNode.graph(); - LogicNode conditionNode = ifNode.condition(); - boolean negateGuardCondition = current == ifNode.trueSuccessor(); - NodeSourcePosition survivingSuccessorPosition = negateGuardCondition ? ifNode.falseSuccessor().getNodeSourcePosition() : ifNode.trueSuccessor().getNodeSourcePosition(); - FixedGuardNode guard = graph.add( - new FixedGuardNode(conditionNode, deopt.getReason(), deopt.getAction(), deopt.getSpeculation(), negateGuardCondition, survivingSuccessorPosition)); + if (isOsrLoopExit(begin) || isCountedLoopExit(ifNode, lazyLoops)) { + moveAsDeoptAfter(begin, deopt); + } else { + // Prioritize the source position of the IfNode + try (DebugCloseable closable = ifNode.withNodeSourcePosition()) { + StructuredGraph graph = ifNode.graph(); + LogicNode conditionNode = ifNode.condition(); + boolean negateGuardCondition = current == ifNode.trueSuccessor(); + NodeSourcePosition survivingSuccessorPosition = negateGuardCondition ? ifNode.falseSuccessor().getNodeSourcePosition() : ifNode.trueSuccessor().getNodeSourcePosition(); + FixedGuardNode guard = graph.add( + new FixedGuardNode(conditionNode, deopt.getReason(), deopt.getAction(), deopt.getSpeculation(), negateGuardCondition, survivingSuccessorPosition)); + FixedWithNextNode pred = (FixedWithNextNode) ifNode.predecessor(); + AbstractBeginNode survivingSuccessor; + if (negateGuardCondition) { + survivingSuccessor = ifNode.falseSuccessor(); + } else { + survivingSuccessor = ifNode.trueSuccessor(); + } + graph.removeSplitPropagate(ifNode, survivingSuccessor); - FixedWithNextNode pred = (FixedWithNextNode) ifNode.predecessor(); - AbstractBeginNode survivingSuccessor; - if (negateGuardCondition) { - survivingSuccessor = ifNode.falseSuccessor(); - } else { - survivingSuccessor = ifNode.trueSuccessor(); + Node newGuard = guard; + if (survivingSuccessor instanceof LoopExitNode) { + newGuard = ProxyNode.forGuard(guard, (LoopExitNode) survivingSuccessor, graph); + } + survivingSuccessor.replaceAtUsages(InputType.Guard, newGuard); + + graph.getDebug().log("Converting deopt on %-5s branch of %s to guard for remaining branch %s.", negateGuardCondition, ifNode, survivingSuccessor); + FixedNode next = pred.next(); + pred.setNext(guard); + guard.setNext(next); + SimplifierTool simplifierTool = GraphUtil.getDefaultSimplifier(null, null, null, false, graph.getAssumptions(), graph.getOptions(), loweringProvider); + survivingSuccessor.simplify(simplifierTool); } - graph.removeSplitPropagate(ifNode, survivingSuccessor); - - Node newGuard = guard; - if (survivingSuccessor instanceof LoopExitNode) { - newGuard = ProxyNode.forGuard(guard, (LoopExitNode) survivingSuccessor, graph); - } - survivingSuccessor.replaceAtUsages(InputType.Guard, newGuard); - - graph.getDebug().log("Converting deopt on %-5s branch of %s to guard for remaining branch %s.", negateGuardCondition, ifNode, survivingSuccessor); - FixedNode next = pred.next(); - pred.setNext(guard); - guard.setNext(next); - SimplifierTool simplifierTool = GraphUtil.getDefaultSimplifier(null, null, null, false, graph.getAssumptions(), graph.getOptions(), loweringProvider); - survivingSuccessor.simplify(simplifierTool); - return; } + return; } else if (current.predecessor() == null || current.predecessor() instanceof ControlSplitNode) { assert current.predecessor() != null || (current instanceof StartNode && current == ((AbstractBeginNode) current).graph().start()); moveAsDeoptAfter((AbstractBeginNode) current, deopt); @@ -239,4 +250,23 @@ public class ConvertDeoptimizeToGuardPhase extends BasePhase { } } } + + private static boolean isOsrLoopExit(AbstractBeginNode node) { + if (!(node instanceof LoopExitNode)) { + return false; + } + return ((LoopExitNode) node).loopBegin().isOsrLoop(); + } + + private static boolean isCountedLoopExit(IfNode ifNode, LazyValue lazyLoops) { + LoopsData loopsData = lazyLoops.get(); + Loop loop = loopsData.getCFG().getNodeToBlock().get(ifNode).getLoop(); + if (loop != null) { + LoopEx loopEx = loopsData.loop(loop); + if (loopEx.detectCounted()) { + return ifNode == loopEx.counted().getLimitTest(); + } + } + return false; + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop.phases/src/org/graalvm/compiler/loop/phases/LoopTransformations.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop.phases/src/org/graalvm/compiler/loop/phases/LoopTransformations.java index e38887e1c63..cffec1889a2 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop.phases/src/org/graalvm/compiler/loop/phases/LoopTransformations.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop.phases/src/org/graalvm/compiler/loop/phases/LoopTransformations.java @@ -54,7 +54,6 @@ import org.graalvm.compiler.nodes.FixedWithNextNode; import org.graalvm.compiler.nodes.IfNode; import org.graalvm.compiler.nodes.LogicNode; import org.graalvm.compiler.nodes.LoopBeginNode; -import org.graalvm.compiler.nodes.LoopExitNode; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.PhiNode; import org.graalvm.compiler.nodes.SafepointNode; @@ -231,11 +230,11 @@ public abstract class LoopTransformations { graph.getDebug().log("LoopTransformations.insertPrePostLoops %s", loop); LoopFragmentWhole preLoop = loop.whole(); CountedLoopInfo preCounted = loop.counted(); - IfNode preLimit = preCounted.getLimitTest(); - assert preLimit != null; LoopBeginNode preLoopBegin = loop.loopBegin(); - LoopExitNode preLoopExitNode = preLoopBegin.getSingleLoopExit(); - FixedNode continuationNode = preLoopExitNode.next(); + AbstractBeginNode preLoopExitNode = preCounted.getCountedExit(); + + assert preLoop.nodes().contains(preLoopBegin); + assert preLoop.nodes().contains(preLoopExitNode); // Each duplication is inserted after the original, ergo create the post loop first LoopFragmentWhole mainLoop = preLoop.duplicate(); @@ -249,9 +248,9 @@ public abstract class LoopTransformations { LoopBeginNode postLoopBegin = postLoop.getDuplicatedNode(preLoopBegin); postLoopBegin.setPostLoop(); - EndNode postEndNode = getBlockEndAfterLoopExit(postLoopBegin); + AbstractBeginNode postLoopExitNode = postLoop.getDuplicatedNode(preLoopExitNode); + EndNode postEndNode = getBlockEndAfterLoopExit(postLoopExitNode); AbstractMergeNode postMergeNode = postEndNode.merge(); - LoopExitNode postLoopExitNode = postLoopBegin.getSingleLoopExit(); // Update the main loop phi initialization to carry from the pre loop for (PhiNode prePhiNode : preLoopBegin.phis()) { @@ -259,13 +258,16 @@ public abstract class LoopTransformations { mainPhiNode.setValueAt(0, prePhiNode); } - EndNode mainEndNode = getBlockEndAfterLoopExit(mainLoopBegin); + AbstractBeginNode mainLoopExitNode = mainLoop.getDuplicatedNode(preLoopExitNode); + EndNode mainEndNode = getBlockEndAfterLoopExit(mainLoopExitNode); AbstractMergeNode mainMergeNode = mainEndNode.merge(); AbstractEndNode postEntryNode = postLoopBegin.forwardEnd(); + // Exits have been merged, find the continuation below the merge + FixedNode continuationNode = mainMergeNode.next(); + // In the case of no Bounds tests, we just flow right into the main loop AbstractBeginNode mainLandingNode = BeginNode.begin(postEntryNode); - LoopExitNode mainLoopExitNode = mainLoopBegin.getSingleLoopExit(); mainLoopExitNode.setNext(mainLandingNode); preLoopExitNode.setNext(mainLoopBegin.forwardEnd()); @@ -337,8 +339,8 @@ public abstract class LoopTransformations { /** * Find the end of the block following the LoopExit. */ - private static EndNode getBlockEndAfterLoopExit(LoopBeginNode curLoopBegin) { - FixedNode node = curLoopBegin.getSingleLoopExit().next(); + private static EndNode getBlockEndAfterLoopExit(AbstractBeginNode exit) { + FixedNode node = exit.next(); // Find the last node after the exit blocks starts return getBlockEnd(node); } @@ -423,6 +425,9 @@ public abstract class LoopTransformations { condition.getDebug().log(DebugContext.VERBOSE_LEVEL, "isUnrollableLoop %s doubling the stride overflows %d", loopBegin, stride); return false; } + if (!loop.canDuplicateLoop()) { + return false; + } if (loopBegin.isMainLoop() || loopBegin.isSimpleLoop()) { // Flow-less loops to partial unroll for now. 3 blocks corresponds to an if that either // exits or continues the loop. There might be fixed and floating work within the loop diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/BasicInductionVariable.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/BasicInductionVariable.java index ae3cee6c3de..0eb34ae3dc1 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/BasicInductionVariable.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/BasicInductionVariable.java @@ -143,7 +143,7 @@ public class BasicInductionVariable extends InductionVariable { } @Override - public ValueNode extremumNode(boolean assumePositiveTripCount, Stamp stamp) { + public ValueNode extremumNode(boolean assumeLoopEntered, Stamp stamp) { Stamp fromStamp = phi.stamp(NodeView.DEFAULT); StructuredGraph graph = graph(); ValueNode stride = strideNode(); @@ -152,7 +152,7 @@ public class BasicInductionVariable extends InductionVariable { stride = IntegerConvertNode.convert(stride, stamp, graph(), NodeView.DEFAULT); initNode = IntegerConvertNode.convert(initNode, stamp, graph(), NodeView.DEFAULT); } - ValueNode maxTripCount = loop.counted().maxTripCountNode(assumePositiveTripCount); + ValueNode maxTripCount = loop.counted().maxTripCountNode(assumeLoopEntered); if (!maxTripCount.stamp(NodeView.DEFAULT).isCompatible(stamp)) { maxTripCount = IntegerConvertNode.convert(maxTripCount, stamp, graph(), NodeView.DEFAULT); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/CountedLoopInfo.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/CountedLoopInfo.java index cb983ffcc71..14fa735c512 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/CountedLoopInfo.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/CountedLoopInfo.java @@ -24,10 +24,12 @@ package org.graalvm.compiler.loop; -import static org.graalvm.compiler.loop.MathUtil.add; -import static org.graalvm.compiler.loop.MathUtil.sub; +import static java.lang.Math.abs; import static org.graalvm.compiler.loop.MathUtil.unsignedDivBefore; +import static org.graalvm.compiler.nodes.calc.BinaryArithmeticNode.add; +import static org.graalvm.compiler.nodes.calc.BinaryArithmeticNode.sub; +import org.graalvm.compiler.core.common.NumUtil; import org.graalvm.compiler.core.common.type.IntegerStamp; import org.graalvm.compiler.core.common.type.Stamp; import org.graalvm.compiler.core.common.util.UnsignedLong; @@ -37,16 +39,16 @@ import org.graalvm.compiler.nodes.AbstractBeginNode; import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.GuardNode; import org.graalvm.compiler.nodes.IfNode; +import org.graalvm.compiler.nodes.LogicNode; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.calc.CompareNode; import org.graalvm.compiler.nodes.calc.ConditionalNode; import org.graalvm.compiler.nodes.calc.IntegerLessThanNode; import org.graalvm.compiler.nodes.calc.NegateNode; import org.graalvm.compiler.nodes.extended.GuardingNode; +import org.graalvm.compiler.nodes.util.GraphUtil; -import jdk.vm.ci.code.CodeUtil; import jdk.vm.ci.meta.DeoptimizationAction; import jdk.vm.ci.meta.DeoptimizationReason; import jdk.vm.ci.meta.SpeculationLog; @@ -92,42 +94,41 @@ public class CountedLoopInfo { * * THIS VALUE SHOULD BE TREATED AS UNSIGNED. * - * @param assumePositive if true the check that the loop is entered at all will be omitted. + * @param assumeLoopEntered if true the check that the loop is entered at all will be omitted. */ - public ValueNode maxTripCountNode(boolean assumePositive) { + public ValueNode maxTripCountNode(boolean assumeLoopEntered) { StructuredGraph graph = iv.valueNode().graph(); Stamp stamp = iv.valueNode().stamp(NodeView.DEFAULT); ValueNode max; ValueNode min; - ValueNode range; ValueNode absStride; if (iv.direction() == Direction.Up) { absStride = iv.strideNode(); - range = sub(graph, end, iv.initNode()); max = end; min = iv.initNode(); } else { assert iv.direction() == Direction.Down; - absStride = graph.maybeAddOrUnique(NegateNode.create(iv.strideNode(), NodeView.DEFAULT)); - range = sub(graph, iv.initNode(), end); + absStride = NegateNode.create(iv.strideNode(), NodeView.DEFAULT); max = iv.initNode(); min = end; } + ValueNode range = sub(max, min); - ConstantNode one = ConstantNode.forIntegerStamp(stamp, 1, graph); + ConstantNode one = ConstantNode.forIntegerStamp(stamp, 1); if (oneOff) { - range = add(graph, range, one); + range = add(range, one); } // round-away-from-zero divison: (range + stride -/+ 1) / stride - ValueNode denominator = add(graph, range, sub(graph, absStride, one)); + ValueNode denominator = add(range, sub(absStride, one)); ValueNode div = unsignedDivBefore(graph, loop.entryPoint(), denominator, absStride, null); - if (assumePositive) { - return div; + if (assumeLoopEntered) { + return graph.addOrUniqueWithInputs(div); } - ConstantNode zero = ConstantNode.forIntegerStamp(stamp, 0, graph); - return graph.unique(new ConditionalNode(graph.unique(new IntegerLessThanNode(max, min)), zero, div)); + ConstantNode zero = ConstantNode.forIntegerStamp(stamp, 0); + LogicNode noEntryCheck = IntegerLessThanNode.create(max, min, NodeView.DEFAULT); + return graph.addOrUniqueWithInputs(ConditionalNode.create(noEntryCheck, zero, div, NodeView.DEFAULT)); } /** @@ -173,7 +174,7 @@ public class CountedLoopInfo { } public boolean isExactTripCount() { - return loop.loopBegin().loopExits().count() == 1; + return loop.loop().getNaturalExits().size() == 1; } public ValueNode exactTripCountNode() { @@ -216,6 +217,15 @@ public class CountedLoopInfo { return body; } + public AbstractBeginNode getCountedExit() { + if (getLimitTest().trueSuccessor() == getBody()) { + return getLimitTest().falseSuccessor(); + } else { + assert getLimitTest().falseSuccessor() == getBody(); + return getLimitTest().trueSuccessor(); + } + } + public Direction getDirection() { return iv.direction(); } @@ -228,30 +238,48 @@ public class CountedLoopInfo { return loop.loopBegin().getOverflowGuard(); } + public boolean counterNeverOverflows() { + if (iv.isConstantStride() && abs(iv.constantStride()) == 1) { + return true; + } + IntegerStamp endStamp = (IntegerStamp) end.stamp(NodeView.DEFAULT); + ValueNode strideNode = iv.strideNode(); + IntegerStamp strideStamp = (IntegerStamp) strideNode.stamp(NodeView.DEFAULT); + GraphUtil.tryKillUnused(strideNode); + if (getDirection() == Direction.Up) { + long max = NumUtil.maxValue(endStamp.getBits()); + return endStamp.upperBound() <= max - (strideStamp.upperBound() - 1) - (oneOff ? 1 : 0); + } else if (getDirection() == Direction.Down) { + long min = NumUtil.minValue(endStamp.getBits()); + return min + (1 - strideStamp.lowerBound()) + (oneOff ? 1 : 0) <= endStamp.lowerBound(); + } + return false; + } + @SuppressWarnings("try") public GuardingNode createOverFlowGuard() { GuardingNode overflowGuard = getOverFlowGuard(); - if (overflowGuard != null) { + if (overflowGuard != null || counterNeverOverflows()) { return overflowGuard; } try (DebugCloseable position = loop.loopBegin().withNodeSourcePosition()) { IntegerStamp stamp = (IntegerStamp) iv.valueNode().stamp(NodeView.DEFAULT); StructuredGraph graph = iv.valueNode().graph(); - CompareNode cond; // we use a negated guard with a < condition to achieve a >= + LogicNode cond; // we use a negated guard with a < condition to achieve a >= ConstantNode one = ConstantNode.forIntegerStamp(stamp, 1, graph); if (iv.direction() == Direction.Up) { - ValueNode v1 = sub(graph, ConstantNode.forIntegerStamp(stamp, CodeUtil.maxValue(stamp.getBits()), graph), sub(graph, iv.strideNode(), one)); + ValueNode v1 = sub(ConstantNode.forIntegerStamp(stamp, NumUtil.maxValue(stamp.getBits())), sub(iv.strideNode(), one)); if (oneOff) { - v1 = sub(graph, v1, one); + v1 = sub(v1, one); } - cond = graph.unique(new IntegerLessThanNode(v1, end)); + cond = graph.addOrUniqueWithInputs(IntegerLessThanNode.create(v1, end, NodeView.DEFAULT)); } else { assert iv.direction() == Direction.Down; - ValueNode v1 = add(graph, ConstantNode.forIntegerStamp(stamp, CodeUtil.minValue(stamp.getBits()), graph), sub(graph, one, iv.strideNode())); + ValueNode v1 = add(ConstantNode.forIntegerStamp(stamp, NumUtil.minValue(stamp.getBits())), sub(one, iv.strideNode())); if (oneOff) { - v1 = add(graph, v1, one); + v1 = add(v1, one); } - cond = graph.unique(new IntegerLessThanNode(end, v1)); + cond = graph.addOrUniqueWithInputs(IntegerLessThanNode.create(end, v1, NodeView.DEFAULT)); } assert graph.getGuardsStage().allowsFloatingGuards(); overflowGuard = graph.unique(new GuardNode(cond, AbstractBeginNode.prevBegin(loop.entryPoint()), DeoptimizationReason.LoopLimitCheck, DeoptimizationAction.InvalidateRecompile, true, diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DefaultLoopPolicies.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DefaultLoopPolicies.java index b0e6e414ed6..a51183d9774 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DefaultLoopPolicies.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DefaultLoopPolicies.java @@ -33,6 +33,7 @@ import java.util.List; import org.graalvm.compiler.core.common.util.UnsignedLong; import org.graalvm.compiler.debug.CounterKey; import org.graalvm.compiler.debug.DebugContext; +import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeBitMap; import org.graalvm.compiler.nodes.AbstractBeginNode; @@ -46,6 +47,7 @@ import org.graalvm.compiler.nodes.MergeNode; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.VirtualState; import org.graalvm.compiler.nodes.VirtualState.VirtualClosure; +import org.graalvm.compiler.nodes.calc.CompareNode; import org.graalvm.compiler.nodes.cfg.Block; import org.graalvm.compiler.nodes.cfg.ControlFlowGraph; import org.graalvm.compiler.nodes.debug.ControlFlowAnchorNode; @@ -64,9 +66,10 @@ public class DefaultLoopPolicies implements LoopPolicies { @Option(help = "", type = OptionType.Expert) public static final OptionKey LoopUnswitchTrivial = new OptionKey<>(10); @Option(help = "", type = OptionType.Expert) public static final OptionKey LoopUnswitchFrequencyBoost = new OptionKey<>(10.0); - @Option(help = "", type = OptionType.Expert) public static final OptionKey FullUnrollMaxNodes = new OptionKey<>(300); + @Option(help = "", type = OptionType.Expert) public static final OptionKey FullUnrollMaxNodes = new OptionKey<>(400); + @Option(help = "", type = OptionType.Expert) public static final OptionKey FullUnrollConstantCompareBoost = new OptionKey<>(15); @Option(help = "", type = OptionType.Expert) public static final OptionKey FullUnrollMaxIterations = new OptionKey<>(600); - @Option(help = "", type = OptionType.Expert) public static final OptionKey ExactFullUnrollMaxNodes = new OptionKey<>(1200); + @Option(help = "", type = OptionType.Expert) public static final OptionKey ExactFullUnrollMaxNodes = new OptionKey<>(800); @Option(help = "", type = OptionType.Expert) public static final OptionKey ExactPartialUnrollMaxNodes = new OptionKey<>(200); @Option(help = "", type = OptionType.Expert) public static final OptionKey UnrollMaxIterations = new OptionKey<>(16); @@ -87,7 +90,7 @@ public class DefaultLoopPolicies implements LoopPolicies { @Override public boolean shouldFullUnroll(LoopEx loop) { - if (!loop.isCounted() || !loop.counted().isConstantMaxTripCount()) { + if (!loop.isCounted() || !loop.counted().isConstantMaxTripCount() || !loop.counted().counterNeverOverflows()) { return false; } OptionValues options = loop.entryPoint().getOptions(); @@ -96,9 +99,27 @@ public class DefaultLoopPolicies implements LoopPolicies { if (maxTrips.equals(0)) { return loop.canDuplicateLoop(); } - int maxNodes = (counted.isExactTripCount() && counted.isConstantExactTripCount()) ? Options.ExactFullUnrollMaxNodes.getValue(options) : Options.FullUnrollMaxNodes.getValue(options); - maxNodes = Math.min(maxNodes, Math.max(0, MaximumDesiredSize.getValue(options) - loop.loopBegin().graph().getNodeCount())); - int size = Math.max(1, loop.size() - 1 - loop.loopBegin().phis().count()); + if (maxTrips.isGreaterThan(Options.FullUnrollMaxIterations.getValue(options))) { + return false; + } + int globalMax = MaximumDesiredSize.getValue(options) - loop.loopBegin().graph().getNodeCount(); + if (globalMax <= 0) { + return false; + } + int maxNodes = counted.isExactTripCount() ? Options.ExactFullUnrollMaxNodes.getValue(options) : Options.FullUnrollMaxNodes.getValue(options); + for (Node usage : counted.getCounter().valueNode().usages()) { + if (usage instanceof CompareNode) { + CompareNode compare = (CompareNode) usage; + if (compare.getY().isConstant()) { + maxNodes += Options.FullUnrollConstantCompareBoost.getValue(options); + } + } + } + maxNodes = Math.min(maxNodes, globalMax); + int size = loop.inside().nodes().count(); + size -= 2; // remove the counted if and its non-exit begin + size -= loop.loopBegin().loopEnds().count(); + GraalError.guarantee(size >= 0, "Wrong size"); /* @formatter:off * The check below should not throw ArithmeticException because: * maxTrips is guaranteed to be >= 1 by the check above @@ -107,7 +128,7 @@ public class DefaultLoopPolicies implements LoopPolicies { * - 1 <= size <= Integer.MAX_VALUE * @formatter:on */ - if (maxTrips.isLessOrEqualTo(Options.FullUnrollMaxIterations.getValue(options)) && maxTrips.minus(1).times(size).isLessOrEqualTo(maxNodes)) { + if (maxTrips.minus(1).times(size).isLessOrEqualTo(maxNodes)) { // check whether we're allowed to unroll this loop return loop.canDuplicateLoop(); } else { @@ -240,5 +261,4 @@ public class DefaultLoopPolicies implements LoopPolicies { return false; } } - } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DerivedConvertedInductionVariable.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DerivedConvertedInductionVariable.java index c8766f6505c..7c2e509f44a 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DerivedConvertedInductionVariable.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DerivedConvertedInductionVariable.java @@ -81,8 +81,8 @@ public class DerivedConvertedInductionVariable extends DerivedInductionVariable } @Override - public ValueNode extremumNode(boolean assumePositiveTripCount, Stamp s) { - return base.extremumNode(assumePositiveTripCount, s); + public ValueNode extremumNode(boolean assumeLoopEntered, Stamp s) { + return base.extremumNode(assumeLoopEntered, s); } @Override diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DerivedOffsetInductionVariable.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DerivedOffsetInductionVariable.java index e3d59eae197..59aad1aa6bd 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DerivedOffsetInductionVariable.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DerivedOffsetInductionVariable.java @@ -99,8 +99,8 @@ public class DerivedOffsetInductionVariable extends DerivedInductionVariable { } @Override - public ValueNode extremumNode(boolean assumePositiveTripCount, Stamp stamp) { - return op(base.extremumNode(assumePositiveTripCount, stamp), IntegerConvertNode.convert(offset, stamp, graph(), NodeView.DEFAULT)); + public ValueNode extremumNode(boolean assumeLoopEntered, Stamp stamp) { + return op(base.extremumNode(assumeLoopEntered, stamp), IntegerConvertNode.convert(offset, stamp, graph(), NodeView.DEFAULT)); } @Override diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DerivedScaledInductionVariable.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DerivedScaledInductionVariable.java index 39842450c12..d7f79a99aef 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DerivedScaledInductionVariable.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/DerivedScaledInductionVariable.java @@ -110,8 +110,8 @@ public class DerivedScaledInductionVariable extends DerivedInductionVariable { } @Override - public ValueNode extremumNode(boolean assumePositiveTripCount, Stamp stamp) { - return mul(graph(), base.extremumNode(assumePositiveTripCount, stamp), IntegerConvertNode.convert(scale, stamp, graph(), NodeView.DEFAULT)); + public ValueNode extremumNode(boolean assumeLoopEntered, Stamp stamp) { + return mul(graph(), base.extremumNode(assumeLoopEntered, stamp), IntegerConvertNode.convert(scale, stamp, graph(), NodeView.DEFAULT)); } @Override diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/InductionVariable.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/InductionVariable.java index c399e2ccc05..08a50a7c196 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/InductionVariable.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/InductionVariable.java @@ -99,7 +99,7 @@ public abstract class InductionVariable { return extremumNode(false, valueNode().stamp(NodeView.DEFAULT)); } - public abstract ValueNode extremumNode(boolean assumePositiveTripCount, Stamp stamp); + public abstract ValueNode extremumNode(boolean assumeLoopEntered, Stamp stamp); public abstract boolean isConstantExtremum(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopEx.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopEx.java index 534d1746ce8..f3a398d247a 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopEx.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopEx.java @@ -76,8 +76,6 @@ import org.graalvm.compiler.nodes.debug.ControlFlowAnchored; import org.graalvm.compiler.nodes.extended.ValueAnchorNode; import org.graalvm.compiler.nodes.util.GraphUtil; -import jdk.vm.ci.code.BytecodeFrame; - public class LoopEx { private final Loop loop; private LoopFragmentInside inside; @@ -85,6 +83,7 @@ public class LoopEx { private CountedLoopInfo counted; private LoopsData data; private EconomicMap ivs; + private boolean countedLoopChecked; LoopEx(Loop loop, LoopsData data) { this.loop = loop; @@ -143,10 +142,12 @@ public class LoopEx { } public boolean isCounted() { + assert countedLoopChecked; return counted != null; } public CountedLoopInfo counted() { + assert countedLoopChecked; return counted; } @@ -211,6 +212,10 @@ public class LoopEx { } public boolean detectCounted() { + if (countedLoopChecked) { + return isCounted(); + } + countedLoopChecked = true; LoopBeginNode loopBegin = loopBegin(); FixedNode next = loopBegin.next(); while (next instanceof FixedGuardNode || next instanceof ValueAnchorNode || next instanceof FullInfopointNode) { @@ -219,8 +224,8 @@ public class LoopEx { if (next instanceof IfNode) { IfNode ifNode = (IfNode) next; boolean negated = false; - if (!loopBegin.isLoopExit(ifNode.falseSuccessor())) { - if (!loopBegin.isLoopExit(ifNode.trueSuccessor())) { + if (!isCfgLoopExit(ifNode.falseSuccessor())) { + if (!isCfgLoopExit(ifNode.trueSuccessor())) { return false; } negated = true; @@ -301,7 +306,7 @@ public class LoopEx { } break; default: - throw GraalError.shouldNotReachHere(); + throw GraalError.shouldNotReachHere(condition.toString()); } counted = new CountedLoopInfo(this, iv, ifNode, limit, oneOff, negated ? ifNode.falseSuccessor() : ifNode.trueSuccessor()); return true; @@ -309,6 +314,11 @@ public class LoopEx { return false; } + private boolean isCfgLoopExit(AbstractBeginNode begin) { + Block block = data.getCFG().blockFor(begin); + return loop.getDepth() > block.getLoopDepth() || loop.isNaturalExit(block); + } + public LoopsData loopsData() { return data; } @@ -321,7 +331,7 @@ public class LoopEx { work.add(cfg.blockFor(branch)); while (!work.isEmpty()) { Block b = work.remove(); - if (loop().getExits().contains(b)) { + if (loop().isLoopExit(b)) { assert !exits.contains(b.getBeginNode()); exits.add(b.getBeginNode()); } else if (blocks.add(b.getBeginNode())) { @@ -465,7 +475,7 @@ public class LoopEx { } if (node instanceof FrameState) { FrameState frameState = (FrameState) node; - if (frameState.bci == BytecodeFrame.AFTER_EXCEPTION_BCI || frameState.bci == BytecodeFrame.UNWIND_BCI) { + if (frameState.isExceptionHandlingBCI()) { return false; } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopFragment.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopFragment.java index 278f408788a..2cae741064d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopFragment.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopFragment.java @@ -56,6 +56,7 @@ import org.graalvm.compiler.nodes.ValuePhiNode; import org.graalvm.compiler.nodes.ValueProxyNode; import org.graalvm.compiler.nodes.VirtualState; import org.graalvm.compiler.nodes.cfg.Block; +import org.graalvm.compiler.nodes.cfg.ControlFlowGraph; import org.graalvm.compiler.nodes.java.MonitorEnterNode; import org.graalvm.compiler.nodes.spi.NodeWithState; import org.graalvm.compiler.nodes.virtual.CommitAllocationNode; @@ -144,7 +145,18 @@ public abstract class LoopFragment { protected abstract void beforeDuplication(); - protected abstract void finishDuplication(); + protected void finishDuplication() { + LoopEx originalLoopEx = original().loop(); + ControlFlowGraph cfg = originalLoopEx.loopsData().getCFG(); + for (LoopExitNode exit : originalLoopEx.loopBegin().loopExits().snapshot()) { + if (!originalLoopEx.loop().isLoopExit(cfg.blockFor(exit))) { + // this LoopExitNode is too low, we need to remove it otherwise it will be below + // merged exits + exit.removeExit(); + } + } + + } protected void patchNodes(final DuplicationReplacement dataFix) { if (isDuplicate() && !nodesReady) { @@ -388,42 +400,6 @@ public abstract class LoopFragment { }; } - public static NodeIterable toHirExits(final Iterable blocks) { - return new NodeIterable() { - - @Override - public Iterator iterator() { - final Iterator it = blocks.iterator(); - return new Iterator() { - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - /** - * Return the true LoopExitNode for this loop or the BeginNode for the block. - */ - @Override - public AbstractBeginNode next() { - Block next = it.next(); - LoopExitNode exit = next.getLoopExit(); - if (exit != null) { - return exit; - } - return next.getBeginNode(); - } - - @Override - public boolean hasNext() { - return it.hasNext(); - } - }; - } - - }; - } - /** * Merges the early exits (i.e. loop exits) that were duplicated as part of this fragment, with * the original fragment's exits. @@ -431,13 +407,12 @@ public abstract class LoopFragment { protected void mergeEarlyExits() { assert isDuplicate(); StructuredGraph graph = graph(); - for (AbstractBeginNode earlyExit : LoopFragment.toHirBlocks(original().loop().loop().getExits())) { - LoopExitNode loopEarlyExit = (LoopExitNode) earlyExit; - FixedNode next = loopEarlyExit.next(); - if (loopEarlyExit.isDeleted() || !this.original().contains(loopEarlyExit)) { + for (AbstractBeginNode earlyExit : LoopFragment.toHirBlocks(original().loop().loop().getLoopExits())) { + FixedNode next = earlyExit.next(); + if (earlyExit.isDeleted() || !this.original().contains(earlyExit)) { continue; } - AbstractBeginNode newEarlyExit = getDuplicatedNode(loopEarlyExit); + AbstractBeginNode newEarlyExit = getDuplicatedNode(earlyExit); if (newEarlyExit == null) { continue; } @@ -446,72 +421,79 @@ public abstract class LoopFragment { EndNode newEnd = graph.add(new EndNode()); merge.addForwardEnd(originalEnd); merge.addForwardEnd(newEnd); - loopEarlyExit.setNext(originalEnd); + earlyExit.setNext(originalEnd); newEarlyExit.setNext(newEnd); merge.setNext(next); - FrameState exitState = loopEarlyExit.stateAfter(); - if (exitState != null) { - FrameState originalExitState = exitState; - exitState = exitState.duplicateWithVirtualState(); - loopEarlyExit.setStateAfter(exitState); - merge.setStateAfter(originalExitState); - /* - * Using the old exit's state as the merge's state is necessary because some of the - * VirtualState nodes contained in the old exit's state may be shared by other - * dominated VirtualStates. Those dominated virtual states need to see the - * proxy->phi update that are applied below. - * - * We now update the original fragment's nodes accordingly: - */ - originalExitState.applyToVirtual(node -> original.nodes.clearAndGrow(node)); - exitState.applyToVirtual(node -> original.nodes.markAndGrow(node)); - } - FrameState finalExitState = exitState; - - for (Node anchored : loopEarlyExit.anchored().snapshot()) { - anchored.replaceFirstInput(loopEarlyExit, merge); + FrameState exitState = null; + if (earlyExit instanceof LoopExitNode) { + LoopExitNode earlyLoopExit = (LoopExitNode) earlyExit; + exitState = earlyLoopExit.stateAfter(); + if (exitState != null) { + FrameState originalExitState = exitState; + exitState = exitState.duplicateWithVirtualState(); + earlyLoopExit.setStateAfter(exitState); + merge.setStateAfter(originalExitState); + /* + * Using the old exit's state as the merge's state is necessary because some of + * the VirtualState nodes contained in the old exit's state may be shared by + * other dominated VirtualStates. Those dominated virtual states need to see the + * proxy->phi update that are applied below. + * + * We now update the original fragment's nodes accordingly: + */ + originalExitState.applyToVirtual(node -> original.nodes.clearAndGrow(node)); + exitState.applyToVirtual(node -> original.nodes.markAndGrow(node)); + } } - boolean newEarlyExitIsLoopExit = newEarlyExit instanceof LoopExitNode; - for (ProxyNode vpn : loopEarlyExit.proxies().snapshot()) { - if (vpn.hasNoUsages()) { - continue; - } - if (vpn.value() == null) { - assert vpn instanceof GuardProxyNode; - vpn.replaceAtUsages(null); - continue; - } - final ValueNode replaceWith; - ValueNode newVpn = prim(newEarlyExitIsLoopExit ? vpn : vpn.value()); - if (newVpn != null) { - PhiNode phi; - if (vpn instanceof ValueProxyNode) { - phi = graph.addWithoutUnique(new ValuePhiNode(vpn.stamp(NodeView.DEFAULT), merge)); - } else if (vpn instanceof GuardProxyNode) { - phi = graph.addWithoutUnique(new GuardPhiNode(merge)); + for (Node anchored : earlyExit.anchored().snapshot()) { + anchored.replaceFirstInput(earlyExit, merge); + } + + if (earlyExit instanceof LoopExitNode) { + LoopExitNode earlyLoopExit = (LoopExitNode) earlyExit; + FrameState finalExitState = exitState; + boolean newEarlyExitIsLoopExit = newEarlyExit instanceof LoopExitNode; + for (ProxyNode vpn : earlyLoopExit.proxies().snapshot()) { + if (vpn.hasNoUsages()) { + continue; + } + if (vpn.value() == null) { + assert vpn instanceof GuardProxyNode; + vpn.replaceAtUsages(null); + continue; + } + final ValueNode replaceWith; + ValueNode newVpn = prim(newEarlyExitIsLoopExit ? vpn : vpn.value()); + if (newVpn != null) { + PhiNode phi; + if (vpn instanceof ValueProxyNode) { + phi = graph.addWithoutUnique(new ValuePhiNode(vpn.stamp(NodeView.DEFAULT), merge)); + } else if (vpn instanceof GuardProxyNode) { + phi = graph.addWithoutUnique(new GuardPhiNode(merge)); + } else { + throw GraalError.shouldNotReachHere(); + } + phi.addInput(vpn); + phi.addInput(newVpn); + replaceWith = phi; } else { - throw GraalError.shouldNotReachHere(); + replaceWith = vpn.value(); } - phi.addInput(vpn); - phi.addInput(newVpn); - replaceWith = phi; - } else { - replaceWith = vpn.value(); - } - vpn.replaceAtMatchingUsages(replaceWith, usage -> { - if (merge.isPhiAtMerge(usage)) { - return false; - } - if (usage instanceof VirtualState) { - VirtualState stateUsage = (VirtualState) usage; - if (finalExitState != null && finalExitState.isPartOfThisState(stateUsage)) { + vpn.replaceAtMatchingUsages(replaceWith, usage -> { + if (merge.isPhiAtMerge(usage)) { return false; } - } - return true; - }); + if (usage instanceof VirtualState) { + VirtualState stateUsage = (VirtualState) usage; + if (finalExitState != null && finalExitState.isPartOfThisState(stateUsage)) { + return false; + } + } + return true; + }); + } } } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopFragmentInside.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopFragmentInside.java index 64c72699f07..a25d9f3f421 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopFragmentInside.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopFragmentInside.java @@ -390,11 +390,6 @@ public class LoopFragmentInside extends LoopFragment { }; } - @Override - protected void finishDuplication() { - // TODO (gd) ? - } - @Override protected void beforeDuplication() { // Nothing to do diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopFragmentWhole.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopFragmentWhole.java index df1bd76ef7a..bc00924750b 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopFragmentWhole.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/LoopFragmentWhole.java @@ -24,9 +24,7 @@ package org.graalvm.compiler.loop; -import jdk.internal.vm.compiler.collections.EconomicSet; import org.graalvm.compiler.core.common.cfg.Loop; -import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.graph.Graph; import org.graalvm.compiler.graph.Graph.DuplicationReplacement; import org.graalvm.compiler.graph.Node; @@ -34,8 +32,6 @@ import org.graalvm.compiler.graph.NodeBitMap; import org.graalvm.compiler.nodes.EndNode; import org.graalvm.compiler.nodes.FixedNode; import org.graalvm.compiler.nodes.LoopBeginNode; -import org.graalvm.compiler.nodes.LoopExitNode; -import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.cfg.Block; @@ -68,7 +64,7 @@ public class LoopFragmentWhole extends LoopFragment { public NodeBitMap nodes() { if (nodes == null) { Loop loop = loop().loop(); - nodes = LoopFragment.computeNodes(graph(), LoopFragment.toHirBlocks(loop.getBlocks()), LoopFragment.toHirExits(loop.getExits())); + nodes = LoopFragment.computeNodes(graph(), LoopFragment.toHirBlocks(loop.getBlocks()), LoopFragment.toHirBlocks(loop.getLoopExits())); } return nodes; } @@ -107,47 +103,9 @@ public class LoopFragmentWhole extends LoopFragment { return loop().entryPoint(); } - @Override - protected void finishDuplication() { - // TODO (gd) ? - } - - void cleanupLoopExits() { - LoopBeginNode loopBegin = original().loop().loopBegin(); - assert nodes == null || nodes.contains(loopBegin); - StructuredGraph graph = loopBegin.graph(); - if (graph.getGuardsStage() == StructuredGraph.GuardsStage.AFTER_FSA) { - // After FrameStateAssignment ControlFlowGraph treats loop exits differently which means - // that the LoopExitNodes can be in a block which post dominates the true loop exit. For - // cloning to work right they must agree. - EconomicSet exits = EconomicSet.create(); - for (Block exitBlock : original().loop().loop().getExits()) { - LoopExitNode exitNode = exitBlock.getLoopExit(); - if (exitNode == null) { - exitNode = graph.add(new LoopExitNode(loopBegin)); - graph.addAfterFixed(exitBlock.getBeginNode(), exitNode); - if (nodes != null) { - nodes.mark(exitNode); - } - graph.getDebug().dump(DebugContext.VERBOSE_LEVEL, graph, "Adjusting loop exit node for %s", loopBegin); - } - exits.add(exitNode); - } - for (LoopExitNode exitNode : loopBegin.loopExits()) { - if (!exits.contains(exitNode)) { - if (nodes != null) { - nodes.clear(exitNode); - } - graph.removeFixed(exitNode); - } - } - } - - } - @Override protected void beforeDuplication() { - cleanupLoopExits(); + // nothing to do } @Override diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/MathUtil.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/MathUtil.java index 37f58174f06..de06c796bdc 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/MathUtil.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.loop/src/org/graalvm/compiler/loop/MathUtil.java @@ -88,7 +88,7 @@ public class MathUtil { if (isConstantOne(divisor)) { return dividend; } - ValueNode div = graph.addOrUniqueWithInputs(createDiv.apply(dividend, divisor)); + ValueNode div = createDiv.apply(dividend, divisor); if (div instanceof FixedBinaryNode) { FixedBinaryNode fixedDiv = (FixedBinaryNode) div; if (before.predecessor() instanceof FixedBinaryNode) { @@ -98,7 +98,7 @@ public class MathUtil { return binaryPredecessor; } } - graph.addBeforeFixed(before, fixedDiv); + graph.addBeforeFixed(before, graph.addOrUniqueWithInputs(fixedDiv)); } return div; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.microbenchmarks/src/org/graalvm/compiler/microbenchmarks/lir/GraalCompilerState.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.microbenchmarks/src/org/graalvm/compiler/microbenchmarks/lir/GraalCompilerState.java index 65693974720..2582e805540 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.microbenchmarks/src/org/graalvm/compiler/microbenchmarks/lir/GraalCompilerState.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.microbenchmarks/src/org/graalvm/compiler/microbenchmarks/lir/GraalCompilerState.java @@ -46,12 +46,13 @@ import org.graalvm.compiler.core.LIRGenerationPhase.LIRGenerationContext; import org.graalvm.compiler.core.common.CompilationIdentifier; import org.graalvm.compiler.core.common.alloc.ComputeBlockOrder; import org.graalvm.compiler.core.common.cfg.AbstractBlockBase; +import org.graalvm.compiler.core.gen.LIRCompilerBackend; +import org.graalvm.compiler.core.gen.LIRGenerationProvider; import org.graalvm.compiler.core.target.Backend; -import org.graalvm.compiler.debug.DebugHandlersFactory; import org.graalvm.compiler.debug.DebugContext; +import org.graalvm.compiler.debug.DebugHandlersFactory; import org.graalvm.compiler.lir.LIR; import org.graalvm.compiler.lir.asm.CompilationResultBuilderFactory; -import org.graalvm.compiler.lir.framemap.FrameMapBuilder; import org.graalvm.compiler.lir.gen.LIRGenerationResult; import org.graalvm.compiler.lir.gen.LIRGeneratorTool; import org.graalvm.compiler.lir.phases.AllocationPhase.AllocationContext; @@ -88,6 +89,7 @@ import jdk.vm.ci.code.TargetDescription; import jdk.vm.ci.meta.ConstantReflectionProvider; import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.meta.ResolvedJavaMethod; +import jdk.vm.ci.meta.SpeculationLog; /** * State providing a new copy of a graph for each invocation of a benchmark. Subclasses of this @@ -374,10 +376,10 @@ public abstract class GraalCompilerState { linearScanOrder = ComputeBlockOrder.computeLinearScanOrder(blocks.length, startBlock); LIR lir = new LIR(cfg, linearScanOrder, codeEmittingOrder, getGraphOptions(), getGraphDebug()); - FrameMapBuilder frameMapBuilder = request.backend.newFrameMapBuilder(registerConfig); - lirGenRes = request.backend.newLIRGenerationResult(graph.compilationId(), lir, frameMapBuilder, request.graph, stub); - lirGenTool = request.backend.newLIRGenerator(lirGenRes); - nodeLirGen = request.backend.newNodeLIRBuilder(request.graph, lirGenTool); + LIRGenerationProvider lirBackend = (LIRGenerationProvider) request.backend; + lirGenRes = lirBackend.newLIRGenerationResult(graph.compilationId(), lir, registerConfig, request.graph, stub); + lirGenTool = lirBackend.newLIRGenerator(lirGenRes); + nodeLirGen = lirBackend.newNodeLIRBuilder(request.graph, lirGenTool); } protected OptionValues getGraphOptions() { @@ -460,8 +462,10 @@ public abstract class GraalCompilerState { */ protected final void emitCode() { int bytecodeSize = request.graph.method() == null ? 0 : request.graph.getBytecodeSize(); + SpeculationLog speculationLog = null; request.compilationResult.setHasUnsafeAccess(request.graph.hasUnsafeAccess()); - GraalCompiler.emitCode(request.backend, request.graph.getAssumptions(), request.graph.method(), request.graph.getMethods(), request.graph.getFields(), bytecodeSize, lirGenRes, + LIRCompilerBackend.emitCode(request.backend, request.graph.getAssumptions(), request.graph.method(), request.graph.getMethods(), request.graph.getFields(), + speculationLog, bytecodeSize, lirGenRes, request.compilationResult, request.installedCodeOwner, request.factory); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes.test/src/org/graalvm/compiler/nodes/test/IfNodeCanonicalizationTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes.test/src/org/graalvm/compiler/nodes/test/IfNodeCanonicalizationTest.java index a513a5348d4..bef77ebd0f7 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes.test/src/org/graalvm/compiler/nodes/test/IfNodeCanonicalizationTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes.test/src/org/graalvm/compiler/nodes/test/IfNodeCanonicalizationTest.java @@ -37,7 +37,7 @@ import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions; import org.graalvm.compiler.nodes.calc.SubNode; import org.graalvm.compiler.phases.common.CanonicalizerPhase; -import org.graalvm.compiler.phases.common.ConvertDeoptimizeToGuardPhase; +import org.graalvm.compiler.loop.phases.ConvertDeoptimizeToGuardPhase; import org.graalvm.compiler.phases.common.IterativeConditionalEliminationPhase; import org.graalvm.compiler.phases.tiers.PhaseContext; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/AbstractFixedGuardNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/AbstractFixedGuardNode.java index fdbb939957b..75fa26fa838 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/AbstractFixedGuardNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/AbstractFixedGuardNode.java @@ -126,6 +126,14 @@ public abstract class AbstractFixedGuardNode extends DeoptimizingFixedWithNextNo try (DebugCloseable position = this.withNodeSourcePosition()) { FixedNode currentNext = next(); setNext(null); + if (currentNext instanceof AbstractBeginNode && currentNext instanceof StateSplit && ((StateSplit) currentNext).stateAfter() != null) { + // Force an extra BeginNode in case any guarded Nodes are inputs to the StateSplit + BeginNode begin = graph().add(new BeginNode()); + begin.setNodeSourcePosition(getNoDeoptSuccessorPosition()); + begin.setNext(currentNext); + currentNext = begin; + } + DeoptimizeNode deopt = graph().add(new DeoptimizeNode(action, reason, speculation)); deopt.setStateBefore(stateBefore()); IfNode ifNode; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/ComputeObjectAddressNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/ComputeObjectAddressNode.java index 9704f4ca73e..f9be9e3972e 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/ComputeObjectAddressNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/ComputeObjectAddressNode.java @@ -30,6 +30,7 @@ import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.graph.NodeClass; import org.graalvm.compiler.nodeinfo.NodeInfo; import org.graalvm.compiler.nodeinfo.NodeSize; +import org.graalvm.compiler.nodes.calc.SignExtendNode; import org.graalvm.compiler.nodes.debug.ControlFlowAnchored; import org.graalvm.compiler.nodes.spi.Lowerable; import org.graalvm.compiler.nodes.spi.LoweringTool; @@ -50,7 +51,7 @@ public final class ComputeObjectAddressNode extends FixedWithNextNode implements public ComputeObjectAddressNode(ValueNode obj, ValueNode offset) { super(TYPE, StampFactory.forKind(JavaKind.Long)); this.object = obj; - this.offset = offset; + this.offset = SignExtendNode.create(offset, 64, NodeView.DEFAULT); } @NodeIntrinsic diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/ConstantNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/ConstantNode.java index 8b059fc4b37..0e290bda354 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/ConstantNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/ConstantNode.java @@ -54,6 +54,7 @@ import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.meta.PrimitiveConstant; +import jdk.vm.ci.meta.ResolvedJavaType; /** * The {@code ConstantNode} represents a {@link Constant constant}. @@ -99,6 +100,10 @@ public final class ConstantNode extends FloatingNode implements LIRLowerable, Ar } } + public ConstantNode(@InjectedNodeParameter Stamp stamp, @InjectedNodeParameter ConstantReflectionProvider constantReflection, @ConstantNodeParameter ResolvedJavaType type) { + this(constantReflection.asJavaClass(type), stamp); + } + /** * @return the constant value represented by this node */ @@ -546,4 +551,7 @@ public final class ConstantNode extends FloatingNode implements LIRLowerable, Ar } return ConstantNode.forInt(length); } + + @NodeIntrinsic + public static native Class forClass(@ConstantNodeParameter ResolvedJavaType type); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/EncodedGraph.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/EncodedGraph.java index 2a97a881565..a1d3e7ec654 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/EncodedGraph.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/EncodedGraph.java @@ -82,7 +82,7 @@ public class EncodedGraph { return startOffset; } - protected Object[] getObjects() { + public Object[] getObjects() { return objects; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/FrameState.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/FrameState.java index e2b46417d8b..1545ffc063d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/FrameState.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/FrameState.java @@ -681,4 +681,8 @@ public final class FrameState extends VirtualState implements IterableNodeType { } return false; } + + public boolean isExceptionHandlingBCI() { + return bci == BytecodeFrame.AFTER_EXCEPTION_BCI || bci == BytecodeFrame.UNWIND_BCI; + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GraphDecoder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GraphDecoder.java index 6807f9dea06..636080aaa97 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GraphDecoder.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GraphDecoder.java @@ -132,6 +132,7 @@ public class GraphDecoder { nodeStartOffsets[i] = encodedGraph.getStartOffset() - reader.getUVInt(); } encodedGraph.nodeStartOffsets = nodeStartOffsets; + graph.setGuardsStage((StructuredGraph.GuardsStage) readObject(this)); } } else { reader = null; @@ -1322,7 +1323,7 @@ public class GraphDecoder { } protected Object readObject(MethodScope methodScope) { - return methodScope.encodedGraph.getObjects()[methodScope.reader.getUVInt()]; + return methodScope.encodedGraph.getObject(methodScope.reader.getUVInt()); } /** diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GraphEncoder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GraphEncoder.java index a9db63e3d36..707ec4b2a50 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GraphEncoder.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/GraphEncoder.java @@ -112,9 +112,13 @@ import jdk.vm.ci.code.Architecture; */ public class GraphEncoder { - /** The orderId that always represents {@code null}. */ + /** + * The orderId that always represents {@code null}. + */ public static final int NULL_ORDER_ID = 0; - /** The orderId of the {@link StructuredGraph#start() start node} of the encoded graph. */ + /** + * The orderId of the {@link StructuredGraph#start() start node} of the encoded graph. + */ public static final int START_NODE_ORDER_ID = 1; /** * The orderId of the first actual node after the {@link StructuredGraph#start() start node}. @@ -148,6 +152,8 @@ public class GraphEncoder { /** The last snapshot of {@link #nodeClasses} that was retrieved. */ protected NodeClass[] nodeClassesArray; + protected DebugContext debug; + /** * Utility method that does everything necessary to encode a single graph. */ @@ -160,7 +166,12 @@ public class GraphEncoder { } public GraphEncoder(Architecture architecture) { + this(architecture, null); + } + + public GraphEncoder(Architecture architecture, DebugContext debug) { this.architecture = architecture; + this.debug = debug; objects = FrequencyEncoder.createEqualityEncoder(); nodeClasses = FrequencyEncoder.createIdentityEncoder(); writer = UnsafeArrayTypeWriter.create(architecture.supportsUnalignedMemoryAccess()); @@ -170,6 +181,7 @@ public class GraphEncoder { * Must be invoked before {@link #finishPrepare()} and {@link #encode}. */ public void prepare(StructuredGraph graph) { + objects.addObject(graph.getGuardsStage()); for (Node node : graph.getNodes()) { NodeClass nodeClass = node.getNodeClass(); nodeClasses.addObject(nodeClass); @@ -288,9 +300,10 @@ public class GraphEncoder { for (int i = 0; i < nodeCount; i++) { writer.putUV(metadataStart - nodeStartOffsets[i]); } + writeObjectId(graph.getGuardsStage()); /* Check that the decoding of the encode graph is the same as the input. */ - assert verifyEncoding(graph, new EncodedGraph(getEncoding(), metadataStart, getObjects(), getNodeClasses(), graph), architecture); + assert verifyEncoding(graph, new EncodedGraph(getEncoding(), metadataStart, getObjects(), getNodeClasses(), graph)); return metadataStart; } @@ -434,10 +447,10 @@ public class GraphEncoder { * original graph. */ @SuppressWarnings("try") - public static boolean verifyEncoding(StructuredGraph originalGraph, EncodedGraph encodedGraph, Architecture architecture) { - DebugContext debug = originalGraph.getDebug(); + public boolean verifyEncoding(StructuredGraph originalGraph, EncodedGraph encodedGraph) { + DebugContext debugContext = debug != null ? debug : originalGraph.getDebug(); // @formatter:off - StructuredGraph decodedGraph = new StructuredGraph.Builder(originalGraph.getOptions(), debug, AllowAssumptions.YES). + StructuredGraph decodedGraph = new StructuredGraph.Builder(originalGraph.getOptions(), debugContext, AllowAssumptions.YES). method(originalGraph.method()). setIsSubstitution(originalGraph.isSubstitution()). trackNodeSourcePosition(originalGraph.trackNodeSourcePosition()). @@ -451,9 +464,9 @@ public class GraphEncoder { GraphComparison.verifyGraphsEqual(originalGraph, decodedGraph); } catch (Throwable ex) { originalGraph.getDebug(); - try (DebugContext.Scope scope = debug.scope("GraphEncoder")) { - debug.dump(DebugContext.VERBOSE_LEVEL, originalGraph, "Original Graph"); - debug.dump(DebugContext.VERBOSE_LEVEL, decodedGraph, "Decoded Graph"); + try (DebugContext.Scope scope = debugContext.scope("GraphEncoder")) { + debugContext.dump(DebugContext.VERBOSE_LEVEL, originalGraph, "Original Graph"); + debugContext.dump(DebugContext.VERBOSE_LEVEL, decodedGraph, "Decoded Graph"); } throw ex; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/IfNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/IfNode.java index 2ec3e71f974..682219b415f 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/IfNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/IfNode.java @@ -78,6 +78,7 @@ import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.meta.PrimitiveConstant; import jdk.vm.ci.meta.ResolvedJavaMethod; import jdk.vm.ci.meta.ResolvedJavaType; +import jdk.vm.ci.meta.TriState; /** * The {@code IfNode} represents a branch that can go one of two directions depending on the outcome @@ -868,6 +869,30 @@ public final class IfNode extends ControlSplitNode implements Simplifiable, LIRL } } + private ValueNode canonicalizeConditionalViaImplies(ValueNode trueValue, ValueNode falseValue) { + ValueNode collapsedTrue = trueValue; + ValueNode collapsedFalse = falseValue; + boolean simplify = false; + if (trueValue instanceof ConditionalNode) { + TriState result = condition().implies(false, ((ConditionalNode) trueValue).condition()); + if (result.isKnown()) { + simplify = true; + collapsedTrue = result.toBoolean() ? ((ConditionalNode) trueValue).trueValue() : ((ConditionalNode) trueValue).falseValue(); + } + } + if (falseValue instanceof ConditionalNode) { + TriState result = condition().implies(true, ((ConditionalNode) falseValue).condition()); + if (result.isKnown()) { + simplify = true; + collapsedFalse = result.toBoolean() ? ((ConditionalNode) falseValue).trueValue() : ((ConditionalNode) falseValue).falseValue(); + } + } + if (simplify) { + return graph().unique(new ConditionalNode(condition(), collapsedTrue, collapsedFalse)); + } + return null; + } + private ValueNode canonicalizeConditionalCascade(SimplifierTool tool, ValueNode trueValue, ValueNode falseValue) { if (trueValue.getStackKind() != falseValue.getStackKind()) { return null; @@ -877,7 +902,16 @@ public final class IfNode extends ControlSplitNode implements Simplifiable, LIRL } if (trueValue.isConstant() && falseValue.isConstant()) { return graph().unique(new ConditionalNode(condition(), trueValue, falseValue)); - } else if (!graph().isAfterExpandLogic()) { + } + ValueNode value = canonicalizeConditionalViaImplies(trueValue, falseValue); + if (value != null) { + return value; + } + if (!graph().isAfterExpandLogic()) { + /* + * !isAfterExpandLogic() => Cannot spawn NormalizeCompareNodes after lowering in the + * ExpandLogicPhase. + */ ConditionalNode conditional = null; ValueNode constant = null; boolean negateCondition; @@ -905,8 +939,11 @@ public final class IfNode extends ControlSplitNode implements Simplifiable, LIRL double shortCutProbability = probability(trueSuccessor()); LogicNode newCondition = LogicNode.or(condition(), negateCondition, conditional.condition(), negateConditionalCondition, shortCutProbability); return graph().unique(new ConditionalNode(newCondition, constant, otherValue)); - } else if (constant.isJavaConstant() && conditional.trueValue().isJavaConstant() && conditional.falseValue().isJavaConstant() && condition() instanceof CompareNode && + } + + if (constant.isJavaConstant() && conditional.trueValue().isJavaConstant() && conditional.falseValue().isJavaConstant() && condition() instanceof CompareNode && conditional.condition() instanceof CompareNode) { + Condition cond1 = ((CompareNode) condition()).condition().asCondition(); if (negateCondition) { cond1 = cond1.negate(); @@ -923,27 +960,19 @@ public final class IfNode extends ControlSplitNode implements Simplifiable, LIRL sameVars = true; cond2 = cond2.mirror(); } - // cond2 is EQ, LT, or GT if (sameVars) { JavaKind stackKind = conditional.trueValue().stamp(NodeView.from(tool)).getStackKind(); assert !stackKind.isNumericFloat(); - long c1 = constant.asJavaConstant().asLong(); - long c2 = conditional.trueValue().asJavaConstant().asLong(); - long c3 = conditional.falseValue().asJavaConstant().asLong(); - // `x cond1 y ? c1 : (x cond2 y ? c2 : c3)` - if (cond1 == Condition.GE && cond2 == Condition.LT) { - // x >= y ? v1 : (x < y ? v2 : v3) => x >= y ? v1 : v2 - return graph().unique(new ConditionalNode(condition(), conditional.trueValue(), constant)); - } else if (cond1 == Condition.GE && cond2 == Condition.GT) { - // x >= y ? v1 : (x > y ? v2 : v3) => x >= y ? v1 : v3 - return graph().unique(new ConditionalNode(condition(), conditional.falseValue(), constant)); - } else if (cond1 == Condition.EQ && cond2 == Condition.EQ) { - // x == y ? v1 : (x == y ? v2 : v3) => x == y ? v1 : v3 - return graph().unique(new ConditionalNode(condition(), conditional.falseValue(), constant)); - } else if (cond1 == Condition.NE && cond2 == Condition.LT) { - // x != y ? v1 : (x < y ? v2 : v3) => x != y ? v1 : v3 - return graph().unique(new ConditionalNode(condition(), conditional.falseValue(), constant)); - } else if (cond1 == Condition.LT && cond2 == Condition.EQ && c1 == -1 && c2 == 0 && c3 == 1) { + + ValueNode v1 = constant; + ValueNode v2 = conditional.trueValue(); + ValueNode v3 = conditional.falseValue(); + + long c1 = v1.asJavaConstant().asLong(); + long c2 = v2.asJavaConstant().asLong(); + long c3 = v3.asJavaConstant().asLong(); + + if (cond1 == Condition.LT && cond2 == Condition.EQ && c1 == -1 && c2 == 0 && c3 == 1) { // x < y ? -1 : (x == y ? 0 : 1) => x cmp y return graph().unique(new NormalizeCompareNode(x, y, stackKind, false)); } else if (cond1 == Condition.LT && cond2 == Condition.EQ && c1 == 1 && c2 == 0 && c3 == -1) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LogicNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LogicNode.java index 8cb8f847904..20e714f8e3d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LogicNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LogicNode.java @@ -91,6 +91,18 @@ public abstract class LogicNode extends FloatingNode implements IndirectCanonica * @param other the other condition. */ public TriState implies(boolean thisNegated, LogicNode other) { + if (this == other) { + return TriState.get(!thisNegated); + } + if (other instanceof LogicNegationNode) { + return flip(this.implies(thisNegated, ((LogicNegationNode) other).getValue())); + } return TriState.UNKNOWN; } + + private static TriState flip(TriState triState) { + return triState.isUnknown() + ? triState + : TriState.get(!triState.toBoolean()); + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LoopBeginNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LoopBeginNode.java index ee4d2834401..a4c28172923 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LoopBeginNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LoopBeginNode.java @@ -53,6 +53,7 @@ public final class LoopBeginNode extends AbstractMergeNode implements IterableNo protected int inversionCount; protected LoopType loopType; protected int unrollFactor; + protected boolean osrLoop; public enum LoopType { SIMPLE_LOOP, @@ -303,11 +304,6 @@ public final class LoopBeginNode extends AbstractMergeNode implements IterableNo return begin instanceof LoopExitNode && ((LoopExitNode) begin).loopBegin() == this; } - public LoopExitNode getSingleLoopExit() { - assert loopExits().count() == 1; - return loopExits().first(); - } - public LoopEndNode getSingleLoopEnd() { assert loopEnds().count() == 1; return loopEnds().first(); @@ -317,12 +313,7 @@ public final class LoopBeginNode extends AbstractMergeNode implements IterableNo public void removeExits() { for (LoopExitNode loopexit : loopExits().snapshot()) { try (DebugCloseable position = graph().withNodeSourcePosition(loopexit)) { - loopexit.removeProxies(); - FrameState loopStateAfter = loopexit.stateAfter(); - graph().replaceFixedWithFixed(loopexit, graph().add(new BeginNode())); - if (loopStateAfter != null) { - GraphUtil.tryKillUnused(loopStateAfter); - } + loopexit.removeExit(); } } } @@ -415,4 +406,12 @@ public final class LoopBeginNode extends AbstractMergeNode implements IterableNo } } } + + public void markOsrLoop() { + osrLoop = true; + } + + public boolean isOsrLoop() { + return osrLoop; + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LoopExitNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LoopExitNode.java index 895e16e7421..d174b66466f 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LoopExitNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/LoopExitNode.java @@ -35,6 +35,7 @@ import org.graalvm.compiler.graph.iterators.NodeIterable; import org.graalvm.compiler.graph.spi.Simplifiable; import org.graalvm.compiler.graph.spi.SimplifierTool; import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.util.GraphUtil; @NodeInfo(allowedUsageTypes = {Association}, cycles = CYCLES_0, size = SIZE_0) public final class LoopExitNode extends BeginStateSplitNode implements IterableNodeType, Simplifiable { @@ -102,6 +103,15 @@ public final class LoopExitNode extends BeginStateSplitNode implements IterableN }); } + public void removeExit() { + this.removeProxies(); + FrameState loopStateAfter = this.stateAfter(); + graph().replaceFixedWithFixed(this, graph().add(new BeginNode())); + if (loopStateAfter != null) { + GraphUtil.tryKillUnused(loopStateAfter); + } + } + @Override public void simplify(SimplifierTool tool) { Node prev = this.predecessor(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/PiNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/PiNode.java index b749ada090f..0bf64924227 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/PiNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/PiNode.java @@ -32,6 +32,7 @@ import org.graalvm.compiler.core.common.type.ObjectStamp; import org.graalvm.compiler.core.common.type.Stamp; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.core.common.type.TypeReference; +import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.graph.IterableNodeType; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeClass; @@ -100,7 +101,7 @@ public class PiNode extends FloatingGuardedNode implements LIRLowerable, Virtual } public static ValueNode create(ValueNode object, Stamp stamp) { - ValueNode value = canonical(object, stamp, null); + ValueNode value = canonical(object, stamp, null, null); if (value != null) { return value; } @@ -108,7 +109,7 @@ public class PiNode extends FloatingGuardedNode implements LIRLowerable, Virtual } public static ValueNode create(ValueNode object, Stamp stamp, ValueNode guard) { - ValueNode value = canonical(object, stamp, (GuardingNode) guard); + ValueNode value = canonical(object, stamp, (GuardingNode) guard, null); if (value != null) { return value; } @@ -117,7 +118,7 @@ public class PiNode extends FloatingGuardedNode implements LIRLowerable, Virtual public static ValueNode create(ValueNode object, ValueNode guard) { Stamp stamp = AbstractPointerStamp.pointerNonNull(object.stamp(NodeView.DEFAULT)); - ValueNode value = canonical(object, stamp, (GuardingNode) guard); + ValueNode value = canonical(object, stamp, (GuardingNode) guard, null); if (value != null) { return value; } @@ -127,7 +128,7 @@ public class PiNode extends FloatingGuardedNode implements LIRLowerable, Virtual @SuppressWarnings("unused") public static boolean intrinsify(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode object, ValueNode guard) { Stamp stamp = AbstractPointerStamp.pointerNonNull(object.stamp(NodeView.DEFAULT)); - ValueNode value = canonical(object, stamp, (GuardingNode) guard); + ValueNode value = canonical(object, stamp, (GuardingNode) guard, null); if (value == null) { value = new PiNode(object, stamp, guard); } @@ -139,7 +140,7 @@ public class PiNode extends FloatingGuardedNode implements LIRLowerable, Virtual public static boolean intrinsify(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode object, ResolvedJavaType toType, boolean exactType, boolean nonNull) { Stamp stamp = StampFactory.object(exactType ? TypeReference.createExactTrusted(toType) : TypeReference.createWithoutAssumptions(toType), nonNull || StampTool.isPointerNonNull(object.stamp(NodeView.DEFAULT))); - ValueNode value = canonical(object, stamp, null); + ValueNode value = canonical(object, stamp, null, null); if (value == null) { value = new PiNode(object, stamp); } @@ -177,13 +178,16 @@ public class PiNode extends FloatingGuardedNode implements LIRLowerable, Virtual ValueNode alias = tool.getAlias(object()); if (alias instanceof VirtualObjectNode) { VirtualObjectNode virtual = (VirtualObjectNode) alias; - if (StampTool.typeOrNull(this) != null && StampTool.typeOrNull(this).isAssignableFrom(virtual.type())) { + ResolvedJavaType type = StampTool.typeOrNull(this, tool.getMetaAccess()); + if (type != null && type.isAssignableFrom(virtual.type())) { tool.replaceWithVirtual(virtual); + } else { + tool.getDebug().log(DebugContext.INFO_LEVEL, "could not virtualize Pi because of type mismatch: %s %s vs %s", this, type, virtual.type()); } } } - public static ValueNode canonical(ValueNode object, Stamp stamp, GuardingNode guard) { + public static ValueNode canonical(ValueNode object, Stamp stamp, GuardingNode guard, PiNode self) { // Use most up to date stamp. Stamp computedStamp = stamp.improveWith(object.stamp(NodeView.DEFAULT)); @@ -201,8 +205,9 @@ public class PiNode extends FloatingGuardedNode implements LIRLowerable, Virtual } } else { for (Node n : guard.asNode().usages()) { - if (n instanceof PiNode) { + if (n instanceof PiNode && n != self) { PiNode otherPi = (PiNode) n; + assert otherPi.guard == guard; if (object == otherPi.object() && computedStamp.equals(otherPi.stamp(NodeView.DEFAULT))) { /* * Two PiNodes with the same guard and same result, so return the one with @@ -221,7 +226,7 @@ public class PiNode extends FloatingGuardedNode implements LIRLowerable, Virtual @Override public Node canonical(CanonicalizerTool tool) { - Node value = canonical(object(), stamp(NodeView.DEFAULT), getGuard()); + Node value = canonical(object(), piStamp(), getGuard(), this); if (value != null) { return value; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/BinaryArithmeticNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/BinaryArithmeticNode.java index 83acfbf47c8..eead2ce4ced 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/BinaryArithmeticNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/BinaryArithmeticNode.java @@ -90,6 +90,26 @@ public abstract class BinaryArithmeticNode extends BinaryNode implements Ari if (result != null) { return result; } + if (forX instanceof ConditionalNode && forY.isConstant() && forX.hasExactlyOneUsage()) { + ConditionalNode conditionalNode = (ConditionalNode) forX; + BinaryOp arithmeticOp = getArithmeticOp(); + ConstantNode trueConstant = tryConstantFold(arithmeticOp, conditionalNode.trueValue(), forY, this.stamp(view), view); + if (trueConstant != null) { + ConstantNode falseConstant = tryConstantFold(arithmeticOp, conditionalNode.falseValue(), forY, this.stamp(view), view); + if (falseConstant != null) { + // @formatter:off + /* The arithmetic is folded into a constant on both sides of the conditional. + * Example: + * (cond ? -5 : 5) + 100 + * canonicalizes to: + * (cond ? 95 : 105) + */ + // @formatter:on + return ConditionalNode.create(conditionalNode.condition, trueConstant, + falseConstant, view); + } + } + } return this; } @@ -118,6 +138,10 @@ public abstract class BinaryArithmeticNode extends BinaryNode implements Ari return AddNode.create(v1, v2, view); } + public static ValueNode add(ValueNode v1, ValueNode v2) { + return add(v1, v2, NodeView.DEFAULT); + } + public static ValueNode mul(StructuredGraph graph, ValueNode v1, ValueNode v2, NodeView view) { return graph.addOrUniqueWithInputs(MulNode.create(v1, v2, view)); } @@ -126,6 +150,10 @@ public abstract class BinaryArithmeticNode extends BinaryNode implements Ari return MulNode.create(v1, v2, view); } + public static ValueNode mul(ValueNode v1, ValueNode v2) { + return mul(v1, v2, NodeView.DEFAULT); + } + public static ValueNode sub(StructuredGraph graph, ValueNode v1, ValueNode v2, NodeView view) { return graph.addOrUniqueWithInputs(SubNode.create(v1, v2, view)); } @@ -134,6 +162,10 @@ public abstract class BinaryArithmeticNode extends BinaryNode implements Ari return SubNode.create(v1, v2, view); } + public static ValueNode sub(ValueNode v1, ValueNode v2) { + return sub(v1, v2, NodeView.DEFAULT); + } + public static ValueNode branchlessMin(ValueNode v1, ValueNode v2, NodeView view) { if (v1.isDefaultConstant() && !v2.isDefaultConstant()) { return branchlessMin(v2, v1, view); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerDivRemNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerDivRemNode.java index 3da24d4c6d5..e8393018078 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerDivRemNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerDivRemNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerEqualsNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerEqualsNode.java index c42a15e19cc..f5ac4a845da 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerEqualsNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerEqualsNode.java @@ -165,15 +165,19 @@ public final class IntegerEqualsNode extends CompareNode implements BinaryCommut ValueNode v1 = null; ValueNode v2 = null; if (addX.getX() == addY.getX()) { + // (x + y) == (x + z) => y == z v1 = addX.getY(); v2 = addY.getY(); } else if (addX.getX() == addY.getY()) { + // (x + y) == (z + x) => y == z v1 = addX.getY(); v2 = addY.getX(); } else if (addX.getY() == addY.getX()) { + // (y + x) == (x + z) => y == z v1 = addX.getX(); v2 = addY.getY(); } else if (addX.getY() == addY.getY()) { + // (y + x) == (z + x) => y == z v1 = addX.getX(); v2 = addY.getX(); } @@ -183,6 +187,64 @@ public final class IntegerEqualsNode extends CompareNode implements BinaryCommut } } + if (forX instanceof SubNode && forY instanceof SubNode) { + SubNode subX = (SubNode) forX; + SubNode subY = (SubNode) forY; + ValueNode v1 = null; + ValueNode v2 = null; + if (subX.getX() == subY.getX()) { + // (x - y) == (x - z) => y == z + v1 = subX.getY(); + v2 = subY.getY(); + } else if (subX.getY() == subY.getY()) { + // (y - x) == (z - x) => y == z + v1 = subX.getX(); + v2 = subY.getX(); + } + if (v1 != null) { + assert v2 != null; + return create(v1, v2, view); + } + } + + if (forX instanceof AddNode) { + AddNode addNode = (AddNode) forX; + if (addNode.getX() == forY) { + // (x + y) == x => y == 0 + return create(addNode.getY(), ConstantNode.forIntegerStamp(view.stamp(addNode), 0), view); + } else if (addNode.getY() == forY) { + // (x + y) == y => x == 0 + return create(addNode.getX(), ConstantNode.forIntegerStamp(view.stamp(addNode), 0), view); + } + } + + if (forY instanceof AddNode) { + AddNode addNode = (AddNode) forY; + if (addNode.getX() == forX) { + // x == (x + y) => y == 0 + return create(addNode.getY(), ConstantNode.forIntegerStamp(view.stamp(addNode), 0), view); + } else if (addNode.getY() == forX) { + // y == (x + y) => x == 0 + return create(addNode.getX(), ConstantNode.forIntegerStamp(view.stamp(addNode), 0), view); + } + } + + if (forX instanceof SubNode) { + SubNode subNode = (SubNode) forX; + if (subNode.getX() == forY) { + // (x - y) == x => y == 0 + return create(subNode.getY(), ConstantNode.forIntegerStamp(view.stamp(subNode), 0), view); + } + } + + if (forY instanceof SubNode) { + SubNode subNode = (SubNode) forY; + if (forX == subNode.getX()) { + // x == (x - y) => y == 0 + return create(subNode.getY(), ConstantNode.forIntegerStamp(view.stamp(subNode), 0), view); + } + } + return super.canonical(constantReflection, metaAccess, options, smallestCompareWidth, condition, unorderedIsTrue, forX, forY, view); } @@ -306,4 +368,19 @@ public final class IntegerEqualsNode extends CompareNode implements BinaryCommut } return TriState.UNKNOWN; } + + @Override + public TriState implies(boolean thisNegated, LogicNode other) { + // x == y => !(x < y) + // x == y => !(y < x) + if (!thisNegated && other instanceof IntegerLessThanNode) { + ValueNode otherX = ((IntegerLessThanNode) other).getX(); + ValueNode otherY = ((IntegerLessThanNode) other).getY(); + if ((getX() == otherX && getY() == otherY) || (getX() == otherY && getY() == otherX)) { + return TriState.FALSE; + } + } + + return super.implies(thisNegated, other); + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerLessThanNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerLessThanNode.java index 0fe622c44f0..bb154ff0190 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerLessThanNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerLessThanNode.java @@ -50,6 +50,7 @@ import jdk.vm.ci.meta.ConstantReflectionProvider; import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.meta.PrimitiveConstant; +import jdk.vm.ci.meta.TriState; @NodeInfo(shortName = "<") public final class IntegerLessThanNode extends IntegerLowerThanNode { @@ -225,7 +226,6 @@ public final class IntegerLessThanNode extends IntegerLowerThanNode { } } } - } } @@ -296,4 +296,29 @@ public final class IntegerLessThanNode extends IntegerLowerThanNode { return StampFactory.forInteger(bits, cast(min, bits), cast(max, bits)); } } + + @Override + public TriState implies(boolean thisNegated, LogicNode other) { + if (!thisNegated) { + if (other instanceof IntegerLessThanNode) { + ValueNode otherX = ((IntegerLessThanNode) other).getX(); + ValueNode otherY = ((IntegerLessThanNode) other).getY(); + // x < y => !y < x + if (getX() == otherY && getY() == otherX) { + return TriState.FALSE; + } + } + + // x < y => !x == y + // x < y => !y == x + if (other instanceof IntegerEqualsNode) { + ValueNode otherX = ((IntegerEqualsNode) other).getX(); + ValueNode otherY = ((IntegerEqualsNode) other).getY(); + if ((getX() == otherX && getY() == otherY) || (getX() == otherY && getY() == otherX)) { + return TriState.FALSE; + } + } + } + return super.implies(thisNegated, other); + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerMulHighNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerMulHighNode.java similarity index 95% rename from src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerMulHighNode.java rename to src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerMulHighNode.java index b34c4f01320..07d7b820425 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerMulHighNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IntegerMulHighNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -22,7 +22,7 @@ */ -package org.graalvm.compiler.replacements.nodes.arithmetic; +package org.graalvm.compiler.nodes.calc; import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_2; import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_2; @@ -37,7 +37,6 @@ import org.graalvm.compiler.nodeinfo.NodeInfo; import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.calc.BinaryArithmeticNode; import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; import jdk.vm.ci.meta.Constant; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IsNullNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IsNullNode.java index dd284480581..147e55ef4d6 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IsNullNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/IsNullNode.java @@ -35,7 +35,6 @@ import org.graalvm.compiler.nodes.CompressionNode; import org.graalvm.compiler.nodes.LogicConstantNode; import org.graalvm.compiler.nodes.LogicNode; import org.graalvm.compiler.nodes.NodeView; -import org.graalvm.compiler.nodes.PiNode; import org.graalvm.compiler.nodes.UnaryOpLogicNode; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.spi.LIRLowerable; @@ -43,7 +42,6 @@ import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; import org.graalvm.compiler.nodes.spi.Virtualizable; import org.graalvm.compiler.nodes.spi.VirtualizerTool; import org.graalvm.compiler.nodes.type.StampTool; -import org.graalvm.compiler.nodes.util.GraphUtil; import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.TriState; @@ -106,11 +104,6 @@ public final class IsNullNode extends UnaryOpLogicNode implements LIRLowerable, return LogicConstantNode.contradiction(); } - if (value instanceof PiNode) { - value = GraphUtil.skipPi(value); - continue; - } - if (value instanceof ConvertNode) { ConvertNode convertNode = (ConvertNode) value; if (convertNode.mayNullCheckSkipConversion()) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/RightShiftNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/RightShiftNode.java index 9b27d849f27..9b8582220f1 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/RightShiftNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/RightShiftNode.java @@ -83,6 +83,15 @@ public final class RightShiftNode extends ShiftNode { return new UnsignedRightShiftNode(forX, forY); } + Stamp xStampGeneric = forX.stamp(view); + if (xStampGeneric instanceof IntegerStamp) { + IntegerStamp xStamp = (IntegerStamp) xStampGeneric; + if (xStamp.lowerBound() >= -1 && xStamp.upperBound() <= 0) { + // Right shift by any amount does not change any bit. + return forX; + } + } + if (forY.isConstant()) { int amount = forY.asJavaConstant().asInt(); int originalAmout = amount; @@ -91,6 +100,16 @@ public final class RightShiftNode extends ShiftNode { if (amount == 0) { return forX; } + + if (xStampGeneric instanceof IntegerStamp) { + IntegerStamp xStamp = (IntegerStamp) xStampGeneric; + + if (xStamp.lowerBound() >> amount == xStamp.upperBound() >> amount) { + // Right shift turns the result of the expression into a constant. + return ConstantNode.forIntegerKind(stamp.getStackKind(), xStamp.lowerBound() >> amount); + } + } + if (forX instanceof ShiftNode) { ShiftNode other = (ShiftNode) forX; if (other.getY().isConstant()) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/SignedDivNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/SignedDivNode.java index a64bcaf6695..bdfccf83461 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/SignedDivNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/SignedDivNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -44,7 +44,7 @@ public class SignedDivNode extends IntegerDivRemNode implements LIRLowerable { public static final NodeClass TYPE = NodeClass.create(SignedDivNode.class); - protected SignedDivNode(ValueNode x, ValueNode y, GuardingNode zeroCheck) { + public SignedDivNode(ValueNode x, ValueNode y, GuardingNode zeroCheck) { this(TYPE, x, y, zeroCheck); } @@ -118,8 +118,8 @@ public class SignedDivNode extends IntegerDivRemNode implements LIRLowerable { } long abs = Math.abs(c); if (CodeUtil.isPowerOf2(abs) && forX.stamp(view) instanceof IntegerStamp) { - ValueNode dividend = forX; IntegerStamp stampX = (IntegerStamp) forX.stamp(view); + ValueNode dividend = forX; int log2 = CodeUtil.log2(abs); // no rounding if dividend is positive or if its low bits are always 0 if (stampX.canBeNegative() || (stampX.upMask() & (abs - 1)) != 0) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/SignedRemNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/SignedRemNode.java index 19ca38c507a..0a2bfc1bd4a 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/SignedRemNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/SignedRemNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,6 +26,7 @@ package org.graalvm.compiler.nodes.calc; import org.graalvm.compiler.core.common.type.IntegerStamp; import org.graalvm.compiler.core.common.type.Stamp; +import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeClass; import org.graalvm.compiler.graph.spi.CanonicalizerTool; import org.graalvm.compiler.nodeinfo.NodeInfo; @@ -37,13 +38,15 @@ import org.graalvm.compiler.nodes.spi.LIRLowerable; import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; import jdk.vm.ci.code.CodeUtil; +import jdk.vm.ci.meta.Constant; +import jdk.vm.ci.meta.PrimitiveConstant; @NodeInfo(shortName = "%") public class SignedRemNode extends IntegerDivRemNode implements LIRLowerable { public static final NodeClass TYPE = NodeClass.create(SignedRemNode.class); - protected SignedRemNode(ValueNode x, ValueNode y, GuardingNode zeroCheck) { + public SignedRemNode(ValueNode x, ValueNode y, GuardingNode zeroCheck) { this(TYPE, x, y, zeroCheck); } @@ -53,7 +56,7 @@ public class SignedRemNode extends IntegerDivRemNode implements LIRLowerable { public static ValueNode create(ValueNode x, ValueNode y, GuardingNode zeroCheck, NodeView view) { Stamp stamp = IntegerStamp.OPS.getRem().foldStamp(x.stamp(view), y.stamp(view)); - return canonical(null, x, y, zeroCheck, stamp, view); + return canonical(null, x, y, zeroCheck, stamp, view, null); } @Override @@ -64,10 +67,10 @@ public class SignedRemNode extends IntegerDivRemNode implements LIRLowerable { @Override public ValueNode canonical(CanonicalizerTool tool, ValueNode forX, ValueNode forY) { NodeView view = NodeView.from(tool); - return canonical(this, forX, forY, getZeroCheck(), stamp(view), view); + return canonical(this, forX, forY, getZeroCheck(), stamp(view), view, tool); } - private static ValueNode canonical(SignedRemNode self, ValueNode forX, ValueNode forY, GuardingNode zeroCheck, Stamp stamp, NodeView view) { + private static ValueNode canonical(SignedRemNode self, ValueNode forX, ValueNode forY, GuardingNode zeroCheck, Stamp stamp, NodeView view, CanonicalizerTool tool) { if (forX.isConstant() && forY.isConstant()) { long y = forY.asJavaConstant().asLong(); if (y == 0) { @@ -81,27 +84,52 @@ public class SignedRemNode extends IntegerDivRemNode implements LIRLowerable { IntegerStamp yStamp = (IntegerStamp) forY.stamp(view); if (constY < 0 && constY != CodeUtil.minValue(yStamp.getBits())) { Stamp newStamp = IntegerStamp.OPS.getRem().foldStamp(forX.stamp(view), forY.stamp(view)); - return canonical(null, forX, ConstantNode.forIntegerStamp(yStamp, -constY), zeroCheck, newStamp, view); + return canonical(null, forX, ConstantNode.forIntegerStamp(yStamp, -constY), zeroCheck, newStamp, view, tool); } if (constY == 1) { return ConstantNode.forIntegerStamp(stamp, 0); - } else if (CodeUtil.isPowerOf2(constY)) { - if (xStamp.isPositive()) { - // x & (y - 1) - return new AndNode(forX, ConstantNode.forIntegerStamp(stamp, constY - 1)); - } else if (xStamp.isNegative()) { - // -((-x) & (y - 1)) - return new NegateNode(new AndNode(new NegateNode(forX), ConstantNode.forIntegerStamp(stamp, constY - 1))); + } else if (CodeUtil.isPowerOf2(constY) && tool != null && tool.allUsagesAvailable()) { + if (allUsagesCompareAgainstZero(self)) { + // x % y == 0 <=> (x & (y-1)) == 0 + return new AndNode(forX, ConstantNode.forIntegerStamp(yStamp, constY - 1)); } else { - // x - ((x / y) << log2(y)) - return SubNode.create(forX, LeftShiftNode.create(SignedDivNode.canonical(forX, constY, view), ConstantNode.forInt(CodeUtil.log2(constY)), view), view); + if (xStamp.isPositive()) { + // x & (y - 1) + return new AndNode(forX, ConstantNode.forIntegerStamp(stamp, constY - 1)); + } else if (xStamp.isNegative()) { + // -((-x) & (y - 1)) + return new NegateNode(new AndNode(new NegateNode(forX), ConstantNode.forIntegerStamp(stamp, constY - 1))); + } } } } return self != null ? self : new SignedRemNode(forX, forY, zeroCheck); } + private static boolean allUsagesCompareAgainstZero(SignedRemNode self) { + int compareAgainstZero = 0; + int usageCount = self.getUsageCount(); + for (int i = 0; i < usageCount; i++) { + Node usage = self.getUsageAt(i); + if (usage instanceof IntegerEqualsNode) { + IntegerEqualsNode equalsNode = (IntegerEqualsNode) usage; + ValueNode node = equalsNode.getY(); + if (node == self) { + node = equalsNode.getX(); + } + if (node instanceof ConstantNode) { + ConstantNode constantNode = (ConstantNode) node; + Constant constant = constantNode.asConstant(); + if (constant instanceof PrimitiveConstant && ((PrimitiveConstant) constant).asLong() == 0) { + compareAgainstZero++; + } + } + } + } + return compareAgainstZero == usageCount; + } + @Override public void generate(NodeLIRBuilderTool gen) { gen.setResult(this, gen.getLIRGeneratorTool().getArithmetic().emitRem(gen.operand(getX()), gen.operand(getY()), gen.state(this))); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/UnsignedRightShiftNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/UnsignedRightShiftNode.java index c28a04e5384..5f0d572daf7 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/UnsignedRightShiftNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/calc/UnsignedRightShiftNode.java @@ -80,6 +80,22 @@ public final class UnsignedRightShiftNode extends ShiftNode { if (amount == 0) { return forX; } + + Stamp xStampGeneric = forX.stamp(view); + if (xStampGeneric instanceof IntegerStamp) { + IntegerStamp xStamp = (IntegerStamp) xStampGeneric; + + if (xStamp.lowerBound() >>> amount == xStamp.upperBound() >>> amount) { + // The result of the shift is constant. + return ConstantNode.forIntegerKind(stamp.getStackKind(), xStamp.lowerBound() >>> amount); + } + + if (amount == xStamp.getBits() - 1 && xStamp.lowerBound() == -1 && xStamp.upperBound() == 0) { + // Shift is equivalent to a negation, i.e., turns -1 into 1 and keeps 0 at 0. + return NegateNode.create(forX, view); + } + } + if (forX instanceof ShiftNode) { ShiftNode other = (ShiftNode) forX; if (other.getY().isConstant()) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/cfg/Block.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/cfg/Block.java index a09de19dfab..a81780be5ad 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/cfg/Block.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/cfg/Block.java @@ -33,13 +33,11 @@ import org.graalvm.compiler.core.common.cfg.Loop; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.nodeinfo.Verbosity; import org.graalvm.compiler.nodes.AbstractBeginNode; -import org.graalvm.compiler.nodes.BeginNode; import org.graalvm.compiler.nodes.FixedNode; import org.graalvm.compiler.nodes.FixedWithNextNode; import org.graalvm.compiler.nodes.InvokeWithExceptionNode; import org.graalvm.compiler.nodes.LoopBeginNode; import org.graalvm.compiler.nodes.LoopEndNode; -import org.graalvm.compiler.nodes.LoopExitNode; import org.graalvm.compiler.nodes.memory.MemoryCheckpoint; import jdk.internal.vm.compiler.word.LocationIdentity; @@ -69,21 +67,6 @@ public final class Block extends AbstractBlockBase { return endNode; } - /** - * Return the {@link LoopExitNode} for this block if it exists. - */ - public LoopExitNode getLoopExit() { - if (beginNode instanceof BeginNode) { - if (beginNode.next() instanceof LoopExitNode) { - return (LoopExitNode) beginNode.next(); - } - } - if (beginNode instanceof LoopExitNode) { - return (LoopExitNode) beginNode; - } - return null; - } - @Override public Loop getLoop() { return loop; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/cfg/ControlFlowGraph.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/cfg/ControlFlowGraph.java index d1431467eb9..2c4d8c6d63e 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/cfg/ControlFlowGraph.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/cfg/ControlFlowGraph.java @@ -24,6 +24,8 @@ package org.graalvm.compiler.nodes.cfg; +import static org.graalvm.compiler.core.common.cfg.AbstractBlockBase.BLOCK_ID_COMPARATOR; + import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; @@ -49,7 +51,6 @@ import org.graalvm.compiler.nodes.LoopEndNode; import org.graalvm.compiler.nodes.LoopExitNode; import org.graalvm.compiler.nodes.MergeNode; import org.graalvm.compiler.nodes.StructuredGraph; -import org.graalvm.compiler.nodes.StructuredGraph.GuardsStage; public final class ControlFlowGraph implements AbstractControlFlowGraph { /** @@ -614,13 +615,26 @@ public final class ControlFlowGraph implements AbstractControlFlowGraph { computeLoopBlocks(endBlock, loop, stack, true); } - if (graph.getGuardsStage() != GuardsStage.AFTER_FSA) { + // Note that at this point, due to traversal order, child loops of `loop` have + // not been discovered yet. + for (Block b : loop.getBlocks()) { + for (Block sux : b.getSuccessors()) { + if (sux.getLoop() != loop) { + assert sux.getLoopDepth() < loop.getDepth(); + loop.getNaturalExits().add(sux); + } + } + } + loop.getNaturalExits().sort(BLOCK_ID_COMPARATOR); + + if (!graph.getGuardsStage().areFrameStatesAtDeopts()) { for (LoopExitNode exit : loopBegin.loopExits()) { Block exitBlock = nodeToBlock.get(exit); assert exitBlock.getPredecessorCount() == 1; computeLoopBlocks(exitBlock.getFirstPredecessor(), loop, stack, true); - loop.addExit(exitBlock); + loop.getLoopExits().add(exitBlock); } + loop.getLoopExits().sort(BLOCK_ID_COMPARATOR); // The following loop can add new blocks to the end of the loop's block // list. @@ -630,65 +644,21 @@ public final class ControlFlowGraph implements AbstractControlFlowGraph { for (Block sux : b.getSuccessors()) { if (sux.getLoop() != loop) { AbstractBeginNode begin = sux.getBeginNode(); - if (!(begin instanceof LoopExitNode && ((LoopExitNode) begin).loopBegin() == loopBegin)) { + if (!loopBegin.isLoopExit(begin)) { + assert !(begin instanceof LoopBeginNode); + assert sux.getLoopDepth() < loop.getDepth(); graph.getDebug().log(DebugContext.VERBOSE_LEVEL, "Unexpected loop exit with %s, including whole branch in the loop", sux); computeLoopBlocks(sux, loop, stack, false); } } } } + } else { + loop.getLoopExits().addAll(loop.getNaturalExits()); } } } } - - /* - * Compute the loop exit blocks after FSA. - */ - if (graph.getGuardsStage() == GuardsStage.AFTER_FSA) { - for (Block b : reversePostOrder) { - if (b.getLoop() != null) { - for (Block succ : b.getSuccessors()) { - // if the loop of the succ is a different one (or none) - if (b.getLoop() != succ.getLoop()) { - // and the succ loop is not a child loop of the curr one - if (succ.getLoop() == null) { - // we might exit multiple loops if b.loops is not a loop at depth 0 - Loop curr = b.getLoop(); - while (curr != null) { - curr.addExit(succ); - curr = curr.getParent(); - } - } else { - /* - * succ also has a loop, might be a child loop - * - * if it is a child loop we do not exit a loop. if it is a loop - * different than b.loop and not a child loop it must be a parent - * loop, thus we exit all loops between b.loop and succ.loop - * - * if we exit multiple loops immediately after each other the - * bytecode parser might generate loop exit nodes after another and - * the CFG will identify them as separate blocks, we just take the - * first one and exit all loops at this one - */ - if (succ.getLoop().getParent() != b.getLoop()) { - assert succ.getLoop().getDepth() < b.getLoop().getDepth(); - // b.loop must not be a transitive parent of succ.loop - assert !Loop.transitiveParentLoop(succ.getLoop(), b.getLoop()); - Loop curr = b.getLoop(); - while (curr != null && curr != succ.getLoop()) { - curr.addExit(succ); - curr = curr.getParent(); - } - } - } - } - } - } - } - } - } private static void computeLoopBlocks(Block start, Loop loop, Block[] stack, boolean usePred) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/JavaReadNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/JavaReadNode.java index a5a1403dc0b..d04007f93d1 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/JavaReadNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/JavaReadNode.java @@ -27,6 +27,7 @@ package org.graalvm.compiler.nodes.extended; import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_2; import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_1; +import org.graalvm.compiler.core.common.type.Stamp; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeClass; @@ -54,7 +55,11 @@ public final class JavaReadNode extends FixedAccessNode implements Lowerable, Gu protected final boolean compressible; public JavaReadNode(JavaKind readKind, AddressNode address, LocationIdentity location, BarrierType barrierType, boolean compressible) { - super(TYPE, address, location, StampFactory.forKind(readKind), barrierType); + this(StampFactory.forKind(readKind), readKind, address, location, barrierType, compressible); + } + + public JavaReadNode(Stamp stamp, JavaKind readKind, AddressNode address, LocationIdentity location, BarrierType barrierType, boolean compressible) { + super(TYPE, address, location, stamp, barrierType); this.readKind = readKind; this.compressible = compressible; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GeneratedInvocationPlugin.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GeneratedInvocationPlugin.java index 548c679a9c0..3c585fc184d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GeneratedInvocationPlugin.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GeneratedInvocationPlugin.java @@ -24,6 +24,9 @@ package org.graalvm.compiler.nodes.graphbuilderconf; +import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; + import java.lang.annotation.Annotation; import java.lang.reflect.Method; @@ -66,6 +69,11 @@ public abstract class GeneratedInvocationPlugin implements InvocationPlugin { return true; } + if (IS_IN_NATIVE_IMAGE || IS_BUILDING_NATIVE_IMAGE) { + // The reflection here is problematic for SVM. + return true; + } + MetaAccessProvider metaAccess = b.getMetaAccess(); ResolvedJavaMethod executeMethod = metaAccess.lookupJavaMethod(getExecuteMethod()); ResolvedJavaType thisClass = metaAccess.lookupJavaType(getClass()); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GraphBuilderConfiguration.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GraphBuilderConfiguration.java index fa5044ecf38..e135390fe53 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GraphBuilderConfiguration.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GraphBuilderConfiguration.java @@ -25,13 +25,15 @@ package org.graalvm.compiler.nodes.graphbuilderconf; import java.util.Arrays; +import java.util.Collections; +import java.util.List; import org.graalvm.compiler.core.common.type.StampPair; import jdk.vm.ci.meta.JavaType; import jdk.vm.ci.meta.ResolvedJavaType; -public class GraphBuilderConfiguration { +public final class GraphBuilderConfiguration { public static class Plugins { private final InvocationPlugins invocationPlugins; @@ -39,7 +41,6 @@ public class GraphBuilderConfiguration { private ParameterPlugin[] parameterPlugins; private TypePlugin[] typePlugins; private InlineInvokePlugin[] inlineInvokePlugins; - private LoopExplosionPlugin loopExplosionPlugin; private ClassInitializationPlugin classInitializationPlugin; private InvokeDynamicPlugin invokeDynamicPlugin; private ProfilingPlugin profilingPlugin; @@ -49,18 +50,21 @@ public class GraphBuilderConfiguration { * {@code copyFrom} become the {@linkplain InvocationPlugins#getParent() default} * {@linkplain #getInvocationPlugins() invocation plugins} in this object. */ - public Plugins(Plugins copyFrom) { - this.invocationPlugins = new InvocationPlugins(copyFrom.invocationPlugins); + public Plugins(Plugins copyFrom, InvocationPlugins invocationPlugins) { + this.invocationPlugins = invocationPlugins != null ? invocationPlugins : new InvocationPlugins(copyFrom.invocationPlugins); this.nodePlugins = copyFrom.nodePlugins; this.parameterPlugins = copyFrom.parameterPlugins; this.typePlugins = copyFrom.typePlugins; this.inlineInvokePlugins = copyFrom.inlineInvokePlugins; - this.loopExplosionPlugin = copyFrom.loopExplosionPlugin; this.classInitializationPlugin = copyFrom.classInitializationPlugin; this.invokeDynamicPlugin = copyFrom.invokeDynamicPlugin; this.profilingPlugin = copyFrom.profilingPlugin; } + public Plugins(Plugins copyFrom) { + this(copyFrom, null); + } + /** * Creates a new set of plugins. * @@ -155,14 +159,6 @@ public class GraphBuilderConfiguration { inlineInvokePlugins = new InlineInvokePlugin[0]; } - public LoopExplosionPlugin getLoopExplosionPlugin() { - return loopExplosionPlugin; - } - - public void setLoopExplosionPlugin(LoopExplosionPlugin plugin) { - this.loopExplosionPlugin = plugin; - } - public ClassInitializationPlugin getClassInitializationPlugin() { return classInitializationPlugin; } @@ -198,13 +194,11 @@ public class GraphBuilderConfiguration { } } - private static final ResolvedJavaType[] EMPTY = new ResolvedJavaType[]{}; - private final boolean eagerResolving; private final boolean unresolvedIsError; private final BytecodeExceptionMode bytecodeExceptionMode; private final boolean omitAssertions; - private final ResolvedJavaType[] skippedExceptionTypes; + private final List skippedExceptionTypes; private final boolean insertFullInfopoints; private final boolean trackNodeSourcePosition; private final Plugins plugins; @@ -229,8 +223,14 @@ public class GraphBuilderConfiguration { Profile } - protected GraphBuilderConfiguration(boolean eagerResolving, boolean unresolvedIsError, BytecodeExceptionMode bytecodeExceptionMode, boolean omitAssertions, boolean insertFullInfopoints, - boolean trackNodeSourcePosition, ResolvedJavaType[] skippedExceptionTypes, Plugins plugins) { + private GraphBuilderConfiguration(boolean eagerResolving, + boolean unresolvedIsError, + BytecodeExceptionMode bytecodeExceptionMode, + boolean omitAssertions, + boolean insertFullInfopoints, + boolean trackNodeSourcePosition, + List skippedExceptionTypes, + Plugins plugins) { this.eagerResolving = eagerResolving; this.unresolvedIsError = unresolvedIsError; this.bytecodeExceptionMode = bytecodeExceptionMode; @@ -248,8 +248,15 @@ public class GraphBuilderConfiguration { */ public GraphBuilderConfiguration copy() { Plugins newPlugins = new Plugins(plugins); - GraphBuilderConfiguration result = new GraphBuilderConfiguration(eagerResolving, unresolvedIsError, bytecodeExceptionMode, omitAssertions, insertFullInfopoints, trackNodeSourcePosition, - skippedExceptionTypes, newPlugins); + GraphBuilderConfiguration result = new GraphBuilderConfiguration( + eagerResolving, + unresolvedIsError, + bytecodeExceptionMode, + omitAssertions, + insertFullInfopoints, + trackNodeSourcePosition, + skippedExceptionTypes, + newPlugins); return result; } @@ -260,43 +267,89 @@ public class GraphBuilderConfiguration { * eagerly resolving elements. */ public GraphBuilderConfiguration withUnresolvedIsError(boolean newUnresolvedIsError) { - return new GraphBuilderConfiguration(eagerResolving, newUnresolvedIsError, bytecodeExceptionMode, omitAssertions, insertFullInfopoints, trackNodeSourcePosition, skippedExceptionTypes, + return new GraphBuilderConfiguration( + eagerResolving, + newUnresolvedIsError, + bytecodeExceptionMode, + omitAssertions, + insertFullInfopoints, + trackNodeSourcePosition, + skippedExceptionTypes, plugins); } public GraphBuilderConfiguration withEagerResolving(boolean newEagerResolving) { - return new GraphBuilderConfiguration(newEagerResolving, unresolvedIsError, bytecodeExceptionMode, omitAssertions, insertFullInfopoints, trackNodeSourcePosition, skippedExceptionTypes, + return new GraphBuilderConfiguration( + newEagerResolving, + unresolvedIsError, + bytecodeExceptionMode, + omitAssertions, + insertFullInfopoints, + trackNodeSourcePosition, + skippedExceptionTypes, plugins); } public GraphBuilderConfiguration withSkippedExceptionTypes(ResolvedJavaType[] newSkippedExceptionTypes) { - return new GraphBuilderConfiguration(eagerResolving, unresolvedIsError, bytecodeExceptionMode, omitAssertions, insertFullInfopoints, trackNodeSourcePosition, newSkippedExceptionTypes, + return new GraphBuilderConfiguration( + eagerResolving, + unresolvedIsError, + bytecodeExceptionMode, + omitAssertions, + insertFullInfopoints, + trackNodeSourcePosition, + Collections.unmodifiableList(Arrays.asList(newSkippedExceptionTypes)), plugins); } public GraphBuilderConfiguration withBytecodeExceptionMode(BytecodeExceptionMode newBytecodeExceptionMode) { - return new GraphBuilderConfiguration(eagerResolving, unresolvedIsError, newBytecodeExceptionMode, omitAssertions, insertFullInfopoints, trackNodeSourcePosition, skippedExceptionTypes, + return new GraphBuilderConfiguration(eagerResolving, + unresolvedIsError, + newBytecodeExceptionMode, + omitAssertions, + insertFullInfopoints, + trackNodeSourcePosition, + skippedExceptionTypes, plugins); } public GraphBuilderConfiguration withOmitAssertions(boolean newOmitAssertions) { - return new GraphBuilderConfiguration(eagerResolving, unresolvedIsError, bytecodeExceptionMode, newOmitAssertions, insertFullInfopoints, trackNodeSourcePosition, skippedExceptionTypes, + return new GraphBuilderConfiguration( + eagerResolving, + unresolvedIsError, + bytecodeExceptionMode, + newOmitAssertions, + insertFullInfopoints, + trackNodeSourcePosition, + skippedExceptionTypes, plugins); } public GraphBuilderConfiguration withFullInfopoints(boolean newInsertFullInfopoints) { - ResolvedJavaType[] newSkippedExceptionTypes = skippedExceptionTypes == EMPTY ? EMPTY : Arrays.copyOf(skippedExceptionTypes, skippedExceptionTypes.length); - return new GraphBuilderConfiguration(eagerResolving, unresolvedIsError, bytecodeExceptionMode, omitAssertions, newInsertFullInfopoints, trackNodeSourcePosition, newSkippedExceptionTypes, + return new GraphBuilderConfiguration( + eagerResolving, + unresolvedIsError, + bytecodeExceptionMode, + omitAssertions, + newInsertFullInfopoints, + trackNodeSourcePosition, + skippedExceptionTypes, plugins); } public GraphBuilderConfiguration withNodeSourcePosition(boolean newTrackNodeSourcePosition) { - ResolvedJavaType[] newSkippedExceptionTypes = skippedExceptionTypes == EMPTY ? EMPTY : Arrays.copyOf(skippedExceptionTypes, skippedExceptionTypes.length); - return new GraphBuilderConfiguration(eagerResolving, unresolvedIsError, bytecodeExceptionMode, omitAssertions, insertFullInfopoints, newTrackNodeSourcePosition, newSkippedExceptionTypes, + return new GraphBuilderConfiguration( + eagerResolving, + unresolvedIsError, + bytecodeExceptionMode, + omitAssertions, + insertFullInfopoints, + newTrackNodeSourcePosition, + skippedExceptionTypes, plugins); } - public ResolvedJavaType[] getSkippedExceptionTypes() { + public List getSkippedExceptionTypes() { return skippedExceptionTypes; } @@ -321,11 +374,27 @@ public class GraphBuilderConfiguration { } public static GraphBuilderConfiguration getDefault(Plugins plugins) { - return new GraphBuilderConfiguration(false, false, BytecodeExceptionMode.Profile, false, false, false, EMPTY, plugins); + return new GraphBuilderConfiguration( + /* eagerResolving: */ false, + /* unresolvedIsError: */ false, + BytecodeExceptionMode.Profile, + /* omitAssertions: */ false, + /* insertFullInfopoints: */ false, + /* trackNodeSourcePosition: */ false, + Collections.emptyList(), + plugins); } public static GraphBuilderConfiguration getSnippetDefault(Plugins plugins) { - return new GraphBuilderConfiguration(true, true, BytecodeExceptionMode.OmitAll, false, false, false, EMPTY, plugins); + return new GraphBuilderConfiguration( + /* eagerResolving: */ true, + /* unresolvedIsError: */ true, + BytecodeExceptionMode.OmitAll, + /* omitAssertions: */ false, + /* insertFullInfopoints: */ false, + /* trackNodeSourcePosition: */ false, + Collections.emptyList(), + plugins); } /** Returns {@code true} if it is an error for a class/field/method resolution to fail. */ diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GraphBuilderContext.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GraphBuilderContext.java index 1cfe75f7180..71efbdc143b 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GraphBuilderContext.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/GraphBuilderContext.java @@ -31,7 +31,6 @@ import static org.graalvm.compiler.core.common.type.StampFactory.objectNonNull; import org.graalvm.compiler.bytecode.Bytecode; import org.graalvm.compiler.bytecode.BytecodeProvider; import org.graalvm.compiler.core.common.type.AbstractPointerStamp; -import org.graalvm.compiler.core.common.type.ObjectStamp; import org.graalvm.compiler.core.common.type.Stamp; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.core.common.type.StampPair; @@ -43,6 +42,7 @@ import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind; import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.DynamicPiNode; import org.graalvm.compiler.nodes.FixedGuardNode; +import org.graalvm.compiler.nodes.Invoke; import org.graalvm.compiler.nodes.LogicNode; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.PiNode; @@ -92,9 +92,9 @@ public interface GraphBuilderContext extends GraphBuilderTool { } /** - * Adds a node to the graph. If the node is in the graph, returns immediately. If the node is a - * {@link StateSplit} with a null {@linkplain StateSplit#stateAfter() frame state}, the frame - * state is initialized. + * Adds a node and all its inputs to the graph. If the node is in the graph, returns + * immediately. If the node is a {@link StateSplit} with a null + * {@linkplain StateSplit#stateAfter() frame state} , the frame state is initialized. * * @param value the value to add to the graph and push to the stack. The * {@code value.getJavaKind()} kind is used when type checking this operation. @@ -108,23 +108,6 @@ public interface GraphBuilderContext extends GraphBuilderTool { return GraphBuilderContextUtil.setStateAfterIfNecessary(this, append(value)); } - /** - * Adds a node and its inputs to the graph. If the node is in the graph, returns immediately. If - * the node is a {@link StateSplit} with a null {@linkplain StateSplit#stateAfter() frame state} - * , the frame state is initialized. - * - * @param value the value to add to the graph and push to the stack. The - * {@code value.getJavaKind()} kind is used when type checking this operation. - * @return a node equivalent to {@code value} in the graph - */ - default T addWithInputs(T value) { - if (value.graph() != null) { - assert !(value instanceof StateSplit) || ((StateSplit) value).stateAfter() != null; - return value; - } - return GraphBuilderContextUtil.setStateAfterIfNecessary(this, append(value)); - } - default ValueNode addNonNullCast(ValueNode value) { AbstractPointerStamp valueStamp = (AbstractPointerStamp) value.stamp(NodeView.DEFAULT); if (valueStamp.nonNull()) { @@ -163,7 +146,7 @@ public interface GraphBuilderContext extends GraphBuilderTool { * @param forceInlineEverything specifies if all invocations encountered in the scope of * handling the replaced invoke are to be force inlined */ - void handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean forceInlineEverything); + Invoke handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean forceInlineEverything); void handleReplacedInvoke(CallTargetNode callTarget, JavaKind resultType); @@ -279,10 +262,8 @@ public interface GraphBuilderContext extends GraphBuilderTool { default ValueNode nullCheckedValue(ValueNode value, DeoptimizationAction action) { if (!StampTool.isPointerNonNull(value)) { LogicNode condition = getGraph().unique(IsNullNode.create(value)); - ObjectStamp receiverStamp = (ObjectStamp) value.stamp(NodeView.DEFAULT); - Stamp stamp = receiverStamp.join(objectNonNull()); FixedGuardNode fixedGuard = append(new FixedGuardNode(condition, NullCheckException, action, true)); - ValueNode nonNullReceiver = getGraph().addOrUniqueWithInputs(PiNode.create(value, stamp, fixedGuard)); + ValueNode nonNullReceiver = getGraph().addOrUniqueWithInputs(PiNode.create(value, objectNonNull(), fixedGuard)); // TODO: Propogating the non-null into the frame state would // remove subsequent null-checks on the same value. However, // it currently causes an assertion failure when merging states. diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/InvocationPlugins.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/InvocationPlugins.java index 069f170026c..0645cc89646 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/InvocationPlugins.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/InvocationPlugins.java @@ -25,6 +25,8 @@ package org.graalvm.compiler.nodes.graphbuilderconf; import static java.lang.String.format; +import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; import static org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins.LateClassPlugins.CLOSED_LATE_CLASS_PLUGIN; import java.lang.reflect.Constructor; @@ -144,6 +146,9 @@ public class InvocationPlugins { OptionalLazySymbol(String name) { this.name = name; + if (IS_BUILDING_NATIVE_IMAGE) { + resolve(); + } } @Override @@ -156,7 +161,7 @@ public class InvocationPlugins { * resolution fails. */ public Class resolve() { - if (resolved == null) { + if (!IS_IN_NATIVE_IMAGE && resolved == null) { Class resolvedOrNull = resolveClass(name, true); resolved = resolvedOrNull == null ? MASK_NULL : resolvedOrNull; } @@ -215,6 +220,20 @@ public class InvocationPlugins { this.methodSubstitutionBytecodeProvider = methodSubstitutionBytecodeProvider; } + /** + * Creates an object for registering {@link InvocationPlugin}s for methods declared by a + * given class. + * + * @param plugins where to register the plugins + * @param declaringClassName the name of the class class declaring the methods for which + * plugins will be registered via this object + */ + public Registration(InvocationPlugins plugins, String declaringClassName) { + this.plugins = plugins; + this.declaringType = new OptionalLazySymbol(declaringClassName); + this.methodSubstitutionBytecodeProvider = null; + } + /** * Creates an object for registering {@link InvocationPlugin}s for methods declared by a * given class. @@ -452,8 +471,8 @@ public class InvocationPlugins { Binding binding = new Binding(plugin, isStatic, name, argumentTypes); bindings.add(binding); - assert Checks.check(this.plugins, declaringType, binding); - assert Checks.checkResolvable(false, declaringType, binding); + assert IS_IN_NATIVE_IMAGE || Checks.check(this.plugins, declaringType, binding); + assert IS_IN_NATIVE_IMAGE || Checks.checkResolvable(false, declaringType, binding); } @Override @@ -736,8 +755,8 @@ public class InvocationPlugins { } @SuppressWarnings("serial") - static class InvocationPlugRegistrationError extends GraalError { - InvocationPlugRegistrationError(Throwable cause) { + static class InvocationPluginRegistrationError extends GraalError { + InvocationPluginRegistrationError(Throwable cause) { super(cause); } } @@ -751,7 +770,7 @@ public class InvocationPlugins { deferrable.run(); } deferredRegistrations = null; - } catch (InvocationPlugRegistrationError t) { + } catch (InvocationPluginRegistrationError t) { throw t; } catch (Throwable t) { /* @@ -765,7 +784,7 @@ public class InvocationPlugins { Runnable rethrow = new Runnable() { @Override public void run() { - throw new InvocationPlugRegistrationError(t); + throw new InvocationPluginRegistrationError(t); } }; deferredRegistrations.add(rethrow); @@ -960,8 +979,8 @@ public class InvocationPlugins { argumentTypes[0] = declaringClass; } Binding binding = put(plugin, isStatic, allowOverwrite, declaringClass, name, argumentTypes); - assert Checks.check(this, declaringClass, binding); - assert Checks.checkResolvable(isOptional, declaringClass, binding); + assert IS_IN_NATIVE_IMAGE || Checks.check(this, declaringClass, binding); + assert IS_IN_NATIVE_IMAGE || Checks.checkResolvable(isOptional, declaringClass, binding); } /** @@ -1004,10 +1023,19 @@ public class InvocationPlugins { if (parent != null) { InvocationPlugin plugin = parent.lookupInvocation(method); if (plugin != null) { + if (IS_IN_NATIVE_IMAGE && plugin instanceof MethodSubstitutionPlugin) { + // Disable method substitutions until GR-13607 + return null; + } return plugin; } } - return get(method); + InvocationPlugin invocationPlugin = get(method); + if (IS_IN_NATIVE_IMAGE && invocationPlugin instanceof MethodSubstitutionPlugin) { + // Disable method substitutions until GR-13607 + return null; + } + return invocationPlugin; } /** @@ -1141,25 +1169,27 @@ public class InvocationPlugins { static final Class[][] SIGS; static { - if (!Assertions.assertionsEnabled()) { + if (!Assertions.assertionsEnabled() && !IS_BUILDING_NATIVE_IMAGE) { throw new GraalError("%s must only be used in assertions", Checks.class.getName()); } ArrayList[]> sigs = new ArrayList<>(MAX_ARITY); - for (Method method : InvocationPlugin.class.getDeclaredMethods()) { - if (!Modifier.isStatic(method.getModifiers()) && method.getName().equals("apply")) { - Class[] sig = method.getParameterTypes(); - assert sig[0] == GraphBuilderContext.class; - assert sig[1] == ResolvedJavaMethod.class; - assert sig[2] == InvocationPlugin.Receiver.class; - assert Arrays.asList(sig).subList(3, sig.length).stream().allMatch(c -> c == ValueNode.class); - while (sigs.size() < sig.length - 2) { - sigs.add(null); + if (!IS_IN_NATIVE_IMAGE) { + for (Method method : InvocationPlugin.class.getDeclaredMethods()) { + if (!Modifier.isStatic(method.getModifiers()) && method.getName().equals("apply")) { + Class[] sig = method.getParameterTypes(); + assert sig[0] == GraphBuilderContext.class; + assert sig[1] == ResolvedJavaMethod.class; + assert sig[2] == InvocationPlugin.Receiver.class; + assert Arrays.asList(sig).subList(3, sig.length).stream().allMatch(c -> c == ValueNode.class); + while (sigs.size() < sig.length - 2) { + sigs.add(null); + } + sigs.set(sig.length - 3, sig); } - sigs.set(sig.length - 3, sig); } + assert sigs.indexOf(null) == -1 : format("need to add an apply() method to %s that takes %d %s arguments ", InvocationPlugin.class.getName(), sigs.indexOf(null), + ValueNode.class.getSimpleName()); } - assert sigs.indexOf(null) == -1 : format("need to add an apply() method to %s that takes %d %s arguments ", InvocationPlugin.class.getName(), sigs.indexOf(null), - ValueNode.class.getSimpleName()); SIGS = sigs.toArray(new Class[sigs.size()][]); } @@ -1276,6 +1306,9 @@ public class InvocationPlugins { if (type instanceof OptionalLazySymbol) { return ((OptionalLazySymbol) type).resolve(); } + if (IS_IN_NATIVE_IMAGE) { + throw new GraalError("Unresolved type in native image image:" + type.getTypeName()); + } return resolveClass(type.getTypeName(), optional); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/MethodSubstitutionPlugin.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/MethodSubstitutionPlugin.java index a0ca9598eb2..4fc559987e7 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/MethodSubstitutionPlugin.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/MethodSubstitutionPlugin.java @@ -24,6 +24,7 @@ package org.graalvm.compiler.nodes.graphbuilderconf; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; import static org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins.resolveType; import java.lang.reflect.Method; @@ -177,6 +178,10 @@ public final class MethodSubstitutionPlugin implements InvocationPlugin { @Override public boolean execute(GraphBuilderContext b, ResolvedJavaMethod targetMethod, InvocationPlugin.Receiver receiver, ValueNode[] argsIncludingReceiver) { + if (IS_IN_NATIVE_IMAGE) { + // these are currently unimplemented + return false; + } ResolvedJavaMethod subst = getSubstitute(b.getMetaAccess()); return b.intrinsify(bytecodeProvider, targetMethod, subst, receiver, argsIncludingReceiver); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/NodePlugin.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/NodePlugin.java index 31d7616a7fc..46f532d9d2e 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/NodePlugin.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/graphbuilderconf/NodePlugin.java @@ -221,6 +221,9 @@ public interface NodePlugin extends GraphBuilderPlugin { * nodes) and fixed nodes must be manually {@linkplain FixedWithNextNode#setNext added} as * successors of {@code afterExceptionLoaded}. * + * The reason for this constraint is that when this plugin runs, it's inserting instructions + * into a different block than the one currently being parsed. + * * @param graph the graph being parsed * @param afterExceptionLoaded the last fixed node after loading the exception * @return the last fixed node after instrumentation diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/NewArrayNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/NewArrayNode.java index 736f7dc297b..f285d533767 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/NewArrayNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/NewArrayNode.java @@ -98,7 +98,7 @@ public class NewArrayNode extends AbstractNewArrayNode implements VirtualizableA ValueNode lengthAlias = tool.getAlias(length()); if (lengthAlias.asConstant() != null) { int constantLength = lengthAlias.asJavaConstant().asInt(); - if (constantLength >= 0 && constantLength < tool.getMaximumEntryCount()) { + if (constantLength >= 0 && constantLength <= tool.getMaximumEntryCount()) { ValueNode[] state = new ValueNode[constantLength]; ConstantNode defaultForKind = constantLength == 0 ? null : defaultElementValue(); for (int i = 0; i < constantLength; i++) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/memory/address/IndexAddressNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/memory/address/IndexAddressNode.java new file mode 100644 index 00000000000..f546aac3f5f --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/memory/address/IndexAddressNode.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.nodes.memory.address; + +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.nodeinfo.InputType; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.spi.Lowerable; +import org.graalvm.compiler.nodes.spi.LoweringTool; + +import jdk.vm.ci.meta.JavaKind; + +/** + * Represents an address that points to an element of a Java array. + */ +@NodeInfo(allowedUsageTypes = InputType.Association) +public class IndexAddressNode extends AddressNode implements Lowerable { + public static final NodeClass TYPE = NodeClass.create(IndexAddressNode.class); + + @Input ValueNode array; + @Input ValueNode index; + + private final JavaKind elementKind; + + public IndexAddressNode(ValueNode array, ValueNode index, JavaKind elementKind) { + super(TYPE); + this.array = array; + this.index = index; + this.elementKind = elementKind; + } + + @Override + public ValueNode getBase() { + return array; + } + + public ValueNode getArray() { + return array; + } + + @Override + public ValueNode getIndex() { + return index; + } + + @Override + public long getMaxConstantDisplacement() { + return Long.MAX_VALUE; + } + + public JavaKind getElementKind() { + return elementKind; + } + + @Override + public void lower(LoweringTool tool) { + tool.getLowerer().lower(this, tool); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/NodeLIRBuilderTool.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/NodeLIRBuilderTool.java index 3f3b6693cad..923255f2566 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/NodeLIRBuilderTool.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/NodeLIRBuilderTool.java @@ -94,4 +94,6 @@ public interface NodeLIRBuilderTool extends NodeValueMap { default OptionValues getOptions() { return getLIRGeneratorTool().getResult().getLIR().getOptions(); } + + void emitReadExceptionObject(ValueNode node); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/Replacements.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/Replacements.java index a525ed5007e..705afdb438e 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/Replacements.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/spi/Replacements.java @@ -64,7 +64,9 @@ public interface Replacements { * @param trackNodeSourcePosition * @return the snippet graph, if any, that is derived from {@code method} */ - StructuredGraph getSnippet(ResolvedJavaMethod method, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition); + default StructuredGraph getSnippet(ResolvedJavaMethod method, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) { + return getSnippet(method, null, args, trackNodeSourcePosition, replaceePosition); + } /** * Gets the snippet graph derived from a given method. diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/type/NarrowOopStamp.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/type/NarrowOopStamp.java index 512c0b92444..4962ee8bceb 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/type/NarrowOopStamp.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/type/NarrowOopStamp.java @@ -44,6 +44,13 @@ public abstract class NarrowOopStamp extends AbstractObjectStamp { this.encoding = encoding; } + @Override + public void accept(Visitor v) { + super.accept(v); + v.visitLong(encoding.getBase()); + v.visitInt(encoding.getShift()); + } + @Override protected abstract AbstractObjectStamp copyWith(ResolvedJavaType type, boolean exactType, boolean nonNull, boolean alwaysNull); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/util/GraphUtil.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/util/GraphUtil.java index be968d299f2..0d01e7630fb 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/util/GraphUtil.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/util/GraphUtil.java @@ -1039,7 +1039,7 @@ public class GraphUtil { return; } - if (newLengthInt >= tool.getMaximumEntryCount()) { + if (newLengthInt > tool.getMaximumEntryCount()) { /* The new array size is higher than maximum allowed size of virtualized objects. */ return; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options.processor/src/org/graalvm/compiler/options/processor/OptionProcessor.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options.processor/src/org/graalvm/compiler/options/processor/OptionProcessor.java index cb31ee6e2f5..59250ea54da 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options.processor/src/org/graalvm/compiler/options/processor/OptionProcessor.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options.processor/src/org/graalvm/compiler/options/processor/OptionProcessor.java @@ -174,7 +174,7 @@ public class OptionProcessor extends AbstractProcessor { } enclosing = enclosing.getEnclosingElement(); } - if (enclosingPackage == null) { + if (enclosingPackage == null || enclosingPackage.isUnnamed()) { processingEnv.getMessager().printMessage(Kind.ERROR, "Option field cannot be declared in the unnamed package", element); return; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionDescriptor.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionDescriptor.java index dbcd2e915d3..f5a7441d00f 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionDescriptor.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionDescriptor.java @@ -34,14 +34,14 @@ import java.util.List; */ public final class OptionDescriptor { - protected final String name; - protected final OptionType optionType; - protected final Class optionValueType; - protected final String help; - protected final List extraHelp; - protected final OptionKey optionKey; - protected final Class declaringClass; - protected final String fieldName; + private final String name; + private final OptionType optionType; + private final Class optionValueType; + private final String help; + private final List extraHelp; + private final OptionKey optionKey; + private final Class declaringClass; + private final String fieldName; private static final String[] NO_EXTRA_HELP = {}; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionKey.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionKey.java index 836b71f96e8..e42961090b5 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionKey.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionKey.java @@ -140,6 +140,18 @@ public class OptionKey { return values.get(this); } + /** + * Gets the value of this option in {@code values} if it is present, otherwise + * {@link #getDefaultValue()}. + */ + @SuppressWarnings("unchecked") + public T getValueOrDefault(EconomicMap, Object> values) { + if (!values.containsKey(this)) { + return defaultValue; + } + return (T) values.get(this); + } + /** * Sets the value of this option in a given map. The * {@link #onValueUpdate(EconomicMap, Object, Object)} method is called once the value is set. diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionValues.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionValues.java index 72dd07bca22..128c6e94e75 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionValues.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionValues.java @@ -213,7 +213,7 @@ public class OptionValues { } String name = namePrefix + e.getKey(); - String assign = containsKey(desc.optionKey) ? ":=" : "="; + String assign = containsKey(desc.getOptionKey()) ? ":=" : "="; String typeName = desc.getOptionKey() instanceof EnumOptionKey ? "String" : desc.getOptionValueType().getSimpleName(); String linePrefix = String.format("%s %s %s ", name, assign, value); int typeStartPos = PROPERTY_LINE_WIDTH - typeName.length(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionsParser.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionsParser.java index 42d994a60cb..6418a62db28 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionsParser.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.options/src/org/graalvm/compiler/options/OptionsParser.java @@ -24,6 +24,9 @@ package org.graalvm.compiler.options; +import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; + import java.util.ArrayList; import java.util.Collection; import java.util.Formatter; @@ -39,11 +42,15 @@ import jdk.internal.vm.compiler.collections.MapCursor; */ public class OptionsParser { + private static volatile List cachedOptionDescriptors; + /** - * Gets an iterable composed of the {@link ServiceLoader}s to be used when looking for - * {@link OptionDescriptors} providers. + * Gets an iterable of available {@link OptionDescriptors}. */ public static Iterable getOptionsLoader() { + if (IS_IN_NATIVE_IMAGE || cachedOptionDescriptors != null) { + return cachedOptionDescriptors; + } boolean java8OrEarlier = System.getProperty("java.specification.version").compareTo("1.9") < 0; ClassLoader loader; if (java8OrEarlier) { @@ -58,7 +65,15 @@ public class OptionsParser { */ loader = ClassLoader.getSystemClassLoader(); } - return ServiceLoader.load(OptionDescriptors.class, loader); + Iterable result = ServiceLoader.load(OptionDescriptors.class, loader); + if (IS_BUILDING_NATIVE_IMAGE) { + ArrayList optionDescriptors = new ArrayList<>(); + for (OptionDescriptors descriptors : result) { + optionDescriptors.add(descriptors); + } + OptionsParser.cachedOptionDescriptors = optionDescriptors; + } + return result; } /** @@ -178,7 +193,7 @@ public class OptionsParser { } } - desc.optionKey.update(values, value); + desc.getOptionKey().update(values, value); } private static long parseLong(String v) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/FixReadsPhase.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/FixReadsPhase.java index a99d90a87a5..cb3dfbb559a 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/FixReadsPhase.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/FixReadsPhase.java @@ -28,6 +28,7 @@ import jdk.internal.vm.compiler.collections.EconomicMap; import jdk.internal.vm.compiler.collections.MapCursor; import org.graalvm.compiler.core.common.GraalOptions; import org.graalvm.compiler.core.common.cfg.BlockMap; +import org.graalvm.compiler.core.common.spi.ConstantFieldProvider; import org.graalvm.compiler.core.common.type.FloatStamp; import org.graalvm.compiler.core.common.type.Stamp; import org.graalvm.compiler.core.common.type.StampFactory; @@ -37,6 +38,7 @@ import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeMap; import org.graalvm.compiler.graph.NodeStack; import org.graalvm.compiler.graph.Position; +import org.graalvm.compiler.graph.spi.CanonicalizerTool; import org.graalvm.compiler.nodeinfo.InputType; import org.graalvm.compiler.nodes.AbstractBeginNode; import org.graalvm.compiler.nodes.AbstractMergeNode; @@ -68,6 +70,7 @@ import org.graalvm.compiler.nodes.memory.FloatingReadNode; import org.graalvm.compiler.nodes.memory.MemoryAccess; import org.graalvm.compiler.nodes.memory.MemoryPhiNode; import org.graalvm.compiler.nodes.util.GraphUtil; +import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.BasePhase; import org.graalvm.compiler.phases.Phase; import org.graalvm.compiler.phases.graph.ScheduledNodeIterator; @@ -76,7 +79,9 @@ import org.graalvm.compiler.phases.schedule.SchedulePhase.SchedulingStrategy; import org.graalvm.compiler.phases.tiers.LowTierContext; import org.graalvm.compiler.phases.tiers.PhaseContext; +import jdk.vm.ci.meta.Assumptions; import jdk.vm.ci.meta.Constant; +import jdk.vm.ci.meta.ConstantReflectionProvider; import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.meta.TriState; @@ -136,7 +141,7 @@ public class FixReadsPhase extends BasePhase { } - protected static class RawConditionalEliminationVisitor implements RecursiveVisitor { + public static class RawConditionalEliminationVisitor implements RecursiveVisitor { protected final NodeMap stampMap; protected final NodeStack undoOperations; @@ -147,8 +152,58 @@ public class FixReadsPhase extends BasePhase { private final BlockMap blockActionStart; private final EconomicMap> endMaps; private final DebugContext debug; + private final RawCanonicalizerTool rawCanonicalizerTool = new RawCanonicalizerTool(); - protected RawConditionalEliminationVisitor(StructuredGraph graph, ScheduleResult schedule, MetaAccessProvider metaAccess, boolean replaceInputsWithConstants) { + private class RawCanonicalizerTool implements NodeView, CanonicalizerTool { + + @Override + public Assumptions getAssumptions() { + return graph.getAssumptions(); + } + + @Override + public MetaAccessProvider getMetaAccess() { + return metaAccess; + } + + @Override + public ConstantReflectionProvider getConstantReflection() { + return null; + } + + @Override + public ConstantFieldProvider getConstantFieldProvider() { + return null; + } + + @Override + public boolean canonicalizeReads() { + return false; + } + + @Override + public boolean allUsagesAvailable() { + return true; + } + + @Override + public Integer smallestCompareWidth() { + return null; + } + + @Override + public OptionValues getOptions() { + return graph.getOptions(); + } + + @Override + public Stamp stamp(ValueNode node) { + return getBestStamp(node); + } + + } + + public RawConditionalEliminationVisitor(StructuredGraph graph, ScheduleResult schedule, MetaAccessProvider metaAccess, boolean replaceInputsWithConstants) { this.graph = graph; this.debug = graph.getDebug(); this.schedule = schedule; @@ -326,8 +381,22 @@ public class FixReadsPhase extends BasePhase { } protected void processUnary(UnaryNode node) { - Stamp newStamp = node.foldStamp(getBestStamp(node.getValue())); + ValueNode value = node.getValue(); + Stamp bestStamp = getBestStamp(value); + Stamp newStamp = node.foldStamp(bestStamp); if (!checkReplaceWithConstant(newStamp, node)) { + if (!bestStamp.equals(value.stamp(NodeView.DEFAULT))) { + ValueNode newNode = node.canonical(rawCanonicalizerTool); + if (newNode != node) { + // Canonicalization successfully triggered. + if (newNode != null && !newNode.isAlive()) { + newNode = graph.addWithoutUniqueWithInputs(newNode); + } + node.replaceAndDelete(newNode); + GraphUtil.tryKillUnused(value); + return; + } + } registerNewValueStamp(node, newStamp); } } @@ -346,10 +415,31 @@ public class FixReadsPhase extends BasePhase { } protected void processBinary(BinaryNode node) { - Stamp xStamp = getBestStamp(node.getX()); - Stamp yStamp = getBestStamp(node.getY()); + + ValueNode x = node.getX(); + ValueNode y = node.getY(); + + Stamp xStamp = getBestStamp(x); + Stamp yStamp = getBestStamp(y); Stamp newStamp = node.foldStamp(xStamp, yStamp); if (!checkReplaceWithConstant(newStamp, node)) { + + if (!xStamp.equals(x.stamp(NodeView.DEFAULT)) || !yStamp.equals(y.stamp(NodeView.DEFAULT))) { + // At least one of the inputs has an improved stamp => attempt to canonicalize + // based on that improvement. + ValueNode newNode = node.canonical(rawCanonicalizerTool); + if (newNode != node) { + // Canonicalization successfully triggered. + if (newNode != null && !newNode.isAlive()) { + newNode = graph.addWithoutUniqueWithInputs(newNode); + } + node.replaceAndDelete(newNode); + GraphUtil.tryKillUnused(x); + GraphUtil.tryKillUnused(y); + return; + } + } + registerNewValueStamp(node, newStamp); } } @@ -469,6 +559,10 @@ public class FixReadsPhase extends BasePhase { protected Stamp getBestStamp(ValueNode value) { ValueNode originalNode = value; + if (!value.isAlive()) { + return value.stamp(NodeView.DEFAULT); + } + StampElement currentStamp = stampMap.getAndGrow(originalNode); if (currentStamp == null) { return value.stamp(NodeView.DEFAULT); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/OptimizeDivPhase.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/OptimizeDivPhase.java new file mode 100644 index 00000000000..4c90f1a262c --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/OptimizeDivPhase.java @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.phases.common; + +import jdk.internal.vm.compiler.collections.Pair; +import org.graalvm.compiler.core.common.type.IntegerStamp; +import org.graalvm.compiler.nodes.ConstantNode; +import org.graalvm.compiler.nodes.FixedNode; +import org.graalvm.compiler.nodes.NodeView; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.calc.BinaryArithmeticNode; +import org.graalvm.compiler.nodes.calc.IntegerDivRemNode; +import org.graalvm.compiler.nodes.calc.IntegerMulHighNode; +import org.graalvm.compiler.nodes.calc.MulNode; +import org.graalvm.compiler.nodes.calc.NarrowNode; +import org.graalvm.compiler.nodes.calc.RightShiftNode; +import org.graalvm.compiler.nodes.calc.SignExtendNode; +import org.graalvm.compiler.nodes.calc.SignedDivNode; +import org.graalvm.compiler.nodes.calc.SignedRemNode; +import org.graalvm.compiler.nodes.calc.UnsignedRightShiftNode; +import org.graalvm.compiler.phases.Phase; + +import jdk.vm.ci.code.CodeUtil; + +public class OptimizeDivPhase extends Phase { + + @Override + protected void run(StructuredGraph graph) { + for (IntegerDivRemNode rem : graph.getNodes().filter(IntegerDivRemNode.class)) { + if (rem instanceof SignedRemNode && divByNonZeroConstant(rem)) { + optimizeRem(rem); + } + } + for (IntegerDivRemNode div : graph.getNodes().filter(IntegerDivRemNode.class)) { + if (div instanceof SignedDivNode && divByNonZeroConstant(div)) { + optimizeSignedDiv((SignedDivNode) div); + } + } + } + + @Override + public float codeSizeIncrease() { + return 5.0f; + } + + protected static boolean divByNonZeroConstant(IntegerDivRemNode divRemNode) { + return divRemNode.getY().isConstant() && divRemNode.getY().asJavaConstant().asLong() != 0; + } + + protected final void optimizeRem(IntegerDivRemNode rem) { + assert rem.getOp() == IntegerDivRemNode.Op.REM; + // Java spec 15.17.3.: (a/b)*b+(a%b) == a + // so a%b == a-(a/b)*b + StructuredGraph graph = rem.graph(); + ValueNode div = findDivForRem(rem); + ValueNode mul = BinaryArithmeticNode.mul(graph, div, rem.getY(), NodeView.DEFAULT); + ValueNode result = BinaryArithmeticNode.sub(graph, rem.getX(), mul, NodeView.DEFAULT); + graph.replaceFixedWithFloating(rem, result); + } + + private ValueNode findDivForRem(IntegerDivRemNode rem) { + if (rem.next() instanceof IntegerDivRemNode) { + IntegerDivRemNode div = (IntegerDivRemNode) rem.next(); + if (div.getOp() == IntegerDivRemNode.Op.DIV && div.getType() == rem.getType() && div.getX() == rem.getX() && div.getY() == rem.getY()) { + return div; + } + } + if (rem.predecessor() instanceof IntegerDivRemNode) { + IntegerDivRemNode div = (IntegerDivRemNode) rem.predecessor(); + if (div.getOp() == IntegerDivRemNode.Op.DIV && div.getType() == rem.getType() && div.getX() == rem.getX() && div.getY() == rem.getY()) { + return div; + } + } + + // not found, create a new one (will be optimized away later) + ValueNode div = rem.graph().addOrUniqueWithInputs(createDiv(rem)); + if (div instanceof FixedNode) { + rem.graph().addAfterFixed(rem, (FixedNode) div); + } + return div; + } + + protected ValueNode createDiv(IntegerDivRemNode rem) { + assert rem instanceof SignedRemNode; + return SignedDivNode.create(rem.getX(), rem.getY(), rem.getZeroCheck(), NodeView.DEFAULT); + } + + protected static void optimizeSignedDiv(SignedDivNode div) { + ValueNode forX = div.getX(); + long c = div.getY().asJavaConstant().asLong(); + assert c != 1 && c != -1 && c != 0; + + IntegerStamp dividendStamp = (IntegerStamp) forX.stamp(NodeView.DEFAULT); + int bitSize = dividendStamp.getBits(); + Pair nums = magicDivideConstants(c, bitSize); + long magicNum = nums.getLeft().longValue(); + int shiftNum = nums.getRight().intValue(); + assert shiftNum >= 0; + ConstantNode m = ConstantNode.forLong(magicNum); + + ValueNode value; + if (bitSize == 32) { + value = new MulNode(new SignExtendNode(forX, 64), m); + if ((c > 0 && magicNum < 0) || (c < 0 && magicNum > 0)) { + // Get upper 32-bits of the result + value = NarrowNode.create(new RightShiftNode(value, ConstantNode.forInt(32)), 32, NodeView.DEFAULT); + if (c > 0) { + value = BinaryArithmeticNode.add(value, forX, NodeView.DEFAULT); + } else { + value = BinaryArithmeticNode.sub(value, forX, NodeView.DEFAULT); + } + if (shiftNum > 0) { + value = new RightShiftNode(value, ConstantNode.forInt(shiftNum)); + } + } else { + value = new RightShiftNode(value, ConstantNode.forInt(32 + shiftNum)); + value = new NarrowNode(value, Integer.SIZE); + } + } else { + assert bitSize == 64; + value = new IntegerMulHighNode(forX, m); + if (c > 0 && magicNum < 0) { + value = BinaryArithmeticNode.add(value, forX, NodeView.DEFAULT); + } else if (c < 0 && magicNum > 0) { + value = BinaryArithmeticNode.sub(value, forX, NodeView.DEFAULT); + } + if (shiftNum > 0) { + value = new RightShiftNode(value, ConstantNode.forInt(shiftNum)); + } + } + + if (c < 0) { + ConstantNode s = ConstantNode.forInt(bitSize - 1); + ValueNode sign = UnsignedRightShiftNode.create(value, s, NodeView.DEFAULT); + value = BinaryArithmeticNode.add(value, sign, NodeView.DEFAULT); + } else if (dividendStamp.canBeNegative()) { + ConstantNode s = ConstantNode.forInt(bitSize - 1); + ValueNode sign = UnsignedRightShiftNode.create(forX, s, NodeView.DEFAULT); + value = BinaryArithmeticNode.add(value, sign, NodeView.DEFAULT); + } + + StructuredGraph graph = div.graph(); + graph.replaceFixed(div, graph.addOrUniqueWithInputs(value)); + } + + /** + * Borrowed from Hacker's Delight by Henry S. Warren, Jr. Figure 10-1. + */ + private static Pair magicDivideConstants(long divisor, int size) { + final long twoW = 1L << (size - 1); // 2 ^ (size - 1). + long t = twoW + (divisor >>> 63); + long ad = Math.abs(divisor); + long anc = t - 1 - Long.remainderUnsigned(t, ad); // Absolute value of nc. + long q1 = Long.divideUnsigned(twoW, anc); // Init. q1 = 2**p/|nc|. + long r1 = Long.remainderUnsigned(twoW, anc); // Init. r1 = rem(2**p, |nc|). + long q2 = Long.divideUnsigned(twoW, ad); // Init. q2 = 2**p/|d|. + long r2 = Long.remainderUnsigned(twoW, ad); // Init. r2 = rem(2**p, |d|). + long delta; + + int p = size - 1; // Init. p. + do { + p = p + 1; + q1 = 2 * q1; // Update q1 = 2**p/|nc|. + r1 = 2 * r1; // Update r1 = rem(2**p, |nc|). + if (Long.compareUnsigned(r1, anc) >= 0) { // Must be an unsigned comparison. + q1 = q1 + 1; + r1 = r1 - anc; + } + q2 = 2 * q2; // Update q2 = 2**p/|d|. + r2 = 2 * r2; // Update r2 = rem(2**p, |d|). + if (Long.compareUnsigned(r2, ad) >= 0) { // Must be an unsigned comparison. + q2 = q2 + 1; + r2 = r2 - ad; + } + delta = ad - r2; + } while (Long.compareUnsigned(q1, delta) < 0 || (q1 == delta && r1 == 0)); + + long magic = CodeUtil.signExtend(q2 + 1, size); + if (divisor < 0) { + magic = -magic; + } + return Pair.create(magic, p - size); + } + +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/RemoveValueProxyPhase.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/RemoveValueProxyPhase.java index abe5075443a..b860bb743ab 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/RemoveValueProxyPhase.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/RemoveValueProxyPhase.java @@ -26,7 +26,6 @@ package org.graalvm.compiler.phases.common; import org.graalvm.compiler.nodes.FrameState; import org.graalvm.compiler.nodes.LoopExitNode; -import org.graalvm.compiler.nodes.ProxyNode; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.util.GraphUtil; import org.graalvm.compiler.phases.Phase; @@ -36,15 +35,14 @@ public class RemoveValueProxyPhase extends Phase { @Override protected void run(StructuredGraph graph) { for (LoopExitNode exit : graph.getNodes(LoopExitNode.TYPE)) { - for (ProxyNode vpn : exit.proxies().snapshot()) { - vpn.replaceAtUsagesAndDelete(vpn.value()); - } - FrameState stateAfter = exit.stateAfter(); - if (stateAfter != null) { + exit.removeProxies(); + FrameState frameState = exit.stateAfter(); + if (frameState != null && frameState.isExceptionHandlingBCI()) { + // The parser will create loop exits with such BCIs on the exception handling path. + // Loop optimizations must avoid duplicating such exits + // We clean them up here otherwise they could survive until code generation exit.setStateAfter(null); - if (stateAfter.hasNoUsages()) { - GraphUtil.killWithUnusedFloatingInputs(stateAfter); - } + GraphUtil.tryKillUnused(frameState); } } graph.setHasValueProxies(false); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases/src/org/graalvm/compiler/phases/graph/ReentrantBlockIterator.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases/src/org/graalvm/compiler/phases/graph/ReentrantBlockIterator.java index daf636870dd..7892d48b547 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases/src/org/graalvm/compiler/phases/graph/ReentrantBlockIterator.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases/src/org/graalvm/compiler/phases/graph/ReentrantBlockIterator.java @@ -77,13 +77,13 @@ public final class ReentrantBlockIterator { EconomicMap blockEndStates = apply(closure, loop.getHeader(), initialState, block -> !(block.getLoop() == loop || block.isLoopHeader())); Block[] predecessors = loop.getHeader().getPredecessors(); - LoopInfo info = new LoopInfo<>(predecessors.length - 1, loop.getExits().size()); + LoopInfo info = new LoopInfo<>(predecessors.length - 1, loop.getLoopExits().size()); for (int i = 1; i < predecessors.length; i++) { StateT endState = blockEndStates.get(predecessors[i].getEndNode()); // make sure all end states are unique objects info.endStates.add(closure.cloneState(endState)); } - for (Block loopExit : loop.getExits()) { + for (Block loopExit : loop.getLoopExits()) { assert loopExit.getPredecessorCount() == 1; assert blockEndStates.containsKey(loopExit.getBeginNode()) : loopExit.getBeginNode() + " " + blockEndStates; StateT exitState = blockEndStates.get(loopExit.getBeginNode()); @@ -210,8 +210,8 @@ public final class ReentrantBlockIterator { List exitStates = closure.processLoop(loop, state); int i = 0; - assert loop.getExits().size() == exitStates.size(); - for (Block exit : loop.getExits()) { + assert loop.getLoopExits().size() == exitStates.size(); + for (Block exit : loop.getLoopExits()) { states.put(exit.getBeginNode(), exitStates.get(i++)); blockQueue.addFirst(exit); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases/src/org/graalvm/compiler/phases/util/GraphOrder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases/src/org/graalvm/compiler/phases/util/GraphOrder.java index 072eb932c1e..7ea9130dd34 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases/src/org/graalvm/compiler/phases/util/GraphOrder.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases/src/org/graalvm/compiler/phases/util/GraphOrder.java @@ -30,6 +30,7 @@ import java.util.List; import jdk.internal.vm.compiler.collections.EconomicMap; import jdk.internal.vm.compiler.collections.Equivalence; import org.graalvm.compiler.core.common.cfg.Loop; +import org.graalvm.compiler.debug.DebugContext; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.GraalGraphError; import org.graalvm.compiler.graph.Node; @@ -153,9 +154,10 @@ public final class GraphOrder { * This method schedules the graph and makes sure that, for every node, all inputs are available * at the position where it is scheduled. This is a very expensive assertion. */ + @SuppressWarnings("try") public static boolean assertSchedulableGraph(final StructuredGraph graph) { assert graph.getGuardsStage() != GuardsStage.AFTER_FSA : "Cannot use the BlockIteratorClosure after FrameState Assignment, HIR Loop Data Structures are no longer valid."; - try { + try (DebugContext.Scope s = graph.getDebug().scope("AssertSchedulableGraph")) { final SchedulePhase schedulePhase = new SchedulePhase(SchedulingStrategy.LATEST_OUT_OF_LOOPS, true); final EconomicMap loopEntryStates = EconomicMap.create(Equivalence.IDENTITY); schedulePhase.apply(graph, false); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.printer/src/org/graalvm/compiler/printer/GraphPrinterDumpHandler.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.printer/src/org/graalvm/compiler/printer/GraphPrinterDumpHandler.java index 38abf456ebb..a9afe0d0594 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.printer/src/org/graalvm/compiler/printer/GraphPrinterDumpHandler.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.printer/src/org/graalvm/compiler/printer/GraphPrinterDumpHandler.java @@ -43,6 +43,7 @@ import org.graalvm.compiler.debug.DebugDumpScope; import org.graalvm.compiler.debug.DebugOptions; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.debug.TTY; +import org.graalvm.compiler.debug.DebugOptions.PrintGraphTarget; import org.graalvm.compiler.graph.Graph; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.options.OptionValues; @@ -124,7 +125,7 @@ public class GraphPrinterDumpHandler implements DebugDumpHandler { @SuppressWarnings("try") public void dump(DebugContext debug, Object object, final String format, Object... arguments) { OptionValues options = debug.getOptions(); - if (object instanceof Graph && DebugOptions.PrintGraph.getValue(options)) { + if (object instanceof Graph && DebugOptions.PrintGraph.getValue(options) != PrintGraphTarget.Disable) { final Graph graph = (Graph) object; ensureInitialized(debug, graph); if (printer == null) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64GraphBuilderPlugins.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64GraphBuilderPlugins.java index e099ca49b36..56fc95a2f03 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64GraphBuilderPlugins.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.aarch64/src/org/graalvm/compiler/replacements/aarch64/AArch64GraphBuilderPlugins.java @@ -31,9 +31,9 @@ import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.Una import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.LOG10; import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.SIN; import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.TAN; -import static org.graalvm.compiler.serviceprovider.GraalServices.JAVA_SPECIFICATION_VERSION; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java11OrEarlier; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.JAVA_SPECIFICATION_VERSION; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java11OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import org.graalvm.compiler.bytecode.BytecodeProvider; import org.graalvm.compiler.lir.aarch64.AArch64ArithmeticLIRGeneratorTool.RoundingMode; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64GraphBuilderPlugins.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64GraphBuilderPlugins.java index 7ca8c43d321..026fb27bf1f 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64GraphBuilderPlugins.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64GraphBuilderPlugins.java @@ -32,9 +32,9 @@ import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.Una import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.LOG10; import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.SIN; import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.TAN; -import static org.graalvm.compiler.serviceprovider.GraalServices.JAVA_SPECIFICATION_VERSION; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java11OrEarlier; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.JAVA_SPECIFICATION_VERSION; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java11OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import java.util.Arrays; @@ -71,7 +71,7 @@ import sun.misc.Unsafe; public class AMD64GraphBuilderPlugins { - public static void register(Plugins plugins, BytecodeProvider replacementsBytecodeProvider, AMD64 arch, boolean arithmeticStubs, boolean explicitUnsafeNullChecks) { + public static void register(Plugins plugins, BytecodeProvider replacementsBytecodeProvider, AMD64 arch, boolean explicitUnsafeNullChecks) { InvocationPlugins invocationPlugins = plugins.getInvocationPlugins(); invocationPlugins.defer(new Runnable() { @Override @@ -82,10 +82,10 @@ public class AMD64GraphBuilderPlugins { registerPlatformSpecificUnsafePlugins(invocationPlugins, replacementsBytecodeProvider, explicitUnsafeNullChecks, new JavaKind[]{JavaKind.Int, JavaKind.Long, JavaKind.Object, JavaKind.Boolean, JavaKind.Byte, JavaKind.Short, JavaKind.Char, JavaKind.Float, JavaKind.Double}); registerUnsafePlugins(invocationPlugins, replacementsBytecodeProvider, explicitUnsafeNullChecks); - registerStringPlugins(invocationPlugins, arch, replacementsBytecodeProvider); - registerStringLatin1Plugins(invocationPlugins, arch, replacementsBytecodeProvider); - registerStringUTF16Plugins(invocationPlugins, arch, replacementsBytecodeProvider); - registerMathPlugins(invocationPlugins, arch, arithmeticStubs, replacementsBytecodeProvider); + registerStringPlugins(invocationPlugins, replacementsBytecodeProvider); + registerStringLatin1Plugins(invocationPlugins, replacementsBytecodeProvider); + registerStringUTF16Plugins(invocationPlugins, replacementsBytecodeProvider); + registerMathPlugins(invocationPlugins, arch, replacementsBytecodeProvider); registerArraysEqualsPlugins(invocationPlugins, replacementsBytecodeProvider); } }); @@ -155,21 +155,15 @@ public class AMD64GraphBuilderPlugins { } } - private static void registerMathPlugins(InvocationPlugins plugins, AMD64 arch, boolean arithmeticStubs, BytecodeProvider bytecodeProvider) { + private static void registerMathPlugins(InvocationPlugins plugins, AMD64 arch, BytecodeProvider bytecodeProvider) { Registration r = new Registration(plugins, Math.class, bytecodeProvider); registerUnaryMath(r, "log", LOG); registerUnaryMath(r, "log10", LOG10); registerUnaryMath(r, "exp", EXP); registerBinaryMath(r, "pow", POW); - if (arithmeticStubs) { - registerUnaryMath(r, "sin", SIN); - registerUnaryMath(r, "cos", COS); - registerUnaryMath(r, "tan", TAN); - } else { - r.registerMethodSubstitution(AMD64MathSubstitutions.class, "sin", double.class); - r.registerMethodSubstitution(AMD64MathSubstitutions.class, "cos", double.class); - r.registerMethodSubstitution(AMD64MathSubstitutions.class, "tan", double.class); - } + registerUnaryMath(r, "sin", SIN); + registerUnaryMath(r, "cos", COS); + registerUnaryMath(r, "tan", TAN); if (arch.getFeatures().contains(CPUFeature.SSE4_1)) { registerRound(r, "rint", RoundingMode.NEAREST); @@ -208,23 +202,19 @@ public class AMD64GraphBuilderPlugins { }); } - private static void registerStringPlugins(InvocationPlugins plugins, AMD64 arch, BytecodeProvider replacementsBytecodeProvider) { + private static void registerStringPlugins(InvocationPlugins plugins, BytecodeProvider replacementsBytecodeProvider) { if (Java8OrEarlier) { Registration r; r = new Registration(plugins, String.class, replacementsBytecodeProvider); r.setAllowOverwrite(true); - if (arch.getFeatures().contains(CPUFeature.SSE4_2)) { - r.registerMethodSubstitution(AMD64StringSubstitutions.class, "indexOf", char[].class, int.class, - int.class, char[].class, int.class, int.class, int.class); - } - if (arch.getFeatures().contains(CPUFeature.SSSE3)) { - r.registerMethodSubstitution(AMD64StringSubstitutions.class, "indexOf", Receiver.class, int.class, int.class); - } + r.registerMethodSubstitution(AMD64StringSubstitutions.class, "indexOf", char[].class, int.class, + int.class, char[].class, int.class, int.class, int.class); + r.registerMethodSubstitution(AMD64StringSubstitutions.class, "indexOf", Receiver.class, int.class, int.class); r.registerMethodSubstitution(AMD64StringSubstitutions.class, "compareTo", Receiver.class, String.class); } } - private static void registerStringLatin1Plugins(InvocationPlugins plugins, AMD64 arch, BytecodeProvider replacementsBytecodeProvider) { + private static void registerStringLatin1Plugins(InvocationPlugins plugins, BytecodeProvider replacementsBytecodeProvider) { if (JAVA_SPECIFICATION_VERSION >= 9) { Registration r = new Registration(plugins, "java.lang.StringLatin1", replacementsBytecodeProvider); r.setAllowOverwrite(true); @@ -232,14 +222,12 @@ public class AMD64GraphBuilderPlugins { r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "compareToUTF16", byte[].class, byte[].class); r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "inflate", byte[].class, int.class, char[].class, int.class, int.class); r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "inflate", byte[].class, int.class, byte[].class, int.class, int.class); - - if (arch.getFeatures().contains(CPUFeature.SSSE3)) { - r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "indexOf", byte[].class, int.class, int.class); - } + r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "indexOf", byte[].class, int.class, int.class); + r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "indexOf", byte[].class, int.class, byte[].class, int.class, int.class); } } - private static void registerStringUTF16Plugins(InvocationPlugins plugins, AMD64 arch, BytecodeProvider replacementsBytecodeProvider) { + private static void registerStringUTF16Plugins(InvocationPlugins plugins, BytecodeProvider replacementsBytecodeProvider) { if (JAVA_SPECIFICATION_VERSION >= 9) { Registration r = new Registration(plugins, "java.lang.StringUTF16", replacementsBytecodeProvider); r.setAllowOverwrite(true); @@ -247,10 +235,9 @@ public class AMD64GraphBuilderPlugins { r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "compareToLatin1", byte[].class, byte[].class); r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "compress", char[].class, int.class, byte[].class, int.class, int.class); r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "compress", byte[].class, int.class, byte[].class, int.class, int.class); - - if (arch.getFeatures().contains(CPUFeature.SSSE3)) { - r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "indexOfCharUnsafe", byte[].class, int.class, int.class, int.class); - } + r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "indexOfCharUnsafe", byte[].class, int.class, int.class, int.class); + r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "indexOfUnsafe", byte[].class, int.class, byte[].class, int.class, int.class); + r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "indexOfLatin1Unsafe", byte[].class, int.class, byte[].class, int.class, int.class); } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64MathSubstitutions.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64MathSubstitutions.java deleted file mode 100644 index b3277792ce2..00000000000 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64MathSubstitutions.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - - -package org.graalvm.compiler.replacements.amd64; - -import org.graalvm.compiler.api.replacements.ClassSubstitution; -import org.graalvm.compiler.api.replacements.MethodSubstitution; -import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; -import org.graalvm.compiler.graph.Node.ConstantNodeParameter; -import org.graalvm.compiler.graph.Node.NodeIntrinsic; -import org.graalvm.compiler.nodes.extended.ForeignCallNode; -import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode; -import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation; - -// JaCoCo Exclude - -/** - * Substitutions for some {@link java.lang.Math} methods that leverage AMD64 instructions for - * selected input values. - */ -@ClassSubstitution(Math.class) -public class AMD64MathSubstitutions { - - private static final double PI_4 = Math.PI / 4; - - // NOTE on snippets below: - // Math.sin(), .cos() and .tan() guarantee a value within 1 ULP of the - // exact result, but x87 trigonometric FPU instructions are only that - // accurate within [-pi/4, pi/4]. Examine the passed value and provide - // a slow path for inputs outside of that interval. - - @MethodSubstitution - public static double sin(double x) { - if (Math.abs(x) < PI_4) { - return UnaryMathIntrinsicNode.compute(x, UnaryOperation.SIN); - } else { - return callDouble1(UnaryOperation.SIN.foreignCallDescriptor, x); - } - } - - @MethodSubstitution - public static double cos(double x) { - if (Math.abs(x) < PI_4) { - return UnaryMathIntrinsicNode.compute(x, UnaryOperation.COS); - } else { - return callDouble1(UnaryOperation.COS.foreignCallDescriptor, x); - } - } - - @MethodSubstitution - public static double tan(double x) { - if (Math.abs(x) < PI_4) { - return UnaryMathIntrinsicNode.compute(x, UnaryOperation.TAN); - } else { - return callDouble1(UnaryOperation.TAN.foreignCallDescriptor, x); - } - } - - @NodeIntrinsic(value = ForeignCallNode.class) - private static native double callDouble1(@ConstantNodeParameter ForeignCallDescriptor descriptor, double value); - - @NodeIntrinsic(value = ForeignCallNode.class) - private static native double callDouble2(@ConstantNodeParameter ForeignCallDescriptor descriptor, double a, double b); -} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64StringLatin1Substitutions.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64StringLatin1Substitutions.java index 811ee02afd5..0b27ee7bacd 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64StringLatin1Substitutions.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64StringLatin1Substitutions.java @@ -30,6 +30,7 @@ import org.graalvm.compiler.api.replacements.Fold.InjectedParameter; import org.graalvm.compiler.api.replacements.MethodSubstitution; import org.graalvm.compiler.nodes.DeoptimizeNode; import org.graalvm.compiler.replacements.nodes.ArrayCompareToNode; +import org.graalvm.compiler.replacements.nodes.ArrayRegionEqualsNode; import org.graalvm.compiler.word.Word; import jdk.internal.vm.compiler.word.Pointer; @@ -89,7 +90,15 @@ public class AMD64StringLatin1Substitutions { return ArrayCompareToNode.compareTo(value, other, value.length, other.length, JavaKind.Byte, JavaKind.Char); } - @MethodSubstitution(optional = true) + private static Word pointer(byte[] target) { + return Word.objectToTrackedPointer(target).add(byteArrayBaseOffset(INJECTED)); + } + + private static Word byteOffsetPointer(byte[] source, int offset) { + return pointer(source).add(offset * byteArrayIndexScale(INJECTED)); + } + + @MethodSubstitution public static int indexOf(byte[] value, int ch, int origFromIndex) { int fromIndex = origFromIndex; if (ch >>> 8 != 0) { @@ -103,7 +112,7 @@ public class AMD64StringLatin1Substitutions { // Note: fromIndex might be near -1>>>1. return -1; } - Pointer sourcePointer = Word.objectToTrackedPointer(value).add(byteArrayBaseOffset(INJECTED)).add(fromIndex); + Pointer sourcePointer = byteOffsetPointer(value, fromIndex); int result = AMD64ArrayIndexOf.indexOf1Byte(sourcePointer, length - fromIndex, (byte) ch); if (result != -1) { return result + fromIndex; @@ -111,6 +120,59 @@ public class AMD64StringLatin1Substitutions { return result; } + @MethodSubstitution + public static int indexOf(byte[] source, int sourceCount, byte[] target, int targetCount, int origFromIndex) { + int fromIndex = origFromIndex; + if (fromIndex >= sourceCount) { + return (targetCount == 0 ? sourceCount : -1); + } + if (fromIndex < 0) { + fromIndex = 0; + } + if (targetCount == 0) { + // The empty string is in every string. + return fromIndex; + } + if (sourceCount - fromIndex < targetCount) { + // The empty string contains nothing except the empty string. + return -1; + } + int totalOffset = fromIndex; + if (targetCount == 1) { + Pointer sourcePointer = byteOffsetPointer(source, totalOffset); + int indexOfResult = AMD64ArrayIndexOf.indexOf1Byte(sourcePointer, sourceCount - fromIndex, target[0]); + if (indexOfResult >= 0) { + return indexOfResult + totalOffset; + } + return indexOfResult; + } else if (targetCount == 2) { + Pointer sourcePointer = byteOffsetPointer(source, totalOffset); + int indexOfResult = AMD64ArrayIndexOf.indexOfTwoConsecutiveBytes(sourcePointer, sourceCount - fromIndex, target[0], target[1]); + if (indexOfResult >= 0) { + return indexOfResult + totalOffset; + } + return indexOfResult; + } else { + int haystackLength = sourceCount - (fromIndex + (targetCount - 2)); + while (haystackLength > 0) { + Pointer sourcePointer = byteOffsetPointer(source, totalOffset); + int indexOfResult = AMD64ArrayIndexOf.indexOfTwoConsecutiveBytes(sourcePointer, haystackLength, target[0], target[1]); + if (indexOfResult < 0) { + return -1; + } + totalOffset += indexOfResult; + haystackLength -= (indexOfResult + 1); + Pointer cmpSourcePointer = byteOffsetPointer(source, totalOffset); + Pointer targetPointer = pointer(target); + if (ArrayRegionEqualsNode.regionEquals(cmpSourcePointer, targetPointer, targetCount, JavaKind.Byte)) { + return totalOffset; + } + totalOffset++; + } + return -1; + } + } + /** * Intrinsic for {@code java.lang.StringLatin1.inflate([BI[CII)V}. * diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64StringSubstitutions.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64StringSubstitutions.java index 9d18765954e..550cbfd72fd 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64StringSubstitutions.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64StringSubstitutions.java @@ -82,7 +82,6 @@ public class AMD64StringSubstitutions { // The empty string contains nothing except the empty string. return -1; } - assert sourceCount - fromIndex > 0 && targetCount > 0; if (targetCount == 1) { Pointer sourcePointer = Word.objectToTrackedPointer(source).add(charArrayBaseOffset(INJECTED)).add(totalOffset * charArrayIndexScale(INJECTED)); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64StringUTF16Substitutions.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64StringUTF16Substitutions.java index 8789d742fda..ea678bbb0ea 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64StringUTF16Substitutions.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.amd64/src/org/graalvm/compiler/replacements/amd64/AMD64StringUTF16Substitutions.java @@ -24,25 +24,27 @@ package org.graalvm.compiler.replacements.amd64; +import jdk.vm.ci.meta.DeoptimizationAction; +import jdk.vm.ci.meta.DeoptimizationReason; +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.MetaAccessProvider; import org.graalvm.compiler.api.replacements.ClassSubstitution; import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.api.replacements.Fold.InjectedParameter; import org.graalvm.compiler.api.replacements.MethodSubstitution; import org.graalvm.compiler.nodes.DeoptimizeNode; +import org.graalvm.compiler.replacements.ReplacementsUtil; +import org.graalvm.compiler.replacements.StringUTF16Substitutions; import org.graalvm.compiler.replacements.nodes.ArrayCompareToNode; +import org.graalvm.compiler.replacements.nodes.ArrayRegionEqualsNode; import org.graalvm.compiler.word.Word; import jdk.internal.vm.compiler.word.Pointer; -import jdk.vm.ci.meta.DeoptimizationAction; -import jdk.vm.ci.meta.DeoptimizationReason; -import jdk.vm.ci.meta.JavaKind; -import jdk.vm.ci.meta.MetaAccessProvider; - // JaCoCo Exclude /** * Substitutions for {@code java.lang.StringUTF16} methods. - * + *

* Since JDK 9. */ @ClassSubstitution(className = "java.lang.StringUTF16", optional = true) @@ -68,9 +70,15 @@ public class AMD64StringUTF16Substitutions { return metaAccess.getArrayIndexScale(JavaKind.Char); } - /** Marker value for the {@link InjectedParameter} injected parameter. */ + /** + * Marker value for the {@link InjectedParameter} injected parameter. + */ static final MetaAccessProvider INJECTED = null; + public static int length(byte[] value) { + return value.length >> 1; + } + /** * @param value is char[] * @param other is char[] @@ -93,9 +101,9 @@ public class AMD64StringUTF16Substitutions { return ArrayCompareToNode.compareTo(other, value, other.length, value.length, JavaKind.Char, JavaKind.Byte); } - @MethodSubstitution(optional = true) + @MethodSubstitution public static int indexOfCharUnsafe(byte[] value, int ch, int fromIndex, int max) { - Pointer sourcePointer = Word.objectToTrackedPointer(value).add(byteArrayBaseOffset(INJECTED)).add(fromIndex * charArrayIndexScale(INJECTED)); + Pointer sourcePointer = charOffsetPointer(value, fromIndex); int result = AMD64ArrayIndexOf.indexOf1Char(sourcePointer, max - fromIndex, (char) ch); if (result != -1) { return result + fromIndex; @@ -103,6 +111,100 @@ public class AMD64StringUTF16Substitutions { return result; } + private static Word pointer(byte[] target) { + return Word.objectToTrackedPointer(target).add(byteArrayBaseOffset(INJECTED)); + } + + private static Word charOffsetPointer(byte[] value, int offset) { + return pointer(value).add(offset * charArrayIndexScale(INJECTED)); + } + + @MethodSubstitution + public static int indexOfUnsafe(byte[] source, int sourceCount, byte[] target, int targetCount, int fromIndex) { + ReplacementsUtil.runtimeAssert(fromIndex >= 0, "StringUTF16.indexOfUnsafe invalid args: fromIndex negative"); + ReplacementsUtil.runtimeAssert(targetCount > 0, "StringUTF16.indexOfUnsafe invalid args: targetCount <= 0"); + ReplacementsUtil.runtimeAssert(targetCount <= length(target), "StringUTF16.indexOfUnsafe invalid args: targetCount > length(target)"); + ReplacementsUtil.runtimeAssert(sourceCount >= targetCount, "StringUTF16.indexOfUnsafe invalid args: sourceCount < targetCount"); + int totalOffset = fromIndex; + if (targetCount == 1) { + Pointer sourcePointer = charOffsetPointer(source, totalOffset); + int indexOfResult = AMD64ArrayIndexOf.indexOf1Char(sourcePointer, sourceCount - fromIndex, StringUTF16Substitutions.getChar(target, 0)); + if (indexOfResult >= 0) { + return indexOfResult + totalOffset; + } + return indexOfResult; + } else if (targetCount == 2) { + Pointer sourcePointer = charOffsetPointer(source, totalOffset); + int indexOfResult = AMD64ArrayIndexOf.indexOfTwoConsecutiveChars(sourcePointer, sourceCount - fromIndex, StringUTF16Substitutions.getChar(target, 0), + StringUTF16Substitutions.getChar(target, 1)); + if (indexOfResult >= 0) { + return indexOfResult + totalOffset; + } + return indexOfResult; + } else { + int haystackLength = sourceCount - (fromIndex + (targetCount - 2)); + while (haystackLength > 0) { + Pointer sourcePointer = charOffsetPointer(source, totalOffset); + int indexOfResult = AMD64ArrayIndexOf.indexOfTwoConsecutiveChars(sourcePointer, haystackLength, StringUTF16Substitutions.getChar(target, 0), + StringUTF16Substitutions.getChar(target, 1)); + if (indexOfResult < 0) { + return -1; + } + totalOffset += indexOfResult; + haystackLength -= (indexOfResult + 1); + Pointer cmpSourcePointer = charOffsetPointer(source, totalOffset); + Pointer targetPointer = pointer(target); + if (ArrayRegionEqualsNode.regionEquals(cmpSourcePointer, targetPointer, targetCount, JavaKind.Char)) { + return totalOffset; + } + totalOffset++; + } + return -1; + } + } + + @MethodSubstitution + public static int indexOfLatin1Unsafe(byte[] source, int sourceCount, byte[] target, int targetCount, int fromIndex) { + ReplacementsUtil.runtimeAssert(fromIndex >= 0, "StringUTF16.indexOfLatin1Unsafe invalid args: fromIndex negative"); + ReplacementsUtil.runtimeAssert(targetCount > 0, "StringUTF16.indexOfLatin1Unsafe invalid args: targetCount <= 0"); + ReplacementsUtil.runtimeAssert(targetCount <= target.length, "StringUTF16.indexOfLatin1Unsafe invalid args: targetCount > length(target)"); + ReplacementsUtil.runtimeAssert(sourceCount >= targetCount, "StringUTF16.indexOfLatin1Unsafe invalid args: sourceCount < targetCount"); + int totalOffset = fromIndex; + if (targetCount == 1) { + Pointer sourcePointer = charOffsetPointer(source, totalOffset); + int indexOfResult = AMD64ArrayIndexOf.indexOf1Char(sourcePointer, sourceCount - fromIndex, (char) Byte.toUnsignedInt(target[0])); + if (indexOfResult >= 0) { + return indexOfResult + totalOffset; + } + return indexOfResult; + } else if (targetCount == 2) { + Pointer sourcePointer = charOffsetPointer(source, totalOffset); + int indexOfResult = AMD64ArrayIndexOf.indexOfTwoConsecutiveChars(sourcePointer, sourceCount - fromIndex, (char) Byte.toUnsignedInt(target[0]), (char) Byte.toUnsignedInt(target[1])); + if (indexOfResult >= 0) { + return indexOfResult + totalOffset; + } + return indexOfResult; + } else { + int haystackLength = sourceCount - (fromIndex + (targetCount - 2)); + while (haystackLength > 0) { + Pointer sourcePointer = charOffsetPointer(source, totalOffset); + int indexOfResult = AMD64ArrayIndexOf.indexOfTwoConsecutiveChars(sourcePointer, haystackLength, (char) Byte.toUnsignedInt(target[0]), (char) Byte.toUnsignedInt(target[1])); + if (indexOfResult < 0) { + return -1; + } + totalOffset += indexOfResult; + haystackLength -= (indexOfResult + 1); + Pointer cmpSourcePointer = charOffsetPointer(source, totalOffset); + Pointer targetPointer = pointer(target); + if (ArrayRegionEqualsNode.regionEquals(cmpSourcePointer, targetPointer, targetCount, JavaKind.Char, JavaKind.Byte)) { + return totalOffset; + } + totalOffset++; + } + return -1; + } + } + /** * Intrinsic for {@code java.lang.StringUTF16.compress([CI[BII)I}. * @@ -129,7 +231,7 @@ public class AMD64StringUTF16Substitutions { * @HotSpotIntrinsicCandidate * public static int compress(byte[] src, int src_indx, byte[] dst, int dst_indx, int len) * - * + *

* In this variant {@code dest} refers to a byte array containing 2 byte per char so * {@code srcIndex} and {@code len} are in terms of char elements and have to be scaled by 2 * when referring to {@code src}. diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.processor/src/org/graalvm/compiler/replacements/processor/GeneratedPlugin.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.processor/src/org/graalvm/compiler/replacements/processor/GeneratedPlugin.java index d0924af6ee9..22b5a441d54 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.processor/src/org/graalvm/compiler/replacements/processor/GeneratedPlugin.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.processor/src/org/graalvm/compiler/replacements/processor/GeneratedPlugin.java @@ -208,7 +208,11 @@ public abstract class GeneratedPlugin { out.printf(" %s arg%d;\n", getErasedType(type), argIdx); out.printf(" if (args[%d].isConstant()) {\n", nodeIdx); if (type.equals(processor.getType("jdk.vm.ci.meta.ResolvedJavaType"))) { - out.printf(" arg%d = %s.asJavaType(args[%d].asConstant());\n", argIdx, deps.use(WellKnownDependency.CONSTANT_REFLECTION), nodeIdx); + out.printf(" jdk.vm.ci.meta.JavaConstant cst = args[%d].asJavaConstant();\n", nodeIdx); + out.printf(" arg%d = %s.asJavaType(cst);\n", argIdx, deps.use(WellKnownDependency.CONSTANT_REFLECTION)); + out.printf(" if (arg%d == null) {\n", argIdx); + out.printf(" arg%d = %s.asObject(jdk.vm.ci.meta.ResolvedJavaType.class, cst);\n", argIdx, deps.use(WellKnownDependency.SNIPPET_REFLECTION)); + out.printf(" }\n"); } else { switch (type.getKind()) { case BOOLEAN: diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/DeoptimizeOnIntegerExactTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/DeoptimizeOnIntegerExactTest.java index 430d9b51d9a..c7764a04f64 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/DeoptimizeOnIntegerExactTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/DeoptimizeOnIntegerExactTest.java @@ -134,6 +134,7 @@ public class DeoptimizeOnIntegerExactTest extends GraalCompilerTest { @Override protected InstalledCode addMethod(DebugContext debug, final ResolvedJavaMethod method, final CompilationResult compilationResult) { - return getBackend().createInstalledCode(debug, method, compilationResult, speculationLog, null, false); + assert speculationLog == compilationResult.getSpeculationLog(); + return getBackend().createInstalledCode(debug, method, compilationResult, null, false); } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/IntegerExactFoldTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/IntegerExactFoldTest.java index 1a1a2bc1872..d57062c5460 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/IntegerExactFoldTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/IntegerExactFoldTest.java @@ -40,23 +40,21 @@ import org.graalvm.compiler.nodes.PiNode; import org.graalvm.compiler.nodes.ReturnNode; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions; -import org.graalvm.compiler.nodes.StructuredGraph.GuardsStage; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.spi.LoweringTool; import org.graalvm.compiler.phases.common.CanonicalizerPhase; +import org.graalvm.compiler.phases.common.GuardLoweringPhase; import org.graalvm.compiler.phases.common.LoweringPhase; import org.graalvm.compiler.phases.tiers.HighTierContext; -import org.graalvm.compiler.phases.tiers.PhaseContext; +import org.graalvm.compiler.phases.tiers.MidTierContext; import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerExactArithmeticNode; import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerExactArithmeticSplitNode; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; -@Ignore @RunWith(Parameterized.class) public class IntegerExactFoldTest extends GraalCompilerTest { private final long lowerBoundA; @@ -97,6 +95,7 @@ public class IntegerExactFoldTest extends GraalCompilerTest { assertNotNull("original node must be in the graph", originalNode); new CanonicalizerPhase().apply(graph, getDefaultHighTierContext()); + ValueNode node = findNode(graph); boolean overflowExpected = node instanceof IntegerExactArithmeticNode; @@ -110,17 +109,19 @@ public class IntegerExactFoldTest extends GraalCompilerTest { Node originalNode = graph.getNodes().filter(x -> x instanceof IntegerExactArithmeticNode).first(); assertNotNull("original node must be in the graph", originalNode); - - graph.setGuardsStage(GuardsStage.FIXED_DEOPTS); CanonicalizerPhase canonicalizer = new CanonicalizerPhase(); - PhaseContext context = new PhaseContext(getProviders()); - new LoweringPhase(canonicalizer, LoweringTool.StandardLoweringStage.HIGH_TIER).apply(graph, context); + HighTierContext highTierContext = getDefaultHighTierContext(); + new LoweringPhase(canonicalizer, LoweringTool.StandardLoweringStage.HIGH_TIER).apply(graph, highTierContext); + MidTierContext midTierContext = getDefaultMidTierContext(); + new GuardLoweringPhase().apply(graph, midTierContext); + new CanonicalizerPhase().apply(graph, midTierContext); + IntegerExactArithmeticSplitNode loweredNode = graph.getNodes().filter(IntegerExactArithmeticSplitNode.class).first(); assertNotNull("the lowered node must be in the graph", loweredNode); loweredNode.getX().setStamp(StampFactory.forInteger(bits, lowerBoundA, upperBoundA)); loweredNode.getY().setStamp(StampFactory.forInteger(bits, lowerBoundB, upperBoundB)); - new CanonicalizerPhase().apply(graph, context); + new CanonicalizerPhase().apply(graph, midTierContext); ValueNode node = findNode(graph); boolean overflowExpected = node instanceof IntegerExactArithmeticSplitNode; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/ReplacementsParseTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/ReplacementsParseTest.java index 57a78a1e878..9d9d8d920a6 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/ReplacementsParseTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/ReplacementsParseTest.java @@ -24,6 +24,7 @@ package org.graalvm.compiler.replacements.test; +import static org.graalvm.compiler.debug.DebugOptions.DumpOnError; import static org.graalvm.compiler.java.BytecodeParserOptions.InlinePartialIntrinsicExitDuringParsing; import java.util.function.Function; @@ -40,15 +41,19 @@ import org.graalvm.compiler.core.common.spi.ForeignCallsProvider; import org.graalvm.compiler.core.common.type.Stamp; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.debug.DebugContext; +import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.GraalGraphError; import org.graalvm.compiler.graph.Node.ConstantNodeParameter; import org.graalvm.compiler.graph.Node.NodeIntrinsic; +import org.graalvm.compiler.java.BytecodeParser.BytecodeParserError; import org.graalvm.compiler.nodes.FrameState; import org.graalvm.compiler.nodes.PiNode; import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions; +import org.graalvm.compiler.nodes.StructuredGraph.Builder; import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.extended.OpaqueNode; import org.graalvm.compiler.nodes.extended.ForeignCallNode; +import org.graalvm.compiler.nodes.extended.OpaqueNode; import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin; import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin.Receiver; @@ -64,8 +69,10 @@ import org.graalvm.compiler.phases.common.FrameStateAssignmentPhase; import org.graalvm.compiler.phases.common.GuardLoweringPhase; import org.graalvm.compiler.phases.common.LoweringPhase; import org.graalvm.compiler.phases.tiers.HighTierContext; +import org.graalvm.compiler.test.GraalTest; import jdk.internal.vm.compiler.word.LocationIdentity; import org.junit.Assert; +import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; @@ -190,6 +197,10 @@ public class ReplacementsParseTest extends ReplacementsTest { static int nonVoidIntrinsicWithOptimizedSplit(int x) { return x; } + + static int div(int x, int y) { + return x / y; + } } @ClassSubstitution(TestObject.class) @@ -287,6 +298,12 @@ public class ReplacementsParseTest extends ReplacementsTest { return x; } + @MethodSubstitution + static int div(int x, int y) { + assert y != 0; + return x / y; + } + public static void nonVoidIntrinsicWithCallStub(int zLen) { nonVoidIntrinsicWithCallStub(STUB_CALL, zLen); } @@ -311,6 +328,7 @@ public class ReplacementsParseTest extends ReplacementsTest { r.registerMethodSubstitution(TestObjectSubstitutions.class, "copyFirstL2R", byte[].class, byte[].class); r.registerMethodSubstitution(TestObjectSubstitutions.class, "nonVoidIntrinsicWithCall", int.class, int.class); r.registerMethodSubstitution(TestObjectSubstitutions.class, "nonVoidIntrinsicWithOptimizedSplit", int.class); + r.registerMethodSubstitution(TestObjectSubstitutions.class, "div", int.class, int.class); if (replacementBytecodeProvider.supportsInvokedynamic()) { r.registerMethodSubstitution(TestObjectSubstitutions.class, "identity", String.class); @@ -357,6 +375,7 @@ public class ReplacementsParseTest extends ReplacementsTest { @Test public void testNextAfter() { + Assume.assumeFalse(GraalTest.Java8OrEarlier); double[] inArray = new double[1024]; double[] outArray = new double[1024]; for (int i = 0; i < inArray.length; i++) { @@ -590,6 +609,23 @@ public class ReplacementsParseTest extends ReplacementsTest { testGraph("nonVoidIntrinsicWithOptimizedSplit"); } + public static int div(int x, int y) { + return TestObject.div(x, y); + } + + @Test + public void testAssertionInMethodSubstitution() { + try { + ResolvedJavaMethod method = getResolvedJavaMethod("div"); + // avoid dumping graphs and printing exception since and exception is expected + OptionValues options = new OptionValues(getInitialOptions(), DumpOnError, false); + parse(new Builder(options, getDebugContext(options, null, method), AllowAssumptions.YES).method(method).compilationId(getCompilationId(method)), getEagerGraphBuilderSuite()); + throw GraalError.shouldNotReachHere("BytecodeParser should have complained about using assertion in an intrinsic."); + } catch (BytecodeParserError e) { + // Expected behavior + } + } + @SuppressWarnings("try") private void testGraph(String name) { StructuredGraph graph = parseEager(name, StructuredGraph.AllowAssumptions.YES); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringCompareToTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringCompareToTest.java index cb0f0d54fba..383cc0e4a96 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringCompareToTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringCompareToTest.java @@ -24,21 +24,23 @@ package org.graalvm.compiler.replacements.test; -import jdk.vm.ci.aarch64.AArch64; -import jdk.vm.ci.amd64.AMD64; -import jdk.vm.ci.meta.ResolvedJavaMethod; +import static org.graalvm.compiler.core.common.GraalOptions.RemoveNeverExecutedCode; + +import java.util.List; + import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.replacements.nodes.ArrayCompareToNode; import org.graalvm.compiler.serviceprovider.GraalServices; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import org.junit.Assert; import org.junit.Assume; import org.junit.Test; -import java.util.List; - -import static org.graalvm.compiler.core.common.GraalOptions.RemoveNeverExecutedCode; +import jdk.vm.ci.aarch64.AArch64; +import jdk.vm.ci.amd64.AMD64; +import jdk.vm.ci.meta.ResolvedJavaMethod; /** * Tests compareTo method intrinsic. @@ -89,7 +91,7 @@ public class StringCompareToTest extends StringSubstitutionTestBase { OptionValues options; boolean needCheckNode = true; - if (GraalServices.Java8OrEarlier) { + if (JavaVersionUtil.Java8OrEarlier) { needCheckNode = false; } else { List vmArgs = GraalServices.getInputArguments(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringIndexOfTestBase.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringIndexOfTestBase.java index de4ed71b51b..1c5de92f3ef 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringIndexOfTestBase.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/StringIndexOfTestBase.java @@ -24,6 +24,8 @@ package org.graalvm.compiler.replacements.test; +import static org.junit.Assume.assumeFalse; + import org.graalvm.compiler.core.test.GraalCompilerTest; import org.junit.Test; import org.junit.runner.RunWith; @@ -31,6 +33,7 @@ import org.junit.runners.Parameterized; import java.util.ArrayList; import java.util.Collection; +import jdk.vm.ci.aarch64.AArch64; @RunWith(value = Parameterized.class) public abstract class StringIndexOfTestBase extends GraalCompilerTest { @@ -44,6 +47,25 @@ public abstract class StringIndexOfTestBase extends GraalCompilerTest { String[] utf16targets = new String[]{"grga " + ((char) 0x10D) + "varak", "grga", ((char) 0x10D) + "varak"}; addTargets(tests, targets); addTargets(tests, utf16targets); + + // Check long targets + // Checkstyle: stop + String lipsum = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata "; + // Checkstyle: resume + String lipsumUTF16 = lipsum + ((char) 0x10D); + int[] subStringLengths = {7, 8, 15, 16, 31, 32, 63, 64}; + for (int len : subStringLengths) { + String target = lipsum.substring(50, 50 + len); + tests.add(new Object[]{lipsum, target}); + tests.add(new Object[]{lipsum, target + "X"}); + tests.add(new Object[]{lipsumUTF16, target}); + tests.add(new Object[]{lipsumUTF16, target + "X"}); + tests.add(new Object[]{lipsumUTF16, target + ((char) 0x10D)}); + } + tests.add(new Object[]{ + "\u0100\u0101\u0102\u0103\u0104\u0105\u0106\u0107\u00f9\u00fa\u00fb\u00fc\u00fd\u00fe\u00ff\u0108\u0109\u010a\u010b\u010c", + "\u00f9\u00fa\u00fb\u00fc\u00fd\u00fe\u00ff"}); + return tests; } @@ -99,6 +121,7 @@ public abstract class StringIndexOfTestBase extends GraalCompilerTest { @Test public void testStringBuilderIndexOfConstant() { + assumeFalse("Disabled on AArch64 due to issues on AArch64; see GR-13100 or JDK-8215792", getTarget().arch instanceof AArch64); /* * Put a copy of the target string in the space after the current string to detect cases * where we search too far. @@ -111,6 +134,7 @@ public abstract class StringIndexOfTestBase extends GraalCompilerTest { @Test public void testStringBuilderIndexOfConstantOffset() { + assumeFalse("Disabled on AArch64 due to issues on AArch64; see GR-13100 or JDK-8215792", getTarget().arch instanceof AArch64); /* * Put a copy of the target string in the space after the current string to detect cases * where we search too far. diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/classfile/ClassfileBytecodeProviderTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/classfile/ClassfileBytecodeProviderTest.java index 3a66eba3194..ce95c485f85 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/classfile/ClassfileBytecodeProviderTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/classfile/ClassfileBytecodeProviderTest.java @@ -195,6 +195,10 @@ public class ClassfileBytecodeProviderTest extends GraalCompilerTest { } protected void checkClass(MetaAccessProvider metaAccess, SnippetReflectionProvider snippetReflection, String className) throws ClassNotFoundException { + if (className.equals("jdk.vm.ci.services.JVMCIClassLoaderFactory")) { + // JVMCIClassLoaderFactory must only be initialized by the VM + return; + } Class c = Class.forName(className, true, getClass().getClassLoader()); ClassfileBytecodeProvider cbp = new ClassfileBytecodeProvider(metaAccess, snippetReflection); for (Method method : c.getDeclaredMethods()) { @@ -205,15 +209,21 @@ public class ClassfileBytecodeProviderTest extends GraalCompilerTest { private static void checkMethod(ClassfileBytecodeProvider cbp, MetaAccessProvider metaAccess, Executable executable) { ResolvedJavaMethod method = metaAccess.lookupJavaMethod(executable); if (method.hasBytecodes()) { - ResolvedJavaMethodBytecode expected = new ResolvedJavaMethodBytecode(method); Bytecode actual = getBytecode(cbp, method); - new BytecodeComparer(expected, actual).compare(); + if (actual != null) { + ResolvedJavaMethodBytecode expected = new ResolvedJavaMethodBytecode(method); + new BytecodeComparer(expected, actual).compare(); + } } } protected static Bytecode getBytecode(ClassfileBytecodeProvider cbp, ResolvedJavaMethod method) { try { return cbp.getBytecode(method); + } catch (UnsupportedClassVersionError e) { + // This can happen when a library containing old class files + // is bundled into a Graal jar (GR-12672). + return null; } catch (Throwable e) { throw new AssertionError(String.format("Error getting bytecode for %s", method.format("%H.%n(%p)")), e); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/CachingPEGraphDecoder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/CachingPEGraphDecoder.java index 53fa18ab25d..840d9624efd 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/CachingPEGraphDecoder.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/CachingPEGraphDecoder.java @@ -44,7 +44,7 @@ import org.graalvm.compiler.nodes.graphbuilderconf.NodePlugin; import org.graalvm.compiler.nodes.graphbuilderconf.ParameterPlugin; import org.graalvm.compiler.phases.OptimisticOptimizations; import org.graalvm.compiler.phases.common.CanonicalizerPhase; -import org.graalvm.compiler.phases.common.ConvertDeoptimizeToGuardPhase; +import org.graalvm.compiler.loop.phases.ConvertDeoptimizeToGuardPhase; import org.graalvm.compiler.phases.tiers.PhaseContext; import org.graalvm.compiler.phases.util.Providers; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/DefaultJavaLoweringProvider.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/DefaultJavaLoweringProvider.java index 4477c342de7..0aba8017a4d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/DefaultJavaLoweringProvider.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/DefaultJavaLoweringProvider.java @@ -47,7 +47,6 @@ import org.graalvm.compiler.api.directives.GraalDirectives; import org.graalvm.compiler.api.replacements.Snippet; import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; import org.graalvm.compiler.core.common.LIRKind; -import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.core.common.spi.ForeignCallsProvider; import org.graalvm.compiler.core.common.type.IntegerStamp; import org.graalvm.compiler.core.common.type.ObjectStamp; @@ -123,6 +122,7 @@ import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType; import org.graalvm.compiler.nodes.memory.ReadNode; import org.graalvm.compiler.nodes.memory.WriteNode; import org.graalvm.compiler.nodes.memory.address.AddressNode; +import org.graalvm.compiler.nodes.memory.address.IndexAddressNode; import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode; import org.graalvm.compiler.nodes.spi.Lowerable; import org.graalvm.compiler.nodes.spi.LoweringProvider; @@ -138,9 +138,7 @@ import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.util.Providers; import org.graalvm.compiler.replacements.SnippetLowerableMemoryNode.SnippetLowering; import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode; -import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode.BinaryOperation; import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode; -import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation; import jdk.internal.vm.compiler.word.LocationIdentity; import jdk.vm.ci.code.CodeUtil; @@ -206,6 +204,8 @@ public abstract class DefaultJavaLoweringProvider implements LoweringProvider { lowerLoadIndexedNode((LoadIndexedNode) n, tool); } else if (n instanceof StoreIndexedNode) { lowerStoreIndexedNode((StoreIndexedNode) n, tool); + } else if (n instanceof IndexAddressNode) { + lowerIndexAddressNode((IndexAddressNode) n); } else if (n instanceof ArrayLengthNode) { lowerArrayLengthNode((ArrayLengthNode) n, tool); } else if (n instanceof LoadHubNode) { @@ -321,27 +321,20 @@ public abstract class DefaultJavaLoweringProvider implements LoweringProvider { ResolvedJavaMethod method = math.graph().method(); if (method != null) { if (method.getAnnotation(Snippet.class) != null) { - /* - * In the context of the snippet use the LIR lowering instead of the Node lowering. - */ + // In the context of SnippetStub, i.e., Graal-generated stubs, use the LIR + // lowering to emit the stub assembly code instead of the Node lowering. return; } if (method.getName().equalsIgnoreCase(math.getOperation().name()) && tool.getMetaAccess().lookupJavaType(Math.class).equals(method.getDeclaringClass())) { - /* - * A root compilation of the intrinsic method should emit the full assembly - * implementation. - */ + // A root compilation of the intrinsic method should emit the full assembly + // implementation. return; } - - } - ForeignCallDescriptor foreignCall = toForeignCall(math.getOperation()); - if (foreignCall != null) { - StructuredGraph graph = math.graph(); - ForeignCallNode call = graph.add(new ForeignCallNode(foreignCalls, toForeignCall(math.getOperation()), math.getX(), math.getY())); - graph.addAfterFixed(tool.lastFixedNode(), call); - math.replaceAtUsages(call); } + StructuredGraph graph = math.graph(); + ForeignCallNode call = graph.add(new ForeignCallNode(foreignCalls, math.getOperation().foreignCallDescriptor, math.getX(), math.getY())); + graph.addAfterFixed(tool.lastFixedNode(), call); + math.replaceAtUsages(call); } private void lowerUnaryMath(UnaryMathIntrinsicNode math, LoweringTool tool) { @@ -350,36 +343,16 @@ public abstract class DefaultJavaLoweringProvider implements LoweringProvider { } ResolvedJavaMethod method = math.graph().method(); if (method != null) { - if (method.getAnnotation(Snippet.class) != null) { - /* - * In the context of the snippet use the LIR lowering instead of the Node lowering. - */ - return; - } if (method.getName().equalsIgnoreCase(math.getOperation().name()) && tool.getMetaAccess().lookupJavaType(Math.class).equals(method.getDeclaringClass())) { - /* - * A root compilation of the intrinsic method should emit the full assembly - * implementation. - */ + // A root compilation of the intrinsic method should emit the full assembly + // implementation. return; } - } - ForeignCallDescriptor foreignCall = toForeignCall(math.getOperation()); - if (foreignCall != null) { - StructuredGraph graph = math.graph(); - ForeignCallNode call = math.graph().add(new ForeignCallNode(foreignCalls, foreignCall, math.getValue())); - graph.addAfterFixed(tool.lastFixedNode(), call); - math.replaceAtUsages(call); - } - } - - protected ForeignCallDescriptor toForeignCall(UnaryOperation operation) { - return operation.foreignCallDescriptor; - } - - protected ForeignCallDescriptor toForeignCall(BinaryOperation operation) { - return operation.foreignCallDescriptor; + StructuredGraph graph = math.graph(); + ForeignCallNode call = math.graph().add(new ForeignCallNode(foreignCalls, math.getOperation().foreignCallDescriptor, math.getValue())); + graph.addAfterFixed(tool.lastFixedNode(), call); + math.replaceAtUsages(call); } protected void lowerVerifyHeap(VerifyHeapNode n) { @@ -476,6 +449,11 @@ public abstract class DefaultJavaLoweringProvider implements LoweringProvider { return graph.unique(new OffsetAddressNode(array, offset)); } + protected void lowerIndexAddressNode(IndexAddressNode indexAddress) { + AddressNode lowered = createArrayAddress(indexAddress.graph(), indexAddress.getArray(), indexAddress.getElementKind(), indexAddress.getIndex()); + indexAddress.replaceAndDelete(lowered); + } + protected void lowerLoadIndexedNode(LoadIndexedNode loadIndexed, LoweringTool tool) { StructuredGraph graph = loadIndexed.graph(); ValueNode array = loadIndexed.array(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/GraphKit.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/GraphKit.java index a0d26c4dd8a..c372ac5b822 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/GraphKit.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/GraphKit.java @@ -24,6 +24,7 @@ package org.graalvm.compiler.replacements; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; import static org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext.CompilationContext.INLINE_AFTER_PARSING; import java.lang.reflect.Method; @@ -358,13 +359,18 @@ public class GraphKit implements GraphBuilderTool { Plugins plugins = new Plugins(graphBuilderPlugins); GraphBuilderConfiguration config = GraphBuilderConfiguration.getSnippetDefault(plugins); - StructuredGraph calleeGraph = new StructuredGraph.Builder(invoke.getOptions(), invoke.getDebug()).method(method).trackNodeSourcePosition( - invoke.graph().trackNodeSourcePosition()).setIsSubstitution(true).build(); - IntrinsicContext initialReplacementContext = new IntrinsicContext(method, method, providers.getReplacements().getDefaultReplacementBytecodeProvider(), INLINE_AFTER_PARSING); - GraphBuilderPhase.Instance instance = createGraphBuilderInstance(metaAccess, providers.getStampProvider(), providers.getConstantReflection(), providers.getConstantFieldProvider(), config, - OptimisticOptimizations.NONE, - initialReplacementContext); - instance.apply(calleeGraph); + StructuredGraph calleeGraph; + if (IS_IN_NATIVE_IMAGE) { + calleeGraph = providers.getReplacements().getSnippet(method, null, false, null); + } else { + calleeGraph = new StructuredGraph.Builder(invoke.getOptions(), invoke.getDebug()).method(method).trackNodeSourcePosition(invoke.graph().trackNodeSourcePosition()).setIsSubstitution( + true).build(); + IntrinsicContext initialReplacementContext = new IntrinsicContext(method, method, providers.getReplacements().getDefaultReplacementBytecodeProvider(), INLINE_AFTER_PARSING); + GraphBuilderPhase.Instance instance = createGraphBuilderInstance(metaAccess, providers.getStampProvider(), providers.getConstantReflection(), providers.getConstantFieldProvider(), config, + OptimisticOptimizations.NONE, + initialReplacementContext); + instance.apply(calleeGraph); + } // Remove all frame states from inlinee calleeGraph.clearAllStateAfter(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/IntrinsicGraphBuilder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/IntrinsicGraphBuilder.java index efe58df122c..c5fc1e341c5 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/IntrinsicGraphBuilder.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/IntrinsicGraphBuilder.java @@ -40,6 +40,7 @@ import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind; import org.graalvm.compiler.nodes.FixedNode; import org.graalvm.compiler.nodes.FixedWithNextNode; import org.graalvm.compiler.nodes.FrameState; +import org.graalvm.compiler.nodes.Invoke; import org.graalvm.compiler.nodes.ParameterNode; import org.graalvm.compiler.nodes.ReturnNode; import org.graalvm.compiler.nodes.StateSplit; @@ -163,7 +164,7 @@ public class IntrinsicGraphBuilder implements GraphBuilderContext, Receiver { } @Override - public void handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean forceInlineEverything) { + public Invoke handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean forceInlineEverything) { throw GraalError.shouldNotReachHere(); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/PEGraphDecoder.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/PEGraphDecoder.java index 623da61f533..fc98b15069b 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/PEGraphDecoder.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/PEGraphDecoder.java @@ -353,7 +353,7 @@ public abstract class PEGraphDecoder extends SimplifyingGraphDecoder { } @Override - public void handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean inlineEverything) { + public Invoke handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean inlineEverything) { throw unimplemented(); } @@ -589,13 +589,19 @@ public abstract class PEGraphDecoder extends SimplifyingGraphDecoder { } } + @SuppressWarnings("try") public void decode(ResolvedJavaMethod method, boolean isSubstitution, boolean trackNodeSourcePosition) { - PEMethodScope methodScope = new PEMethodScope(graph, null, null, lookupEncodedGraph(method, null, null, isSubstitution, trackNodeSourcePosition), method, null, 0, loopExplosionPlugin, null); - decode(createInitialLoopScope(methodScope, null)); - cleanupGraph(methodScope); + try (DebugContext.Scope scope = debug.scope("PEGraphDecode", graph)) { + EncodedGraph encodedGraph = lookupEncodedGraph(method, null, null, isSubstitution, trackNodeSourcePosition); + PEMethodScope methodScope = new PEMethodScope(graph, null, null, encodedGraph, method, null, 0, loopExplosionPlugin, null); + decode(createInitialLoopScope(methodScope, null)); + cleanupGraph(methodScope); - debug.dump(DebugContext.VERBOSE_LEVEL, graph, "After graph cleanup"); - assert graph.verify(); + debug.dump(DebugContext.VERBOSE_LEVEL, graph, "After graph cleanup"); + assert graph.verify(); + } catch (Throwable t) { + throw debug.handle(t); + } try { /* Check that the control flow graph can be computed, to catch problems early. */ @@ -737,11 +743,19 @@ public abstract class PEGraphDecoder extends SimplifyingGraphDecoder { Invoke invoke = invokeData.invoke; ResolvedJavaMethod targetMethod = callTarget.targetMethod(); + if (loopScope.methodScope.encodedGraph.isCallToOriginal(targetMethod)) { + return false; + } + InvocationPlugin invocationPlugin = getInvocationPlugin(targetMethod); if (invocationPlugin == null) { return false; } + if (loopScope.methodScope.encodedGraph.isCallToOriginal(targetMethod)) { + return false; + } + ValueNode[] arguments = callTarget.arguments().toArray(new ValueNode[0]); FixedWithNextNode invokePredecessor = (FixedWithNextNode) invoke.asNode().predecessor(); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/ReplacementsImpl.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/ReplacementsImpl.java index 10c24ef6c4c..b7b45d9533e 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/ReplacementsImpl.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/ReplacementsImpl.java @@ -81,10 +81,13 @@ import org.graalvm.compiler.nodes.graphbuilderconf.MethodSubstitutionPlugin; import org.graalvm.compiler.nodes.java.MethodCallTargetNode; import org.graalvm.compiler.nodes.spi.Replacements; import org.graalvm.compiler.nodes.spi.StampProvider; +import org.graalvm.compiler.options.Option; +import org.graalvm.compiler.options.OptionKey; +import org.graalvm.compiler.options.OptionType; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.OptimisticOptimizations; import org.graalvm.compiler.phases.common.CanonicalizerPhase; -import org.graalvm.compiler.phases.common.ConvertDeoptimizeToGuardPhase; +import org.graalvm.compiler.loop.phases.ConvertDeoptimizeToGuardPhase; import org.graalvm.compiler.phases.common.DeadCodeEliminationPhase; import org.graalvm.compiler.phases.tiers.PhaseContext; import org.graalvm.compiler.phases.util.Providers; @@ -99,6 +102,13 @@ import jdk.vm.ci.meta.ResolvedJavaType; public class ReplacementsImpl implements Replacements, InlineInvokePlugin { + public static class Options { + // @formatter:off + @Option(help = "This is a testing option to exercise the SymbolicSnippetEncoder", type = OptionType.Expert) + public static final OptionKey UseEncodedSnippets = new OptionKey<>(false); + // @formatter:on + } + protected final OptionValues options; public Providers getProviders() { @@ -223,11 +233,6 @@ public class ReplacementsImpl implements Replacements, InlineInvokePlugin { private static final TimerKey SnippetPreparationTime = DebugContext.timer("SnippetPreparationTime"); - @Override - public StructuredGraph getSnippet(ResolvedJavaMethod method, Object[] args, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) { - return getSnippet(method, null, args, trackNodeSourcePosition, replaceePosition); - } - private static final AtomicInteger nextDebugContextId = new AtomicInteger(); public DebugContext openDebugContext(String idPrefix, ResolvedJavaMethod method) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetCounter.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetCounter.java index 513573476c8..945153e02c2 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetCounter.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetCounter.java @@ -35,6 +35,9 @@ import org.graalvm.compiler.core.common.GraalOptions; * snippet specific metrics. */ public final class SnippetCounter implements Comparable { + + public static final SnippetCounter DISABLED_COUNTER = new SnippetCounter(null, "Disabled", "Disabled"); + /** * A group of related counters. */ @@ -130,7 +133,7 @@ public final class SnippetCounter implements Comparable { * compile-time constant {@link SnippetCounter} object. */ public void inc() { - if (group != null) { + if (getGroup() != null) { SnippetCounterNode.increment(this); } } @@ -140,7 +143,7 @@ public final class SnippetCounter implements Comparable { * compile-time constant {@link SnippetCounter} object. */ public void add(int increment) { - if (group != null) { + if (getGroup() != null) { SnippetCounterNode.add(this, increment); } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetIntegerHistogram.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetIntegerHistogram.java index 2470e25a010..4ee563a3228 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetIntegerHistogram.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetIntegerHistogram.java @@ -29,6 +29,8 @@ package org.graalvm.compiler.replacements; * gathering snippet specific metrics. */ public final class SnippetIntegerHistogram { + public static final SnippetIntegerHistogram DISABLED_COUNTER = new SnippetIntegerHistogram(null, 1, "Disabled", "Disabled"); + private final SnippetCounter.Group group; private final String name; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetTemplate.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetTemplate.java index 72d199ae3ce..236fccd9427 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetTemplate.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/SnippetTemplate.java @@ -25,6 +25,7 @@ package org.graalvm.compiler.replacements; import static java.util.FormattableFlags.ALTERNATE; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; import static org.graalvm.compiler.debug.DebugContext.DEFAULT_LOG_STREAM; import static org.graalvm.compiler.debug.DebugContext.applyFormattingFlagsAndWidth; import static org.graalvm.compiler.debug.DebugOptions.DebugStubsAndSnippets; @@ -212,8 +213,9 @@ public class SnippetTemplate { constantParameters[0] = true; } - // Retrieve the names only when assertions are turned on. - assert initNames(method, count); + // Retrieve the names only when assertions are turned on. Parameter annotations are + // unsupported in the native image. + assert IS_IN_NATIVE_IMAGE || initNames(method, count); } final boolean[] constantParameters; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StandardGraphBuilderPlugins.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StandardGraphBuilderPlugins.java index 6a3e474f614..8ab184b2592 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StandardGraphBuilderPlugins.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StandardGraphBuilderPlugins.java @@ -33,8 +33,8 @@ import static jdk.vm.ci.code.MemoryBarriers.LOAD_STORE; import static jdk.vm.ci.code.MemoryBarriers.STORE_LOAD; import static jdk.vm.ci.code.MemoryBarriers.STORE_STORE; import static org.graalvm.compiler.nodes.NamedLocationIdentity.OFF_HEAP_LOCATION; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java11OrEarlier; -import static org.graalvm.compiler.serviceprovider.GraalServices.Java8OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java11OrEarlier; +import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.Java8OrEarlier; import java.lang.reflect.Array; import java.lang.reflect.Field; @@ -54,8 +54,6 @@ import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.Edges; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeList; -import org.graalvm.compiler.java.IntegerExactOpSpeculation; -import org.graalvm.compiler.java.IntegerExactOpSpeculation.IntegerExactOp; import org.graalvm.compiler.nodes.AbstractBeginNode; import org.graalvm.compiler.nodes.BeginNode; import org.graalvm.compiler.nodes.ConstantNode; @@ -66,6 +64,7 @@ import org.graalvm.compiler.nodes.FixedWithNextNode; import org.graalvm.compiler.nodes.IfNode; import org.graalvm.compiler.nodes.LogicNode; import org.graalvm.compiler.nodes.MergeNode; +import org.graalvm.compiler.nodes.NamedLocationIdentity; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.StateSplit; import org.graalvm.compiler.nodes.StructuredGraph; @@ -93,6 +92,9 @@ import org.graalvm.compiler.nodes.extended.BoxNode; import org.graalvm.compiler.nodes.extended.BranchProbabilityNode; import org.graalvm.compiler.nodes.extended.BytecodeExceptionNode.BytecodeExceptionKind; import org.graalvm.compiler.nodes.extended.GetClassNode; +import org.graalvm.compiler.nodes.extended.GuardingNode; +import org.graalvm.compiler.nodes.extended.JavaReadNode; +import org.graalvm.compiler.nodes.extended.JavaWriteNode; import org.graalvm.compiler.nodes.extended.MembarNode; import org.graalvm.compiler.nodes.extended.OpaqueNode; import org.graalvm.compiler.nodes.extended.RawLoadNode; @@ -113,6 +115,8 @@ import org.graalvm.compiler.nodes.java.LoadFieldNode; import org.graalvm.compiler.nodes.java.RegisterFinalizerNode; import org.graalvm.compiler.nodes.java.UnsafeCompareAndExchangeNode; import org.graalvm.compiler.nodes.java.UnsafeCompareAndSwapNode; +import org.graalvm.compiler.nodes.memory.HeapAccess; +import org.graalvm.compiler.nodes.memory.address.IndexAddressNode; import org.graalvm.compiler.nodes.type.StampTool; import org.graalvm.compiler.nodes.util.GraphUtil; import org.graalvm.compiler.nodes.virtual.EnsureVirtualizedNode; @@ -120,13 +124,16 @@ import org.graalvm.compiler.replacements.nodes.ProfileBooleanNode; import org.graalvm.compiler.replacements.nodes.ReverseBytesNode; import org.graalvm.compiler.replacements.nodes.VirtualizableInvokeMacroNode; import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerAddExactNode; +import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerAddExactOverflowNode; import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerAddExactSplitNode; -import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerExactArithmeticNode; import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerExactArithmeticSplitNode; import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerMulExactNode; +import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerMulExactOverflowNode; import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerMulExactSplitNode; import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerSubExactNode; +import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerSubExactOverflowNode; import org.graalvm.compiler.replacements.nodes.arithmetic.IntegerSubExactSplitNode; +import org.graalvm.compiler.serviceprovider.SpeculationReasonGroup; import jdk.internal.vm.compiler.word.LocationIdentity; import jdk.vm.ci.code.BytecodePosition; @@ -197,8 +204,10 @@ public class StandardGraphBuilderPlugins { public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) { if (receiver.isConstant()) { String s = snippetReflection.asObject(String.class, (JavaConstant) receiver.get().asConstant()); - b.addPush(JavaKind.Int, b.add(ConstantNode.forInt(s.hashCode()))); - return true; + if (s != null) { + b.addPush(JavaKind.Int, b.add(ConstantNode.forInt(s.hashCode()))); + return true; + } } return false; } @@ -221,12 +230,32 @@ public class StandardGraphBuilderPlugins { }); } else { r.registerMethodSubstitution(JDK9StringSubstitutions.class, "equals", Receiver.class, Object.class); + Registration utf16sub = new Registration(plugins, StringUTF16Substitutions.class, bytecodeProvider); + utf16sub.register2("getCharDirect", byte[].class, int.class, new InvocationPlugin() { + @Override + public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode arg1, ValueNode arg2) { + b.addPush(JavaKind.Char, new JavaReadNode(JavaKind.Char, new IndexAddressNode(arg1, arg2, JavaKind.Byte), NamedLocationIdentity.getArrayLocation(JavaKind.Byte), + HeapAccess.BarrierType.NONE, false)); + return true; + } + }); + utf16sub.register3("putCharDirect", byte[].class, int.class, int.class, new InvocationPlugin() { + @Override + public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode arg1, ValueNode arg2, ValueNode arg3) { + b.add(new JavaWriteNode(JavaKind.Char, new IndexAddressNode(arg1, arg2, JavaKind.Byte), NamedLocationIdentity.getArrayLocation(JavaKind.Byte), arg3, + HeapAccess.BarrierType.NONE, false)); + return true; + } + }); final Registration latin1r = new Registration(plugins, "java.lang.StringLatin1", bytecodeProvider); latin1r.register5("indexOf", byte[].class, int.class, byte[].class, int.class, int.class, new StringLatin1IndexOfConstantPlugin()); final Registration utf16r = new Registration(plugins, "java.lang.StringUTF16", bytecodeProvider); utf16r.register5("indexOfUnsafe", byte[].class, int.class, byte[].class, int.class, int.class, new StringUTF16IndexOfConstantPlugin()); + utf16r.setAllowOverwrite(true); + utf16r.registerMethodSubstitution(StringUTF16Substitutions.class, "getChar", byte[].class, int.class); + utf16r.registerMethodSubstitution(StringUTF16Substitutions.class, "putChar", byte[].class, int.class, int.class); Registration sr = new Registration(plugins, JDK9StringSubstitutions.class); sr.register1("getValue", String.class, new InvocationPlugin() { @@ -529,16 +558,47 @@ public class StandardGraphBuilderPlugins { }); } - private static ValueNode createIntegerExactArithmeticNode(ValueNode x, ValueNode y, SpeculationReason speculation, IntegerExactOp op) { + public enum IntegerExactOp { + INTEGER_ADD_EXACT, + INTEGER_INCREMENT_EXACT, + INTEGER_SUBTRACT_EXACT, + INTEGER_DECREMENT_EXACT, + INTEGER_MULTIPLY_EXACT + } + + private static GuardingNode createIntegerExactArithmeticGuardNode(GraphBuilderContext b, ValueNode x, ValueNode y, IntegerExactOp op) { + LogicNode overflowCheck; + switch (op) { + case INTEGER_ADD_EXACT: + case INTEGER_INCREMENT_EXACT: { + overflowCheck = new IntegerAddExactOverflowNode(x, y); + break; + } + case INTEGER_SUBTRACT_EXACT: + case INTEGER_DECREMENT_EXACT: { + overflowCheck = new IntegerSubExactOverflowNode(x, y); + break; + } + case INTEGER_MULTIPLY_EXACT: { + overflowCheck = new IntegerMulExactOverflowNode(x, y); + break; + } + default: + throw GraalError.shouldNotReachHere("Unknown integer exact operation."); + } + return b.add(new FixedGuardNode(overflowCheck, DeoptimizationReason.ArithmeticException, DeoptimizationAction.InvalidateRecompile, true)); + } + + private static ValueNode createIntegerExactArithmeticNode(GraphBuilderContext b, ValueNode x, ValueNode y, IntegerExactOp op) { switch (op) { case INTEGER_ADD_EXACT: case INTEGER_INCREMENT_EXACT: - return new IntegerAddExactNode(x, y, speculation); + return new IntegerAddExactNode(x, y, createIntegerExactArithmeticGuardNode(b, x, y, op)); case INTEGER_SUBTRACT_EXACT: case INTEGER_DECREMENT_EXACT: - return new IntegerSubExactNode(x, y, speculation); + return new IntegerSubExactNode(x, y, createIntegerExactArithmeticGuardNode(b, x, y, op)); case INTEGER_MULTIPLY_EXACT: - return new IntegerMulExactNode(x, y, speculation); + return new IntegerMulExactNode(x, y, createIntegerExactArithmeticGuardNode(b, x, y, op)); default: throw GraalError.shouldNotReachHere("Unknown integer exact operation."); } @@ -559,15 +619,15 @@ public class StandardGraphBuilderPlugins { } } - private static boolean createIntegerExactOperation(GraphBuilderContext b, JavaKind kind, ValueNode x, ValueNode y, IntegerExactOp op) { - BytecodeExceptionKind exceptionKind = kind == JavaKind.Int ? BytecodeExceptionKind.INTEGER_EXACT_OVERFLOW : BytecodeExceptionKind.LONG_EXACT_OVERFLOW; - AbstractBeginNode exceptionEdge = b.genExplicitExceptionEdge(exceptionKind); - if (exceptionEdge != null) { + private static void createIntegerExactOperation(GraphBuilderContext b, JavaKind kind, ValueNode x, ValueNode y, IntegerExactOp op) { + if (b.needsExplicitException()) { + BytecodeExceptionKind exceptionKind = kind == JavaKind.Int ? BytecodeExceptionKind.INTEGER_EXACT_OVERFLOW : BytecodeExceptionKind.LONG_EXACT_OVERFLOW; + AbstractBeginNode exceptionEdge = b.genExplicitExceptionEdge(exceptionKind); IntegerExactArithmeticSplitNode split = b.addPush(kind, createIntegerExactSplit(x, y, exceptionEdge, op)); split.setNext(b.add(new BeginNode())); - return true; + } else { + b.addPush(kind, createIntegerExactArithmeticNode(b, x, y, op)); } - return false; } private static void registerMathPlugins(InvocationPlugins plugins, boolean allowDeoptimization) { @@ -575,12 +635,12 @@ public class StandardGraphBuilderPlugins { if (allowDeoptimization) { for (JavaKind kind : new JavaKind[]{JavaKind.Int, JavaKind.Long}) { Class type = kind.toJavaClass(); - r.register1("decrementExact", type, new InvocationPlugin() { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x) { ConstantNode y = b.add(ConstantNode.forIntegerKind(kind, 1)); - return createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_DECREMENT_EXACT); + createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_DECREMENT_EXACT); + return true; } }); @@ -588,29 +648,31 @@ public class StandardGraphBuilderPlugins { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x) { ConstantNode y = b.add(ConstantNode.forIntegerKind(kind, 1)); - return createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_INCREMENT_EXACT); + createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_INCREMENT_EXACT); + return true; } }); - r.register2("addExact", type, type, new InvocationPlugin() { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x, ValueNode y) { - return createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_ADD_EXACT); + createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_ADD_EXACT); + return true; } }); - r.register2("subtractExact", type, type, new InvocationPlugin() { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x, ValueNode y) { - return createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_SUBTRACT_EXACT); + createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_SUBTRACT_EXACT); + return true; } }); r.register2("multiplyExact", type, type, new InvocationPlugin() { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x, ValueNode y) { - return createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_MULTIPLY_EXACT); - } + createIntegerExactOperation(b, kind, x, y, IntegerExactOp.INTEGER_MULTIPLY_EXACT); + return true; + } }); } } @@ -1100,23 +1162,7 @@ public class StandardGraphBuilderPlugins { } } - private static final class DirectiveSpeculationReason implements SpeculationLog.SpeculationReason { - private final BytecodePosition pos; - - private DirectiveSpeculationReason(BytecodePosition pos) { - this.pos = pos; - } - - @Override - public int hashCode() { - return pos.hashCode(); - } - - @Override - public boolean equals(Object obj) { - return obj instanceof DirectiveSpeculationReason && ((DirectiveSpeculationReason) obj).pos.equals(this.pos); - } - } + private static final SpeculationReasonGroup DIRECTIVE_SPECULATIONS = new SpeculationReasonGroup("GraalDirective", BytecodePosition.class); private static void registerGraalDirectivesPlugins(InvocationPlugins plugins) { Registration r = new Registration(plugins, GraalDirectives.class); @@ -1139,9 +1185,9 @@ public class StandardGraphBuilderPlugins { r.register0("deoptimizeAndInvalidateWithSpeculation", new InvocationPlugin() { @Override public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) { - GraalError.guarantee(b.getGraph().getSpeculationLog() != null, "A speculation log is need to use `deoptimizeAndInvalidateWithSpeculation`"); + GraalError.guarantee(b.getGraph().getSpeculationLog() != null, "A speculation log is needed to use `deoptimizeAndInvalidateWithSpeculation`"); BytecodePosition pos = new BytecodePosition(null, b.getMethod(), b.bci()); - DirectiveSpeculationReason reason = new DirectiveSpeculationReason(pos); + SpeculationReason reason = DIRECTIVE_SPECULATIONS.createSpeculationReason(pos); Speculation speculation; if (b.getGraph().getSpeculationLog().maySpeculate(reason)) { speculation = b.getGraph().getSpeculationLog().speculate(reason); @@ -1324,7 +1370,7 @@ public class StandardGraphBuilderPlugins { b.add(new DeoptimizeNode(DeoptimizationAction.InvalidateReprofile, DeoptimizationReason.TransferToInterpreter)); } else if (falseCount == 0 || trueCount == 0) { boolean expected = falseCount == 0; - LogicNode condition = b.addWithInputs( + LogicNode condition = b.add( IntegerEqualsNode.create(b.getConstantReflection(), b.getMetaAccess(), b.getOptions(), null, result, b.add(ConstantNode.forBoolean(!expected)), NodeView.DEFAULT)); b.append(new FixedGuardNode(condition, DeoptimizationReason.UnreachedCode, DeoptimizationAction.InvalidateReprofile, true)); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StringUTF16Substitutions.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StringUTF16Substitutions.java new file mode 100644 index 00000000000..378c3d69af3 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StringUTF16Substitutions.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.replacements; + +import org.graalvm.compiler.api.replacements.ClassSubstitution; +import org.graalvm.compiler.api.replacements.MethodSubstitution; +import org.graalvm.compiler.nodes.extended.JavaReadNode; +import org.graalvm.compiler.nodes.extended.JavaWriteNode; +import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin; + +// JaCoCo Exclude + +/** + * Substitutions for {@code StringUTF16} methods for JDK9 and later. + */ +@ClassSubstitution(className = "java.lang.StringUTF16", optional = true) +public class StringUTF16Substitutions { + + @MethodSubstitution + public static char getChar(byte[] value, int i) { + ReplacementsUtil.runtimeAssert((i << 1) + 1 < value.length, "Trusted caller missed bounds check"); + return getCharDirect(value, i << 1); + } + + /** + * Will be intrinsified with an {@link InvocationPlugin} to a {@link JavaReadNode}. + */ + public static native char getCharDirect(byte[] value, int i); + + @MethodSubstitution + public static void putChar(byte[] value, int i, int c) { + ReplacementsUtil.runtimeAssert((i << 1) + 1 < value.length, "Trusted caller missed bounds check"); + putCharDirect(value, i << 1, c); + } + + /** + * Will be intrinsified with an {@link InvocationPlugin} to a {@link JavaWriteNode}. + */ + public static native void putCharDirect(byte[] value, int i, int c); +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyCallNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyCallNode.java index bbb1497a837..015987dad4e 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyCallNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyCallNode.java @@ -201,14 +201,19 @@ public final class ArrayCopyCallNode extends AbstractMemoryCheckpoint implements arraycopy(src, srcPos, dest, destPos, length, JavaKind.Object, LocationIdentity.any(), false, false, false, heapWordSize); } - public static void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter JavaKind elementKind, @ConstantNodeParameter int heapWordSize) { - arraycopy(src, srcPos, dest, destPos, length, elementKind, false, false, false, heapWordSize); + public static void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter JavaKind elementKind, @ConstantNodeParameter LocationIdentity locationIdentity, + @ConstantNodeParameter int heapWordSize) { + arraycopy(src, srcPos, dest, destPos, length, elementKind, locationIdentity, false, false, false, heapWordSize); } public static void disjointArraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter JavaKind elementKind, @ConstantNodeParameter int heapWordSize) { arraycopy(src, srcPos, dest, destPos, length, elementKind, false, true, false, heapWordSize); } + public static void disjointArraycopyKillsAny(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter JavaKind elementKind, @ConstantNodeParameter int heapWordSize) { + arraycopy(src, srcPos, dest, destPos, length, elementKind, LocationIdentity.any(), false, true, false, heapWordSize); + } + public static void disjointUninitializedArraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter JavaKind elementKind, @ConstantNodeParameter int heapWordSize) { arraycopy(src, srcPos, dest, destPos, length, elementKind, false, true, true, heapWordSize); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyNode.java index 9cc988304b4..6acf9a32670 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopyNode.java @@ -35,23 +35,30 @@ import org.graalvm.compiler.nodes.spi.LoweringTool; import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode; import jdk.internal.vm.compiler.word.LocationIdentity; -import jdk.vm.ci.meta.JavaKind; - @NodeInfo public final class ArrayCopyNode extends BasicArrayCopyNode implements Lowerable { public static final NodeClass TYPE = NodeClass.create(ArrayCopyNode.class); - private JavaKind elementKind; + protected final boolean forceAnyLocation; public ArrayCopyNode(int bci, ValueNode src, ValueNode srcPos, ValueNode dst, ValueNode dstPos, ValueNode length) { + this(bci, src, srcPos, dst, dstPos, length, false); + } + + public ArrayCopyNode(int bci, ValueNode src, ValueNode srcPos, ValueNode dst, ValueNode dstPos, ValueNode length, boolean forceAnyLocation) { super(TYPE, src, srcPos, dst, dstPos, length, null, bci); - elementKind = ArrayCopySnippets.Templates.selectComponentKind(this); + this.forceAnyLocation = forceAnyLocation; + if (!forceAnyLocation) { + elementKind = ArrayCopySnippets.Templates.selectComponentKind(this); + } else { + assert elementKind == null; + } } @Override public LocationIdentity getLocationIdentity() { - if (elementKind == null) { + if (!forceAnyLocation && elementKind == null) { elementKind = ArrayCopySnippets.Templates.selectComponentKind(this); } if (elementKind != null) { @@ -64,4 +71,8 @@ public final class ArrayCopyNode extends BasicArrayCopyNode implements Lowerable public void lower(LoweringTool tool) { tool.getLowerer().lower(this, tool); } + + public boolean killsAnyLocation() { + return forceAnyLocation; + } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopySnippets.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopySnippets.java index a6e3e3a6b0d..c9f2eb70680 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopySnippets.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopySnippets.java @@ -24,6 +24,7 @@ package org.graalvm.compiler.replacements.arraycopy; +import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY; @@ -34,7 +35,6 @@ import java.util.EnumMap; import jdk.internal.vm.compiler.collections.UnmodifiableEconomicMap; import org.graalvm.compiler.api.directives.GraalDirectives; -import org.graalvm.compiler.api.replacements.Fold; import org.graalvm.compiler.api.replacements.Fold.InjectedParameter; import org.graalvm.compiler.api.replacements.Snippet; import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; @@ -120,8 +120,8 @@ public abstract class ArrayCopySnippets implements Snippets { @Snippet public void arraycopyExactSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, - @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter, - @ConstantParameter Counters counters) { + @ConstantParameter JavaKind elementKind, @ConstantParameter LocationIdentity locationIdentity, + @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter, @ConstantParameter Counters counters) { Object nonNullSrc = GraalDirectives.guardingNonNull(src); Object nonNullDest = GraalDirectives.guardingNonNull(dest); checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); @@ -130,19 +130,19 @@ public abstract class ArrayCopySnippets implements Snippets { elementKindCounter.inc(); elementKindCopiedCounter.add(length); - ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind, heapWordSize()); + ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind, locationIdentity, heapWordSize()); } @Snippet public void arraycopyUnrolledSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, - @ConstantParameter JavaKind elementKind, @ConstantParameter int unrolledLength, @ConstantParameter Counters counters) { + @ConstantParameter JavaKind elementKind, @ConstantParameter LocationIdentity locationIdentity, @ConstantParameter int unrolledLength, @ConstantParameter Counters counters) { Object nonNullSrc = GraalDirectives.guardingNonNull(src); Object nonNullDest = GraalDirectives.guardingNonNull(dest); checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); incrementLengthCounter(length, counters); - unrolledArraycopyWork(nonNullSrc, srcPos, nonNullDest, destPos, unrolledLength, elementKind); + unrolledArraycopyWork(nonNullSrc, srcPos, nonNullDest, destPos, unrolledLength, elementKind, locationIdentity); } @Snippet @@ -179,15 +179,9 @@ public abstract class ArrayCopySnippets implements Snippets { System.arraycopy(src, srcPos, dest, destPos, length); } - @Fold - static LocationIdentity getArrayLocation(JavaKind kind) { - return NamedLocationIdentity.getArrayLocation(kind); - } - - private static void unrolledArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, JavaKind elementKind) { + private static void unrolledArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, JavaKind elementKind, LocationIdentity arrayLocation) { int scale = ReplacementsUtil.arrayIndexScale(INJECTED_META_ACCESS, elementKind); int arrayBaseOffset = ReplacementsUtil.getArrayBaseOffset(INJECTED_META_ACCESS, elementKind); - LocationIdentity arrayLocation = getArrayLocation(elementKind); long sourceOffset = arrayBaseOffset + (long) srcPos * scale; long destOffset = arrayBaseOffset + (long) destPos * scale; @@ -258,7 +252,9 @@ public abstract class ArrayCopySnippets implements Snippets { } private static void incrementLengthCounter(int length, Counters counters) { - counters.lengthHistogram.inc(length); + if (!IS_BUILDING_NATIVE_IMAGE) { + counters.lengthHistogram.inc(length); + } } private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) { @@ -466,13 +462,16 @@ public abstract class ArrayCopySnippets implements Snippets { assert arrayTypeCheck != ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK; args.addConst("arrayTypeCheck", arrayTypeCheck); } + Object locationIdentity = arraycopy.killsAnyLocation() ? LocationIdentity.any() : NamedLocationIdentity.getArrayLocation(elementKind); if (snippetInfo == arraycopyUnrolledSnippet) { args.addConst("elementKind", elementKind != null ? elementKind : JavaKind.Illegal); + args.addConst("locationIdentity", locationIdentity); args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt()); } if (snippetInfo == arraycopyExactSnippet) { assert elementKind != null; args.addConst("elementKind", elementKind); + args.addConst("locationIdentity", locationIdentity); args.addConst("elementKindCounter", counters.arraycopyCallCounters.get(elementKind)); args.addConst("elementKindCopiedCounter", counters.arraycopyCallCopiedCounters.get(elementKind)); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCompareToNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCompareToNode.java index 7e65965544e..d69a5a7e181 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCompareToNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayCompareToNode.java @@ -125,7 +125,7 @@ public final class ArrayCompareToNode extends FixedWithNextNode implements LIRLo @Override public LocationIdentity getLocationIdentity() { - return NamedLocationIdentity.getArrayLocation(kind1); + return kind1 != kind2 ? LocationIdentity.ANY_LOCATION : NamedLocationIdentity.getArrayLocation(kind1); } @Override diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayRegionEqualsNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayRegionEqualsNode.java index d15dc7e597f..96a0d4b5c73 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayRegionEqualsNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/ArrayRegionEqualsNode.java @@ -55,7 +55,8 @@ public final class ArrayRegionEqualsNode extends FixedWithNextNode implements LI public static final NodeClass TYPE = NodeClass.create(ArrayRegionEqualsNode.class); /** {@link JavaKind} of the arrays to compare. */ - private final JavaKind kind; + private final JavaKind kind1; + private final JavaKind kind2; /** Pointer to first array region to be tested for equality. */ @Input private ValueNode array1; @@ -68,16 +69,21 @@ public final class ArrayRegionEqualsNode extends FixedWithNextNode implements LI @OptionalInput(Memory) private MemoryNode lastLocationAccess; - public ArrayRegionEqualsNode(ValueNode array1, ValueNode array2, ValueNode length, @ConstantNodeParameter JavaKind kind) { + public ArrayRegionEqualsNode(ValueNode array1, ValueNode array2, ValueNode length, @ConstantNodeParameter JavaKind kind1, @ConstantNodeParameter JavaKind kind2) { super(TYPE, StampFactory.forKind(JavaKind.Boolean)); - this.kind = kind; + this.kind1 = kind1; + this.kind2 = kind2; this.array1 = array1; this.array2 = array2; this.length = length; } + public static boolean regionEquals(Pointer array1, Pointer array2, int length, @ConstantNodeParameter JavaKind kind) { + return regionEquals(array1, array2, length, kind, kind); + } + @NodeIntrinsic - public static native boolean regionEquals(Pointer array1, Pointer array2, int length, @ConstantNodeParameter JavaKind kind); + public static native boolean regionEquals(Pointer array1, Pointer array2, int length, @ConstantNodeParameter JavaKind kind1, @ConstantNodeParameter JavaKind kind2); @Override public void generate(NodeLIRBuilderTool gen) { @@ -85,13 +91,18 @@ public final class ArrayRegionEqualsNode extends FixedWithNextNode implements LI if (length.isConstant()) { constantLength = length.asJavaConstant().asInt(); } - Value result = gen.getLIRGeneratorTool().emitArrayEquals(kind, gen.operand(array1), gen.operand(array2), gen.operand(length), constantLength, true); + Value result; + if (kind1 == kind2) { + result = gen.getLIRGeneratorTool().emitArrayEquals(kind1, gen.operand(array1), gen.operand(array2), gen.operand(length), constantLength, true); + } else { + result = gen.getLIRGeneratorTool().emitArrayEquals(kind1, kind2, gen.operand(array1), gen.operand(array2), gen.operand(length), constantLength, true); + } gen.setResult(this, result); } @Override public LocationIdentity getLocationIdentity() { - return NamedLocationIdentity.getArrayLocation(kind); + return kind1 != kind2 ? LocationIdentity.ANY_LOCATION : NamedLocationIdentity.getArrayLocation(kind1); } @Override diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BasicArrayCopyNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BasicArrayCopyNode.java index 50e83b61e49..36c508e0168 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BasicArrayCopyNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BasicArrayCopyNode.java @@ -83,15 +83,12 @@ public class BasicArrayCopyNode extends AbstractMemoryCheckpoint implements Virt public BasicArrayCopyNode(NodeClass type, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, JavaKind elementKind, int bci) { super(type, StampFactory.forKind(JavaKind.Void)); this.bci = bci; - args = new NodeInputList<>(this, new ValueNode[]{src, srcPos, dest, destPos, length}); + this.args = new NodeInputList<>(this, new ValueNode[]{src, srcPos, dest, destPos, length}); this.elementKind = elementKind != JavaKind.Illegal ? elementKind : null; } public BasicArrayCopyNode(NodeClass type, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, JavaKind elementKind) { - super(type, StampFactory.forKind(JavaKind.Void)); - this.bci = BytecodeFrame.INVALID_FRAMESTATE_BCI; - args = new NodeInputList<>(this, new ValueNode[]{src, srcPos, dest, destPos, length}); - this.elementKind = elementKind != JavaKind.Illegal ? elementKind : null; + this(type, src, srcPos, dest, destPos, length, elementKind, BytecodeFrame.INVALID_FRAMESTATE_BCI); } public ValueNode getSource() { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BasicObjectCloneNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BasicObjectCloneNode.java index 2ec5b4abad1..ced7c29cf93 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BasicObjectCloneNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BasicObjectCloneNode.java @@ -154,7 +154,7 @@ public abstract class BasicObjectCloneNode extends MacroStateSplitNode implement return; } int constantLength = lengthAlias.asJavaConstant().asInt(); - if (constantLength >= 0 && constantLength < tool.getMaximumEntryCount()) { + if (constantLength >= 0 && constantLength <= tool.getMaximumEntryCount()) { ValueNode[] state = new ValueNode[constantLength]; ResolvedJavaType componentType = type.getComponentType(); for (int i = 0; i < constantLength; i++) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BinaryMathIntrinsicNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BinaryMathIntrinsicNode.java index 1432aa02394..bcf6cefd8c9 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BinaryMathIntrinsicNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/BinaryMathIntrinsicNode.java @@ -26,6 +26,8 @@ package org.graalvm.compiler.replacements.nodes; import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_UNKNOWN; import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_1; + +import jdk.vm.ci.meta.Value; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.core.common.type.FloatStamp; import org.graalvm.compiler.core.common.type.PrimitiveStamp; @@ -46,10 +48,9 @@ import org.graalvm.compiler.nodes.calc.SqrtNode; import org.graalvm.compiler.nodes.spi.ArithmeticLIRLowerable; import org.graalvm.compiler.nodes.spi.Lowerable; import org.graalvm.compiler.nodes.spi.LoweringTool; -import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; import jdk.vm.ci.meta.JavaKind; -import jdk.vm.ci.meta.Value; +import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; @NodeInfo(nameTemplate = "MathIntrinsic#{p#operation/s}", cycles = CYCLES_UNKNOWN, size = SIZE_1) public final class BinaryMathIntrinsicNode extends BinaryNode implements ArithmeticLIRLowerable, Lowerable { @@ -106,6 +107,7 @@ public final class BinaryMathIntrinsicNode extends BinaryNode implements Arithme @Override public void generate(NodeLIRBuilderTool nodeValueMap, ArithmeticLIRGeneratorTool gen) { + // We can only reach here in the math stubs Value xValue = nodeValueMap.operand(getX()); Value yValue = nodeValueMap.operand(getY()); Value result; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/MacroNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/MacroNode.java index 1f5f9dabdd6..1952349deb7 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/MacroNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/MacroNode.java @@ -25,6 +25,7 @@ package org.graalvm.compiler.replacements.nodes; import static jdk.vm.ci.code.BytecodeFrame.isPlaceholderBci; +import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_UNKNOWN; import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_UNKNOWN; @@ -205,13 +206,15 @@ public abstract class MacroNode extends FixedWithNextNode implements Lowerable, if (invoke.stateAfter() == null) { ResolvedJavaMethod method = graph().method(); - if (method.getAnnotation(MethodSubstitution.class) != null || method.getAnnotation(Snippet.class) != null) { - // One cause for this is that a MacroNode is created for a method that - // no longer needs a MacroNode. For example, Class.getComponentType() - // only needs a MacroNode prior to JDK9 as it was given a non-native - // implementation in JDK9. - throw new GraalError("%s macro created for call to %s in %s must be lowerable to a snippet or intrinsic graph. " + - "Maybe a macro node is not needed for this method in the current JDK?", getClass().getSimpleName(), targetMethod.format("%h.%n(%p)"), graph()); + if (!IS_IN_NATIVE_IMAGE) { + if (method.getAnnotation(MethodSubstitution.class) != null || method.getAnnotation(Snippet.class) != null) { + // One cause for this is that a MacroNode is created for a method that + // no longer needs a MacroNode. For example, Class.getComponentType() + // only needs a MacroNode prior to JDK9 as it was given a non-native + // implementation in JDK9. + throw new GraalError("%s macro created for call to %s in %s must be lowerable to a snippet or intrinsic graph. " + + "Maybe a macro node is not needed for this method in the current JDK?", getClass().getSimpleName(), targetMethod.format("%h.%n(%p)"), graph()); + } } throw new GraalError("%s: cannot lower to invoke without state: %s", graph(), this); } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/UnaryMathIntrinsicNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/UnaryMathIntrinsicNode.java index 223222c158b..fb3fa3c7b2e 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/UnaryMathIntrinsicNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/UnaryMathIntrinsicNode.java @@ -27,6 +27,7 @@ package org.graalvm.compiler.replacements.nodes; import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_64; import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_1; +import jdk.vm.ci.meta.Value; import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; import org.graalvm.compiler.core.common.type.FloatStamp; import org.graalvm.compiler.core.common.type.PrimitiveStamp; @@ -44,10 +45,9 @@ import org.graalvm.compiler.nodes.calc.UnaryNode; import org.graalvm.compiler.nodes.spi.ArithmeticLIRLowerable; import org.graalvm.compiler.nodes.spi.Lowerable; import org.graalvm.compiler.nodes.spi.LoweringTool; -import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; import jdk.vm.ci.meta.JavaKind; -import jdk.vm.ci.meta.Value; +import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; @NodeInfo(nameTemplate = "MathIntrinsic#{p#operation/s}", cycles = CYCLES_64, size = SIZE_1) public final class UnaryMathIntrinsicNode extends UnaryNode implements ArithmeticLIRLowerable, Lowerable { @@ -87,6 +87,43 @@ public final class UnaryMathIntrinsicNode extends UnaryNode implements Arithmeti throw new GraalError("unknown op %s", this); } } + + public Stamp computeStamp(Stamp valueStamp) { + if (valueStamp instanceof FloatStamp) { + FloatStamp floatStamp = (FloatStamp) valueStamp; + switch (this) { + case COS: + case SIN: { + boolean nonNaN = floatStamp.lowerBound() != Double.NEGATIVE_INFINITY && floatStamp.upperBound() != Double.POSITIVE_INFINITY && floatStamp.isNonNaN(); + return StampFactory.forFloat(JavaKind.Double, -1.0, 1.0, nonNaN); + } + case TAN: { + boolean nonNaN = floatStamp.lowerBound() != Double.NEGATIVE_INFINITY && floatStamp.upperBound() != Double.POSITIVE_INFINITY && floatStamp.isNonNaN(); + return StampFactory.forFloat(JavaKind.Double, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, nonNaN); + } + case LOG: + case LOG10: { + double lowerBound = compute(floatStamp.lowerBound()); + double upperBound = compute(floatStamp.upperBound()); + if (floatStamp.contains(0.0)) { + // 0.0 and -0.0 infinity produces -Inf + lowerBound = Double.NEGATIVE_INFINITY; + } + boolean nonNaN = floatStamp.lowerBound() >= 0.0 && floatStamp.isNonNaN(); + return StampFactory.forFloat(JavaKind.Double, lowerBound, upperBound, nonNaN); + } + case EXP: { + double lowerBound = Math.exp(floatStamp.lowerBound()); + double upperBound = Math.exp(floatStamp.upperBound()); + boolean nonNaN = floatStamp.isNonNaN(); + return StampFactory.forFloat(JavaKind.Double, lowerBound, upperBound, nonNaN); + } + + } + } + return StampFactory.forKind(JavaKind.Double); + } + } public UnaryOperation getOperation() { @@ -109,50 +146,14 @@ public final class UnaryMathIntrinsicNode extends UnaryNode implements Arithmeti } protected UnaryMathIntrinsicNode(ValueNode value, UnaryOperation op) { - super(TYPE, computeStamp(value.stamp(NodeView.DEFAULT), op), value); + super(TYPE, op.computeStamp(value.stamp(NodeView.DEFAULT)), value); assert value.stamp(NodeView.DEFAULT) instanceof FloatStamp && PrimitiveStamp.getBits(value.stamp(NodeView.DEFAULT)) == 64; this.operation = op; } @Override public Stamp foldStamp(Stamp valueStamp) { - return computeStamp(valueStamp, getOperation()); - } - - static Stamp computeStamp(Stamp valueStamp, UnaryOperation op) { - if (valueStamp instanceof FloatStamp) { - FloatStamp floatStamp = (FloatStamp) valueStamp; - switch (op) { - case COS: - case SIN: { - boolean nonNaN = floatStamp.lowerBound() != Double.NEGATIVE_INFINITY && floatStamp.upperBound() != Double.POSITIVE_INFINITY && floatStamp.isNonNaN(); - return StampFactory.forFloat(JavaKind.Double, -1.0, 1.0, nonNaN); - } - case TAN: { - boolean nonNaN = floatStamp.lowerBound() != Double.NEGATIVE_INFINITY && floatStamp.upperBound() != Double.POSITIVE_INFINITY && floatStamp.isNonNaN(); - return StampFactory.forFloat(JavaKind.Double, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, nonNaN); - } - case LOG: - case LOG10: { - double lowerBound = op.compute(floatStamp.lowerBound()); - double upperBound = op.compute(floatStamp.upperBound()); - if (floatStamp.contains(0.0)) { - // 0.0 and -0.0 infinity produces -Inf - lowerBound = Double.NEGATIVE_INFINITY; - } - boolean nonNaN = floatStamp.lowerBound() >= 0.0 && floatStamp.isNonNaN(); - return StampFactory.forFloat(JavaKind.Double, lowerBound, upperBound, nonNaN); - } - case EXP: { - double lowerBound = Math.exp(floatStamp.lowerBound()); - double upperBound = Math.exp(floatStamp.upperBound()); - boolean nonNaN = floatStamp.isNonNaN(); - return StampFactory.forFloat(JavaKind.Double, lowerBound, upperBound, nonNaN); - } - - } - } - return StampFactory.forKind(JavaKind.Double); + return getOperation().computeStamp(valueStamp); } @Override @@ -162,6 +163,7 @@ public final class UnaryMathIntrinsicNode extends UnaryNode implements Arithmeti @Override public void generate(NodeLIRBuilderTool nodeValueMap, ArithmeticLIRGeneratorTool gen) { + // We can only reach here in the math stubs Value input = nodeValueMap.operand(getValue()); Value result; switch (getOperation()) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerAddExactNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerAddExactNode.java index 7dc3ff315bc..b1008a946b1 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerAddExactNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerAddExactNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -37,35 +37,32 @@ import org.graalvm.compiler.graph.NodeClass; import org.graalvm.compiler.graph.spi.CanonicalizerTool; import org.graalvm.compiler.nodeinfo.InputType; import org.graalvm.compiler.nodeinfo.NodeInfo; -import org.graalvm.compiler.nodes.AbstractBeginNode; import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.calc.AddNode; -import org.graalvm.compiler.nodes.extended.AnchoringNode; -import org.graalvm.compiler.nodes.spi.LoweringTool; import jdk.vm.ci.code.CodeUtil; import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.JavaKind; -import jdk.vm.ci.meta.SpeculationLog.SpeculationReason; +import org.graalvm.compiler.nodes.extended.GuardedNode; +import org.graalvm.compiler.nodes.extended.GuardingNode; /** * Node representing an exact integer addition that will throw an {@link ArithmeticException} in * case the addition would overflow the 32 bit range. */ @NodeInfo(cycles = CYCLES_2, size = SIZE_2) -public final class IntegerAddExactNode extends AddNode implements IntegerExactArithmeticNode { +public final class IntegerAddExactNode extends AddNode implements GuardedNode, IntegerExactArithmeticNode { public static final NodeClass TYPE = NodeClass.create(IntegerAddExactNode.class); - @OptionalInput(InputType.Anchor) protected AnchoringNode anchor; - protected final SpeculationReason speculation; + @Input(InputType.Guard) protected GuardingNode guard; - public IntegerAddExactNode(ValueNode x, ValueNode y, SpeculationReason speculation) { + public IntegerAddExactNode(ValueNode x, ValueNode y, GuardingNode guard) { super(TYPE, x, y); setStamp(x.stamp(NodeView.DEFAULT).unrestricted()); assert x.stamp(NodeView.DEFAULT).isCompatible(y.stamp(NodeView.DEFAULT)) && x.stamp(NodeView.DEFAULT) instanceof IntegerStamp; - this.speculation = speculation; + this.guard = guard; } @Override @@ -129,13 +126,10 @@ public final class IntegerAddExactNode extends AddNode implements IntegerExactAr @Override public ValueNode canonical(CanonicalizerTool tool, ValueNode forX, ValueNode forY) { if (forX.isConstant() && !forY.isConstant()) { - return new IntegerAddExactNode(forY, forX, speculation).canonical(tool); + return new IntegerAddExactNode(forY, forX, guard).canonical(tool); } - if (forX.isConstant()) { - ConstantNode constantNode = canonicalXconstant(forX, forY); - if (constantNode != null) { - return constantNode; - } + if (forX.isConstant() && forY.isConstant()) { + return canonicalXYconstant(forX, forY); } else if (forY.isConstant()) { long c = forY.asJavaConstant().asLong(); if (c == 0) { @@ -148,48 +142,31 @@ public final class IntegerAddExactNode extends AddNode implements IntegerExactAr return this; } - private static ConstantNode canonicalXconstant(ValueNode forX, ValueNode forY) { + private ValueNode canonicalXYconstant(ValueNode forX, ValueNode forY) { JavaConstant xConst = forX.asJavaConstant(); JavaConstant yConst = forY.asJavaConstant(); - if (xConst != null && yConst != null) { - assert xConst.getJavaKind() == yConst.getJavaKind(); - try { - if (xConst.getJavaKind() == JavaKind.Int) { - return ConstantNode.forInt(Math.addExact(xConst.asInt(), yConst.asInt())); - } else { - assert xConst.getJavaKind() == JavaKind.Long; - return ConstantNode.forLong(Math.addExact(xConst.asLong(), yConst.asLong())); - } - } catch (ArithmeticException ex) { - // The operation will result in an overflow exception, so do not canonicalize. + assert xConst.getJavaKind() == yConst.getJavaKind(); + try { + if (xConst.getJavaKind() == JavaKind.Int) { + return ConstantNode.forInt(Math.addExact(xConst.asInt(), yConst.asInt())); + } else { + assert xConst.getJavaKind() == JavaKind.Long; + return ConstantNode.forLong(Math.addExact(xConst.asLong(), yConst.asLong())); } + } catch (ArithmeticException ex) { + // The operation will result in an overflow exception, so do not canonicalize. } - return null; + return this; } @Override - public IntegerExactArithmeticSplitNode createSplit(AbstractBeginNode next, AbstractBeginNode deopt) { - return graph().add(new IntegerAddExactSplitNode(stamp(NodeView.DEFAULT), getX(), getY(), next, deopt)); + public GuardingNode getGuard() { + return guard; } @Override - public SpeculationReason getSpeculation() { - return speculation; - } - - @Override - public AnchoringNode getAnchor() { - return anchor; - } - - @Override - public void setAnchor(AnchoringNode x) { - updateUsagesInterface(this.anchor, x); - this.anchor = x; - } - - @Override - public void lower(LoweringTool tool) { - IntegerExactArithmeticSplitNode.lower(tool, this); + public void setGuard(GuardingNode guard) { + updateUsagesInterface(this.guard, guard); + this.guard = guard; } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerAddExactOverflowNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerAddExactOverflowNode.java new file mode 100644 index 00000000000..9bef4a01be6 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerAddExactOverflowNode.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + + +package org.graalvm.compiler.replacements.nodes.arithmetic; + +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.JavaKind; +import org.graalvm.compiler.core.common.type.IntegerStamp; +import org.graalvm.compiler.core.common.type.Stamp; +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.graph.spi.Canonicalizable.BinaryCommutative; +import org.graalvm.compiler.graph.spi.CanonicalizerTool; +import org.graalvm.compiler.graph.spi.Simplifiable; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.AbstractBeginNode; +import org.graalvm.compiler.nodes.LogicConstantNode; +import org.graalvm.compiler.nodes.NodeView; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.calc.BinaryNode; + +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_2; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_2; + +@NodeInfo(cycles = CYCLES_2, size = SIZE_2) +public final class IntegerAddExactOverflowNode extends IntegerExactOverflowNode implements Simplifiable, BinaryCommutative { + public static final NodeClass TYPE = NodeClass.create(IntegerAddExactOverflowNode.class); + + public IntegerAddExactOverflowNode(ValueNode x, ValueNode y) { + super(TYPE, x, y); + } + + @Override + public ValueNode canonical(CanonicalizerTool tool, ValueNode forX, ValueNode forY) { + if (forX.isConstant() && !forY.isConstant()) { + return new IntegerAddExactOverflowNode(forY, forX).canonical(tool); + } + if (forX.isConstant() && forY.isConstant()) { + return canonicalXYconstant(forX, forY); + } else if (forY.isConstant()) { + long c = forY.asJavaConstant().asLong(); + if (c == 0) { + return LogicConstantNode.forBoolean(false); + } + } + if (!IntegerStamp.addCanOverflow((IntegerStamp) forX.stamp(NodeView.DEFAULT), (IntegerStamp) forY.stamp(NodeView.DEFAULT))) { + return LogicConstantNode.forBoolean(false); + } + return this; + } + + private static LogicConstantNode canonicalXYconstant(ValueNode forX, ValueNode forY) { + JavaConstant xConst = forX.asJavaConstant(); + JavaConstant yConst = forY.asJavaConstant(); + assert xConst.getJavaKind() == yConst.getJavaKind(); + try { + if (xConst.getJavaKind() == JavaKind.Int) { + Math.addExact(xConst.asInt(), yConst.asInt()); + } else { + assert xConst.getJavaKind() == JavaKind.Long; + Math.addExact(xConst.asLong(), yConst.asLong()); + } + } catch (ArithmeticException ex) { + // Always overflows + return LogicConstantNode.forBoolean(true); + } + // Never overflows + return LogicConstantNode.forBoolean(false); + } + + @Override + protected IntegerExactArithmeticSplitNode createSplit(Stamp splitStamp, AbstractBeginNode next, AbstractBeginNode overflow) { + return new IntegerAddExactSplitNode(splitStamp, x, y, next, overflow); + } + + @Override + protected Class getCoupledType() { + return IntegerAddExactNode.class; + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerExactArithmeticNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerExactArithmeticNode.java index 73624b86ca4..d87237d026d 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerExactArithmeticNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerExactArithmeticNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,20 +24,5 @@ package org.graalvm.compiler.replacements.nodes.arithmetic; -import org.graalvm.compiler.nodes.AbstractBeginNode; -import org.graalvm.compiler.nodes.extended.AnchoringNode; -import org.graalvm.compiler.nodes.spi.Lowerable; - -import jdk.vm.ci.meta.SpeculationLog.SpeculationReason; - -public interface IntegerExactArithmeticNode extends Lowerable { - - IntegerExactArithmeticSplitNode createSplit(AbstractBeginNode next, AbstractBeginNode deopt); - - SpeculationReason getSpeculation(); - - AnchoringNode getAnchor(); - - void setAnchor(AnchoringNode x); - +public interface IntegerExactArithmeticNode { } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerExactArithmeticSplitNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerExactArithmeticSplitNode.java index 1fd8831f7c2..184070ef010 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerExactArithmeticSplitNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerExactArithmeticSplitNode.java @@ -32,21 +32,11 @@ import org.graalvm.compiler.graph.NodeClass; import org.graalvm.compiler.graph.spi.Simplifiable; import org.graalvm.compiler.nodeinfo.NodeInfo; import org.graalvm.compiler.nodes.AbstractBeginNode; -import org.graalvm.compiler.nodes.BeginNode; import org.graalvm.compiler.nodes.ControlSplitNode; -import org.graalvm.compiler.nodes.DeoptimizeNode; -import org.graalvm.compiler.nodes.FixedNode; -import org.graalvm.compiler.nodes.FixedWithNextNode; -import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.calc.FloatingNode; import org.graalvm.compiler.nodes.spi.LIRLowerable; -import org.graalvm.compiler.nodes.spi.LoweringTool; import org.graalvm.compiler.nodes.spi.NodeLIRBuilderTool; -import jdk.vm.ci.meta.DeoptimizationAction; -import jdk.vm.ci.meta.DeoptimizationReason; -import jdk.vm.ci.meta.SpeculationLog; import jdk.vm.ci.meta.Value; @NodeInfo(cycles = CYCLES_2, cyclesRationale = "add+cmp", size = SIZE_2) @@ -117,23 +107,6 @@ public abstract class IntegerExactArithmeticSplitNode extends ControlSplitNode i protected abstract Value generateArithmetic(NodeLIRBuilderTool generator); - static void lower(LoweringTool tool, IntegerExactArithmeticNode node) { - if (node.asNode().graph().getGuardsStage() == StructuredGraph.GuardsStage.FIXED_DEOPTS) { - FloatingNode floatingNode = (FloatingNode) node; - FixedWithNextNode previous = tool.lastFixedNode(); - FixedNode next = previous.next(); - previous.setNext(null); - StructuredGraph graph = floatingNode.graph(); - DeoptimizeNode deopt = graph.add(new DeoptimizeNode(DeoptimizationAction.InvalidateReprofile, DeoptimizationReason.ArithmeticException, - node.getSpeculation() == null ? SpeculationLog.NO_SPECULATION : graph.getSpeculationLog().speculate(node.getSpeculation()))); - AbstractBeginNode normalBegin = graph.add(new BeginNode()); - normalBegin.setNext(next); - IntegerExactArithmeticSplitNode split = node.createSplit(normalBegin, BeginNode.begin(deopt)); - previous.setNext(split); - floatingNode.replaceAndDelete(split); - } - } - @Override public int getSuccessorCount() { return 2; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerExactOverflowNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerExactOverflowNode.java new file mode 100644 index 00000000000..22c760625aa --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerExactOverflowNode.java @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + + +package org.graalvm.compiler.replacements.nodes.arithmetic; + +import org.graalvm.compiler.core.common.type.Stamp; +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.graph.spi.Canonicalizable; +import org.graalvm.compiler.graph.spi.Simplifiable; +import org.graalvm.compiler.graph.spi.SimplifierTool; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.AbstractBeginNode; +import org.graalvm.compiler.nodes.IfNode; +import org.graalvm.compiler.nodes.LogicNode; +import org.graalvm.compiler.nodes.NodeView; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.calc.BinaryNode; + +import java.util.List; + +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_2; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_2; + +@NodeInfo(cycles = CYCLES_2, size = SIZE_2) +public abstract class IntegerExactOverflowNode extends LogicNode implements Canonicalizable.Binary, Simplifiable { + public static final NodeClass TYPE = NodeClass.create(IntegerExactOverflowNode.class); + @Input protected ValueNode x; + @Input protected ValueNode y; + + public IntegerExactOverflowNode(NodeClass c, ValueNode x, ValueNode y) { + super(c); + assert x != null && y != null; + this.x = x; + this.y = y; + } + + @Override + public ValueNode getX() { + return x; + } + + @Override + public ValueNode getY() { + return y; + } + + /** + * Make sure the overflow detection nodes have the same order of inputs as the exact arithmetic + * nodes. + * + * @return the original node or another node with the same inputs, ignoring ordering. + */ + @SuppressWarnings("deprecation") + public LogicNode maybeCommuteInputs() { + assert this instanceof BinaryCommutative; + if (!y.isConstant() && (x.isConstant() || x.getId() > y.getId())) { + ValueNode tmp = x; + x = y; + y = tmp; + if (graph() != null) { + // See if this node already exists + LogicNode duplicate = graph().findDuplicate(this); + if (duplicate != null) { + return duplicate; + } + } + } + return this; + } + + protected abstract IntegerExactArithmeticSplitNode createSplit(Stamp splitStamp, AbstractBeginNode next, AbstractBeginNode overflow); + + protected abstract Class getCoupledType(); + + @Override + public void simplify(SimplifierTool tool) { + // Find all ifs that this node feeds into + for (IfNode ifNode : usages().filter(IfNode.class).snapshot()) { + // Replace the if with exact split + AbstractBeginNode next = ifNode.falseSuccessor(); + AbstractBeginNode overflow = ifNode.trueSuccessor(); + ifNode.clearSuccessors(); + + // Try to find corresponding exact nodes that could be combined with the split. They + // would be directly + // linked to the BeginNode of the false branch. + List coupledNodes = next.usages().filter(getCoupledType()).filter(n -> { + BinaryNode exact = (BinaryNode) n; + return exact.getX() == getX() && exact.getY() == getY(); + }).snapshot(); + + Stamp splitStamp = x.stamp(NodeView.DEFAULT).unrestricted(); + if (!coupledNodes.isEmpty()) { + splitStamp = coupledNodes.iterator().next().stamp(NodeView.DEFAULT); + } + IntegerExactArithmeticSplitNode split = graph().add(createSplit(splitStamp, next, overflow)); + ifNode.replaceAndDelete(split); + + coupledNodes.forEach(n -> n.replaceAndDelete(split)); + } + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerMulExactNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerMulExactNode.java index fc7d6a6ab3f..201d450b600 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerMulExactNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerMulExactNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,34 +32,31 @@ import org.graalvm.compiler.graph.NodeClass; import org.graalvm.compiler.graph.spi.CanonicalizerTool; import org.graalvm.compiler.nodeinfo.InputType; import org.graalvm.compiler.nodeinfo.NodeInfo; -import org.graalvm.compiler.nodes.AbstractBeginNode; import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.calc.MulNode; -import org.graalvm.compiler.nodes.extended.AnchoringNode; -import org.graalvm.compiler.nodes.spi.LoweringTool; +import org.graalvm.compiler.nodes.extended.GuardedNode; +import org.graalvm.compiler.nodes.extended.GuardingNode; import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.JavaKind; -import jdk.vm.ci.meta.SpeculationLog.SpeculationReason; /** * Node representing an exact integer multiplication that will throw an {@link ArithmeticException} * in case the addition would overflow the 32 bit range. */ @NodeInfo(cycles = CYCLES_4, cyclesRationale = "mul+cmp", size = SIZE_2) -public final class IntegerMulExactNode extends MulNode implements IntegerExactArithmeticNode { +public final class IntegerMulExactNode extends MulNode implements GuardedNode, IntegerExactArithmeticNode { public static final NodeClass TYPE = NodeClass.create(IntegerMulExactNode.class); - @OptionalInput(InputType.Anchor) protected AnchoringNode anchor; - protected final SpeculationReason speculation; + @Input(InputType.Guard) protected GuardingNode guard; - public IntegerMulExactNode(ValueNode x, ValueNode y, SpeculationReason speculation) { + public IntegerMulExactNode(ValueNode x, ValueNode y, GuardingNode guard) { super(TYPE, x, y); setStamp(x.stamp(NodeView.DEFAULT).unrestricted()); assert x.stamp(NodeView.DEFAULT).isCompatible(y.stamp(NodeView.DEFAULT)) && x.stamp(NodeView.DEFAULT) instanceof IntegerStamp; - this.speculation = speculation; + this.guard = guard; } @Override @@ -76,10 +73,10 @@ public final class IntegerMulExactNode extends MulNode implements IntegerExactAr @Override public ValueNode canonical(CanonicalizerTool tool, ValueNode forX, ValueNode forY) { if (forX.isConstant() && !forY.isConstant()) { - return new IntegerMulExactNode(forY, forX, speculation).canonical(tool); + return new IntegerMulExactNode(forY, forX, guard).canonical(tool); } - if (forX.isConstant()) { - return canonicalXconstant(forX, forY); + if (forX.isConstant() && forY.isConstant()) { + return canonicalXYconstant(forX, forY); } else if (forY.isConstant()) { long c = forY.asJavaConstant().asLong(); if (c == 1) { @@ -95,7 +92,7 @@ public final class IntegerMulExactNode extends MulNode implements IntegerExactAr return this; } - private ValueNode canonicalXconstant(ValueNode forX, ValueNode forY) { + private ValueNode canonicalXYconstant(ValueNode forX, ValueNode forY) { JavaConstant xConst = forX.asJavaConstant(); JavaConstant yConst = forY.asJavaConstant(); assert xConst.getJavaKind() == yConst.getJavaKind(); @@ -113,28 +110,13 @@ public final class IntegerMulExactNode extends MulNode implements IntegerExactAr } @Override - public IntegerExactArithmeticSplitNode createSplit(AbstractBeginNode next, AbstractBeginNode deopt) { - return graph().add(new IntegerMulExactSplitNode(stamp(NodeView.DEFAULT), getX(), getY(), next, deopt)); + public GuardingNode getGuard() { + return guard; } @Override - public SpeculationReason getSpeculation() { - return speculation; - } - - @Override - public AnchoringNode getAnchor() { - return anchor; - } - - @Override - public void setAnchor(AnchoringNode x) { - updateUsagesInterface(this.anchor, x); - this.anchor = x; - } - - @Override - public void lower(LoweringTool tool) { - IntegerExactArithmeticSplitNode.lower(tool, this); + public void setGuard(GuardingNode guard) { + updateUsagesInterface(this.guard, guard); + this.guard = guard; } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerMulExactOverflowNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerMulExactOverflowNode.java new file mode 100644 index 00000000000..97962344664 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerMulExactOverflowNode.java @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + + +package org.graalvm.compiler.replacements.nodes.arithmetic; + +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.JavaKind; +import org.graalvm.compiler.core.common.type.IntegerStamp; +import org.graalvm.compiler.core.common.type.Stamp; +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.graph.spi.Canonicalizable; +import org.graalvm.compiler.graph.spi.CanonicalizerTool; +import org.graalvm.compiler.graph.spi.Simplifiable; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.AbstractBeginNode; +import org.graalvm.compiler.nodes.LogicConstantNode; +import org.graalvm.compiler.nodes.NodeView; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.calc.BinaryNode; + +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_4; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_2; + +@NodeInfo(cycles = CYCLES_4, cyclesRationale = "mul+cmp", size = SIZE_2) +public class IntegerMulExactOverflowNode extends IntegerExactOverflowNode implements Simplifiable, Canonicalizable.BinaryCommutative { + public static final NodeClass TYPE = NodeClass.create(IntegerMulExactOverflowNode.class); + + public IntegerMulExactOverflowNode(ValueNode x, ValueNode y) { + super(TYPE, x, y); + } + + @Override + public ValueNode canonical(CanonicalizerTool tool, ValueNode forX, ValueNode forY) { + if (forX.isConstant() && !forY.isConstant()) { + return new IntegerMulExactOverflowNode(forY, forX).canonical(tool); + } + if (forX.isConstant() && forY.isConstant()) { + return canonicalXYconstant(forX, forY); + } else if (forY.isConstant()) { + long c = forY.asJavaConstant().asLong(); + if (c == 1 || c == 0) { + return LogicConstantNode.forBoolean(false); + } + } + if (!IntegerStamp.multiplicationCanOverflow((IntegerStamp) x.stamp(NodeView.DEFAULT), (IntegerStamp) y.stamp(NodeView.DEFAULT))) { + return LogicConstantNode.forBoolean(false); + } + return this; + } + + private static LogicConstantNode canonicalXYconstant(ValueNode forX, ValueNode forY) { + JavaConstant xConst = forX.asJavaConstant(); + JavaConstant yConst = forY.asJavaConstant(); + assert xConst.getJavaKind() == yConst.getJavaKind(); + try { + if (xConst.getJavaKind() == JavaKind.Int) { + Math.multiplyExact(xConst.asInt(), yConst.asInt()); + } else { + assert xConst.getJavaKind() == JavaKind.Long; + Math.multiplyExact(xConst.asLong(), yConst.asLong()); + } + } catch (ArithmeticException ex) { + return LogicConstantNode.forBoolean(true); + } + return LogicConstantNode.forBoolean(false); + } + + @Override + protected IntegerExactArithmeticSplitNode createSplit(Stamp splitStamp, AbstractBeginNode next, AbstractBeginNode overflow) { + return new IntegerMulExactSplitNode(splitStamp, x, y, next, overflow); + } + + @Override + protected Class getCoupledType() { + return IntegerMulExactNode.class; + } + +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerSubExactNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerSubExactNode.java index 31e3a2af770..e311b700a09 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerSubExactNode.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerSubExactNode.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,35 +32,32 @@ import org.graalvm.compiler.graph.NodeClass; import org.graalvm.compiler.graph.spi.CanonicalizerTool; import org.graalvm.compiler.nodeinfo.InputType; import org.graalvm.compiler.nodeinfo.NodeInfo; -import org.graalvm.compiler.nodes.AbstractBeginNode; import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.calc.SubNode; -import org.graalvm.compiler.nodes.extended.AnchoringNode; -import org.graalvm.compiler.nodes.spi.LoweringTool; +import org.graalvm.compiler.nodes.extended.GuardedNode; +import org.graalvm.compiler.nodes.extended.GuardingNode; import org.graalvm.compiler.nodes.util.GraphUtil; import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.JavaKind; -import jdk.vm.ci.meta.SpeculationLog.SpeculationReason; /** * Node representing an exact integer substraction that will throw an {@link ArithmeticException} in * case the addition would overflow the 32 bit range. */ @NodeInfo(cycles = CYCLES_2, size = SIZE_2) -public final class IntegerSubExactNode extends SubNode implements IntegerExactArithmeticNode { +public final class IntegerSubExactNode extends SubNode implements GuardedNode, IntegerExactArithmeticNode { public static final NodeClass TYPE = NodeClass.create(IntegerSubExactNode.class); - @OptionalInput(InputType.Anchor) protected AnchoringNode anchor; - protected final SpeculationReason speculation; + @Input(InputType.Guard) protected GuardingNode guard; - public IntegerSubExactNode(ValueNode x, ValueNode y, SpeculationReason speculation) { + public IntegerSubExactNode(ValueNode x, ValueNode y, GuardingNode guard) { super(TYPE, x, y); setStamp(x.stamp(NodeView.DEFAULT).unrestricted()); assert x.stamp(NodeView.DEFAULT).isCompatible(y.stamp(NodeView.DEFAULT)) && x.stamp(NodeView.DEFAULT) instanceof IntegerStamp; - this.speculation = speculation; + this.guard = guard; } @Override @@ -111,28 +108,13 @@ public final class IntegerSubExactNode extends SubNode implements IntegerExactAr } @Override - public IntegerExactArithmeticSplitNode createSplit(AbstractBeginNode next, AbstractBeginNode deopt) { - return graph().add(new IntegerSubExactSplitNode(stamp(NodeView.DEFAULT), getX(), getY(), next, deopt)); + public GuardingNode getGuard() { + return guard; } @Override - public SpeculationReason getSpeculation() { - return speculation; - } - - @Override - public AnchoringNode getAnchor() { - return anchor; - } - - @Override - public void setAnchor(AnchoringNode x) { - updateUsagesInterface(this.anchor, x); - this.anchor = x; - } - - @Override - public void lower(LoweringTool tool) { - IntegerExactArithmeticSplitNode.lower(tool, this); + public void setGuard(GuardingNode guard) { + updateUsagesInterface(this.guard, guard); + this.guard = guard; } } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerSubExactOverflowNode.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerSubExactOverflowNode.java new file mode 100644 index 00000000000..4e022431f3c --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/nodes/arithmetic/IntegerSubExactOverflowNode.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + + +package org.graalvm.compiler.replacements.nodes.arithmetic; + +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.JavaKind; +import org.graalvm.compiler.core.common.type.IntegerStamp; +import org.graalvm.compiler.core.common.type.Stamp; +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.graph.spi.CanonicalizerTool; +import org.graalvm.compiler.graph.spi.Simplifiable; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.AbstractBeginNode; +import org.graalvm.compiler.nodes.LogicConstantNode; +import org.graalvm.compiler.nodes.NodeView; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.calc.BinaryNode; +import org.graalvm.compiler.nodes.util.GraphUtil; + +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_2; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_2; + +@NodeInfo(cycles = CYCLES_2, size = SIZE_2) +public class IntegerSubExactOverflowNode extends IntegerExactOverflowNode implements Simplifiable { + public static final NodeClass TYPE = NodeClass.create(IntegerSubExactOverflowNode.class); + + public IntegerSubExactOverflowNode(ValueNode x, ValueNode y) { + super(TYPE, x, y); + } + + @Override + public ValueNode canonical(CanonicalizerTool tool, ValueNode forX, ValueNode forY) { + if (GraphUtil.unproxify(forX) == GraphUtil.unproxify(forY)) { + return LogicConstantNode.forBoolean(false); + } + if (forX.isConstant() && forY.isConstant()) { + return canonicalXYconstant(forX, forY); + } else if (forY.isConstant()) { + long c = forY.asJavaConstant().asLong(); + if (c == 0) { + return LogicConstantNode.forBoolean(false); + } + } + if (!IntegerStamp.subtractionCanOverflow((IntegerStamp) x.stamp(NodeView.DEFAULT), (IntegerStamp) y.stamp(NodeView.DEFAULT))) { + return LogicConstantNode.forBoolean(false); + } + return this; + } + + private static LogicConstantNode canonicalXYconstant(ValueNode forX, ValueNode forY) { + JavaConstant xConst = forX.asJavaConstant(); + JavaConstant yConst = forY.asJavaConstant(); + assert xConst.getJavaKind() == yConst.getJavaKind(); + try { + if (xConst.getJavaKind() == JavaKind.Int) { + Math.subtractExact(xConst.asInt(), yConst.asInt()); + } else { + assert xConst.getJavaKind() == JavaKind.Long; + Math.subtractExact(xConst.asLong(), yConst.asLong()); + } + } catch (ArithmeticException ex) { + // Always overflows + return LogicConstantNode.forBoolean(true); + } + // Never overflows + return LogicConstantNode.forBoolean(false); + } + + @Override + protected IntegerExactArithmeticSplitNode createSplit(Stamp splitStamp, AbstractBeginNode next, AbstractBeginNode overflow) { + return new IntegerSubExactSplitNode(splitStamp, x, y, next, overflow); + } + + @Override + protected Class getCoupledType() { + return IntegerSubExactNode.class; + } + +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/GraalServices.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/GraalServices.java index d9eedbe380b..2748019372c 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/GraalServices.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/GraalServices.java @@ -28,12 +28,14 @@ import static java.lang.Thread.currentThread; import java.io.IOException; import java.io.InputStream; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.ServiceConfigurationError; import java.util.ServiceLoader; import java.util.concurrent.atomic.AtomicLong; +import jdk.vm.ci.meta.SpeculationLog.SpeculationReason; import jdk.vm.ci.runtime.JVMCI; import jdk.vm.ci.services.JVMCIPermission; import jdk.vm.ci.services.Services; @@ -43,30 +45,6 @@ import jdk.vm.ci.services.Services; */ public final class GraalServices { - private static int getJavaSpecificationVersion() { - String value = System.getProperty("java.specification.version"); - if (value.startsWith("1.")) { - value = value.substring(2); - } - return Integer.parseInt(value); - } - - /** - * The integer value corresponding to the value of the {@code java.specification.version} system - * property after any leading {@code "1."} has been stripped. - */ - public static final int JAVA_SPECIFICATION_VERSION = getJavaSpecificationVersion(); - - /** - * Determines if the Java runtime is version 8 or earlier. - */ - public static final boolean Java8OrEarlier = JAVA_SPECIFICATION_VERSION <= 8; - - /** - * Determines if the Java runtime is version 11 or earlier. - */ - public static final boolean Java11OrEarlier = JAVA_SPECIFICATION_VERSION <= 11; - private GraalServices() { } @@ -194,6 +172,44 @@ public final class GraalServices { return false; } + /** + * An implementation of {@link SpeculationReason} based on direct, unencoded values. + */ + static final class DirectSpeculationReason implements SpeculationReason { + final int groupId; + final String groupName; + final Object[] context; + + DirectSpeculationReason(int groupId, String groupName, Object[] context) { + this.groupId = groupId; + this.groupName = groupName; + this.context = context; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof DirectSpeculationReason) { + DirectSpeculationReason that = (DirectSpeculationReason) obj; + return this.groupId == that.groupId && Arrays.equals(this.context, that.context); + } + return false; + } + + @Override + public int hashCode() { + return groupId + Arrays.hashCode(this.context); + } + + @Override + public String toString() { + return String.format("%s@%d%s", groupName, groupId, Arrays.toString(context)); + } + } + + static SpeculationReason createSpeculationReason(int groupId, String groupName, Object... context) { + return new DirectSpeculationReason(groupId, groupName, context); + } + /** * Gets a unique identifier for this execution such as a process ID or a * {@linkplain #getGlobalTimeStamp() fixed timestamp}. diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/JavaVersionUtil.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/JavaVersionUtil.java new file mode 100644 index 00000000000..eab36f828db --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/JavaVersionUtil.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.serviceprovider; + +/** + * Interface to query which JDK version Graal is running on. + */ +public final class JavaVersionUtil { + + private static int getJavaSpecificationVersion() { + String value = System.getProperty("java.specification.version"); + if (value.startsWith("1.")) { + value = value.substring(2); + } + return Integer.parseInt(value); + } + + /** + * The integer value corresponding to the value of the {@code java.specification.version} system + * property after any leading {@code "1."} has been stripped. + */ + public static final int JAVA_SPECIFICATION_VERSION = getJavaSpecificationVersion(); + + /** + * Determines if the Java runtime is version 8 or earlier. + */ + public static final boolean Java8OrEarlier = JAVA_SPECIFICATION_VERSION <= 8; + + /** + * Determines if the Java runtime is version 11 or earlier. + */ + public static final boolean Java11OrEarlier = JAVA_SPECIFICATION_VERSION <= 11; + + private JavaVersionUtil() { + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/SpeculationReasonGroup.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/SpeculationReasonGroup.java new file mode 100644 index 00000000000..c30f3ad9f7f --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.serviceprovider/src/org/graalvm/compiler/serviceprovider/SpeculationReasonGroup.java @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.compiler.serviceprovider; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import jdk.vm.ci.code.BytecodePosition; +import jdk.vm.ci.meta.ResolvedJavaMethod; +import jdk.vm.ci.meta.ResolvedJavaType; +import jdk.vm.ci.meta.SpeculationLog.SpeculationReason; + +/** + * Facility for creating speculation reasons partitioned in groups. + */ +public final class SpeculationReasonGroup { + + private final int id; + private final String name; + private final Class[] signature; + + private static final AtomicInteger nextId = new AtomicInteger(1); + + /** + * Creates speculation group whose context will always match {@code signature}. + */ + public SpeculationReasonGroup(String name, Class... signature) { + this.id = nextId.get(); + this.name = name; + this.signature = signature; + for (Class c : signature) { + if (!isOfSupportedType(c)) { + throw new IllegalArgumentException("Unsupported speculation context type: " + c.getName()); + } + } + } + + @Override + public String toString() { + return String.format("%s{id:%d, sig=%s}", name, id, Arrays.toString(signature)); + } + + /** + * Creates a speculation reason described by this group. + * + * @param context the components of the reason instance being created + */ + public SpeculationReason createSpeculationReason(Object... context) { + assert checkSignature(context); + return GraalServices.createSpeculationReason(id, name, context); + } + + private static final Set> SUPPORTED_EXACT_TYPES = new HashSet<>(Arrays.asList( + String.class, + int.class, + long.class, + float.class, + double.class, + BytecodePosition.class)); + + private static boolean isOfSupportedType(Class c) { + if (SUPPORTED_EXACT_TYPES.contains(c)) { + return true; + } + if (Enum.class.isAssignableFrom(c)) { + // Trust the ordinal of an Enum to be unique + return true; + } + if (SpeculationContextObject.class.isAssignableFrom(c)) { + return true; + } + if (ResolvedJavaMethod.class.isAssignableFrom(c) || ResolvedJavaType.class.isAssignableFrom(c)) { + // Only the JVMCI implementation specific concrete subclasses + // of these types will be accepted but we cannot test for that + // here since we are in JVMCI implementation agnostic code. + return true; + } + return false; + } + + static Class toBox(Class c) { + if (c == int.class) { + return Integer.class; + } + if (c == long.class) { + return Long.class; + } + if (c == float.class) { + return Float.class; + } + if (c == double.class) { + return Double.class; + } + return c; + } + + private boolean checkSignature(Object[] context) { + assert signature.length == context.length : name + ": Incorrect number of context arguments. Expected " + signature.length + ", got " + context.length; + for (int i = 0; i < context.length; i++) { + Object o = context[i]; + Class c = signature[i]; + if (o != null) { + if (c == ResolvedJavaMethod.class || c == ResolvedJavaType.class || SpeculationContextObject.class.isAssignableFrom(c)) { + c.cast(o); + } else { + Class oClass = o.getClass(); + assert toBox(c) == oClass : name + ": Context argument " + i + " is not a " + c.getName() + " but a " + oClass.getName(); + } + } else { + if (c.isPrimitive() || Enum.class.isAssignableFrom(c)) { + throw new AssertionError(name + ": Cannot pass null for argument " + i); + } + } + } + return true; + } + + /** + * Denotes part of a {@linkplain SpeculationReasonGroup#createSpeculationReason(Object...) + * reason} that can have its attributes {@linkplain #accept(Visitor) visited}. + */ + public interface SpeculationContextObject { + void accept(Visitor v); + + public interface Visitor { + void visitBoolean(boolean v); + + void visitByte(byte v); + + void visitChar(char v); + + void visitShort(short v); + + void visitInt(int v); + + void visitLong(long v); + + void visitFloat(float v); + + void visitDouble(double v); + + void visitObject(Object v); + } + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.test/src/org/graalvm/compiler/test/GraalTest.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.test/src/org/graalvm/compiler/test/GraalTest.java index ce4157f3f07..621c34d601a 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.test/src/org/graalvm/compiler/test/GraalTest.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.test/src/org/graalvm/compiler/test/GraalTest.java @@ -44,6 +44,7 @@ import org.graalvm.compiler.debug.DebugHandlersFactory; import org.graalvm.compiler.debug.GlobalMetrics; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.serviceprovider.GraalServices; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import org.junit.After; import org.junit.Assert; import org.junit.AssumptionViolatedException; @@ -72,8 +73,8 @@ public class GraalTest { } } - public static final boolean Java8OrEarlier = GraalServices.Java8OrEarlier; - public static final boolean Java11OrEarlier = GraalServices.Java11OrEarlier; + public static final boolean Java8OrEarlier = JavaVersionUtil.Java8OrEarlier; + public static final boolean Java11OrEarlier = JavaVersionUtil.Java11OrEarlier; protected Method getMethod(String methodName) { return getMethod(getClass(), methodName); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.test/src/org/graalvm/compiler/test/SubprocessUtil.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.test/src/org/graalvm/compiler/test/SubprocessUtil.java index 9d0a4d18696..7b331a09355 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.test/src/org/graalvm/compiler/test/SubprocessUtil.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.test/src/org/graalvm/compiler/test/SubprocessUtil.java @@ -37,7 +37,7 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.graalvm.compiler.serviceprovider.GraalServices; +import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import org.graalvm.util.CollectionsUtil; import org.junit.Assume; @@ -249,7 +249,7 @@ public final class SubprocessUtil { return new Subprocess(command, process.waitFor(), output); } - private static final boolean isJava8OrEarlier = GraalServices.Java8OrEarlier; + private static final boolean isJava8OrEarlier = JavaVersionUtil.Java8OrEarlier; private static boolean hasArg(String optionName) { if (optionName.equals("-cp") || optionName.equals("-classpath")) { diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.virtual/src/org/graalvm/compiler/virtual/phases/ea/EffectsClosure.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.virtual/src/org/graalvm/compiler/virtual/phases/ea/EffectsClosure.java index 49bbf8631e3..2bad29716c0 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.virtual/src/org/graalvm/compiler/virtual/phases/ea/EffectsClosure.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.virtual/src/org/graalvm/compiler/virtual/phases/ea/EffectsClosure.java @@ -295,7 +295,7 @@ public abstract class EffectsClosure> e protected final List processLoop(Loop loop, BlockT initialState) { if (initialState.isDead()) { ArrayList states = new ArrayList<>(); - for (int i = 0; i < loop.getExits().size(); i++) { + for (int i = 0; i < loop.getLoopExits().size(); i++) { states.add(initialState); } return states; @@ -347,7 +347,7 @@ public abstract class EffectsClosure> e blockEffects.get(loop.getHeader()).insertAll(mergeProcessor.mergeEffects, 0); loopMergeEffects.put(loop, mergeProcessor.afterMergeEffects); - assert info.exitStates.size() == loop.getExits().size(); + assert info.exitStates.size() == loop.getLoopExits().size(); loopEntryStates.put((LoopBeginNode) loop.getHeader().getBeginNode(), loopEntryState); assert assertExitStatesNonEmpty(loop, info); @@ -412,8 +412,8 @@ public abstract class EffectsClosure> e } private boolean assertExitStatesNonEmpty(Loop loop, LoopInfo info) { - for (int i = 0; i < loop.getExits().size(); i++) { - assert info.exitStates.get(i) != null : "no loop exit state at " + loop.getExits().get(i) + " / " + loop.getHeader(); + for (int i = 0; i < loop.getLoopExits().size(); i++) { + assert info.exitStates.get(i) != null : "no loop exit state at " + loop.getLoopExits().get(i) + " / " + loop.getHeader(); } return true; } diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.graphio/src/org/graalvm/graphio/GraphOutput.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.graphio/src/org/graalvm/graphio/GraphOutput.java index 69e0f3cefc8..057e52577bd 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.graphio/src/org/graalvm/graphio/GraphOutput.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.graphio/src/org/graalvm/graphio/GraphOutput.java @@ -28,6 +28,7 @@ import java.io.Closeable; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.nio.ByteBuffer; import java.nio.channels.WritableByteChannel; import java.util.Collections; import java.util.Map; @@ -37,8 +38,9 @@ import java.util.Map; * * @param the type of graph this instance handles * @param the type of methods this instance handles + * @since 1.0 a {@link WritableByteChannel} is implemented */ -public final class GraphOutput implements Closeable { +public final class GraphOutput implements Closeable, WritableByteChannel { private final GraphProtocol printer; private GraphOutput(GraphProtocol p) { @@ -106,6 +108,30 @@ public final class GraphOutput implements Closeable { printer.close(); } + /** + * Checks if the {@link GraphOutput} is open. + * + * @return true if the {@link GraphOutput} is open. + * @since 1.0 + */ + @Override + public boolean isOpen() { + return printer.isOpen(); + } + + /** + * Writes raw bytes into {@link GraphOutput}. + * + * @param src the bytes to write + * @return the number of bytes written, possibly zero + * @throws IOException in case of IO error + * @since 1.0 + */ + @Override + public int write(ByteBuffer src) throws IOException { + return printer.write(src); + } + /** * Builder to configure and create an instance of {@link GraphOutput}. * @@ -121,6 +147,7 @@ public final class GraphOutput implements Closeable { private GraphBlocks blocks = DefaultGraphBlocks.empty(); private int major = 4; private int minor = 0; + private boolean embeddedGraphOutput; Builder(GraphStructure structure) { this.structure = structure; @@ -142,6 +169,22 @@ public final class GraphOutput implements Closeable { return this; } + /** + * Sets {@link GraphOutput} as embedded. The embedded {@link GraphOutput} shares + * {@link WritableByteChannel channel} with another already open non parent + * {@link GraphOutput}. The embedded {@link GraphOutput} flushes data after each + * {@link GraphOutput#print print}, {@link GraphOutput#beginGroup beginGroup} and + * {@link GraphOutput#endGroup endGroup} call. + * + * @param embedded if {@code true} the builder creates an embedded {@link GraphOutput} + * @return this builder + * @since 1.0 + */ + public Builder embedded(boolean embedded) { + this.embeddedGraphOutput = embedded; + return this; + } + /** * Associates different implementation of types. * @@ -226,7 +269,7 @@ public final class GraphOutput implements Closeable { private GraphOutput buildImpl(ElementsAndLocations e, WritableByteChannel channel) throws IOException { // @formatter:off ProtocolImpl p = new ProtocolImpl<>( - major, minor, structure, types, blocks, + major, minor, embeddedGraphOutput, structure, types, blocks, e == null ? null : e.elements, e == null ? null : e.locations, channel ); diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.graphio/src/org/graalvm/graphio/GraphProtocol.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.graphio/src/org/graalvm/graphio/GraphProtocol.java index 41625633b7f..56eb519e4ed 100644 --- a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.graphio/src/org/graalvm/graphio/GraphProtocol.java +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.graphio/src/org/graalvm/graphio/GraphProtocol.java @@ -78,10 +78,12 @@ abstract class GraphProtocol 6 || (major == 6 && minor > 0)) { throw new IllegalArgumentException("Unrecognized version " + major + "." + minor); } @@ -90,7 +92,11 @@ abstract class GraphProtocol parent) { @@ -99,36 +105,67 @@ abstract class GraphProtocol properties, int id, String format, Object... args) throws IOException { - writeByte(BEGIN_GRAPH); - if (versionMajor >= 3) { - writeInt(id); - writeString(format); - writeInt(args.length); - for (Object a : args) { - writePropertyObject(graph, a); + printing = true; + try { + writeByte(BEGIN_GRAPH); + if (versionMajor >= 3) { + writeInt(id); + writeString(format); + writeInt(args.length); + for (Object a : args) { + writePropertyObject(graph, a); + } + } else { + writePoolObject(formatTitle(graph, id, format, args)); } - } else { - writePoolObject(formatTitle(graph, id, format, args)); + writeGraph(graph, properties); + flushEmbedded(); + flush(); + } finally { + printing = false; } - writeGraph(graph, properties); - flush(); } public final void beginGroup(Graph noGraph, String name, String shortName, ResolvedJavaMethod method, int bci, Map properties) throws IOException { - writeByte(BEGIN_GROUP); - writePoolObject(name); - writePoolObject(shortName); - writePoolObject(method); - writeInt(bci); - writeProperties(noGraph, properties); + printing = true; + try { + writeByte(BEGIN_GROUP); + writePoolObject(name); + writePoolObject(shortName); + writePoolObject(method); + writeInt(bci); + writeProperties(noGraph, properties); + flushEmbedded(); + } finally { + printing = false; + } } public final void endGroup() throws IOException { - writeByte(CLOSE_GROUP); + printing = true; + try { + writeByte(CLOSE_GROUP); + flushEmbedded(); + } finally { + printing = false; + } + } + + final int write(ByteBuffer src) throws IOException { + if (printing) { + throw new IllegalStateException("Trying to write during graph print."); + } + constantPool.reset(); + return writeBytesRaw(src); + } + + final boolean isOpen() { + return channel.isOpen(); } @Override @@ -280,6 +317,13 @@ abstract class GraphProtocol elements; private final GraphLocations locations; - ProtocolImpl(int major, int minor, GraphStructure structure, GraphTypes enums, GraphBlocks blocks, + ProtocolImpl(int major, int minor, boolean embedded, GraphStructure structure, GraphTypes enums, GraphBlocks blocks, GraphElements elements, GraphLocations locs, WritableByteChannel channel) throws IOException { - super(channel, major, minor); + super(channel, major, minor, embedded); this.structure = structure; this.types = enums; this.blocks = blocks; diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.util/src/org/graalvm/util/DirectAnnotationAccess.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.util/src/org/graalvm/util/DirectAnnotationAccess.java new file mode 100644 index 00000000000..890dfb6c247 --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.util/src/org/graalvm/util/DirectAnnotationAccess.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.util; + +//Checkstyle: allow reflection +import java.lang.annotation.Annotation; +import java.lang.reflect.AnnotatedElement; + +/** + * Wrapper class for annotation access. The purpose of this class is to encapsulate the + * AnnotatedElement.getAnnotation() to avoid the use of the "Checkstyle: allow direct annotation + * access " and "Checkstyle: disallow direct annotation access" comments for situations where the + * annotation access doesn't need to guarded, i.e., in runtime code or code that accesses annotation + * on non-user types. See {@link GuardedAnnotationAccess} for details on these checkstyle rules. + */ +public class DirectAnnotationAccess { + + public static boolean isAnnotationPresent(AnnotatedElement element, Class annotationClass) { + return element.getAnnotation(annotationClass) != null; + } + + public static T getAnnotation(AnnotatedElement element, Class annotationType) { + return element.getAnnotation(annotationType); + } +} diff --git a/src/jdk.internal.vm.compiler/share/classes/org.graalvm.util/src/org/graalvm/util/GuardedAnnotationAccess.java b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.util/src/org/graalvm/util/GuardedAnnotationAccess.java new file mode 100644 index 00000000000..ee5489d050b --- /dev/null +++ b/src/jdk.internal.vm.compiler/share/classes/org.graalvm.util/src/org/graalvm/util/GuardedAnnotationAccess.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +package org.graalvm.util; + +import java.lang.annotation.Annotation; +import java.lang.reflect.AnnotatedElement; + +/** + * Wrapper class for annotation access that defends against + * https://bugs.openjdk.java.net/browse/JDK-7183985: when an annotation declares a Class array + * parameter and one of the referenced classes is not present on the classpath parsing the + * annotations will result in an ArrayStoreException instead of caching of a + * TypeNotPresentExceptionProxy. This is a problem in JDK8 but was fixed in JDK11+. This wrapper + * class also defends against incomplete class path issues. If the element for which annotations are + * queried is a JMVCI value, i.e., a HotSpotResolvedJavaField, or HotSpotResolvedJavaMethod, the + * annotations are read via HotSpotJDKReflection using the + * getFieldAnnotation()/getMethodAnnotation() methods which first construct the field/method object + * via CompilerToVM.asReflectionField()/CompilerToVM.asReflectionExecutable() which eagerly try to + * resolve the types referenced in the element signature. If a field declared type or a method + * return type is missing then JVMCI throws a NoClassDefFoundError. + */ +public final class GuardedAnnotationAccess { + + public static boolean isAnnotationPresent(AnnotatedElement element, Class annotationClass) { + return getAnnotation(element, annotationClass) != null; + } + + public static T getAnnotation(AnnotatedElement element, Class annotationType) { + try { + return element.getAnnotation(annotationType); + } catch (ArrayStoreException | NoClassDefFoundError e) { + /* + * Returning null essentially means that the element doesn't declare the annotationType, + * but we cannot know that since the annotation parsing failed. However, this allows us + * to defend against crashing the image builder if the above JDK bug is encountered in + * user code or if the user code references types missing from the classpath. + */ + return null; + } + } + + public static Annotation[] getAnnotations(AnnotatedElement element) { + try { + return element.getAnnotations(); + } catch (ArrayStoreException | NoClassDefFoundError e) { + /* + * Returning an empty array essentially means that the element doesn't declare any + * annotations, but we know that it is not true since the reason the annotation parsing + * failed is because some annotation referenced a missing class. However, this allows us + * to defend against crashing the image builder if the above JDK bug is encountered in + * user code or if the user code references types missing from the classpath. + */ + return new Annotation[0]; + } + } + + public static T getDeclaredAnnotation(AnnotatedElement element, Class annotationType) { + try { + return element.getDeclaredAnnotation(annotationType); + } catch (ArrayStoreException | NoClassDefFoundError e) { + /* + * Returning null essentially means that the element doesn't declare the annotationType, + * but we cannot know that since the annotation parsing failed. However, this allows us + * to defend against crashing the image builder if the above JDK bug is encountered in + * user code or if the user code references types missing from the classpath. + */ + return null; + } + } + + public static Annotation[] getDeclaredAnnotations(AnnotatedElement element) { + try { + return element.getDeclaredAnnotations(); + } catch (ArrayStoreException | NoClassDefFoundError e) { + /* + * Returning an empty array essentially means that the element doesn't declare any + * annotations, but we know that it is not true since the reason the annotation parsing + * failed is because it at least one annotation referenced a missing class. However, + * this allows us to defend against crashing the image builder if the above JDK bug is + * encountered in user code or if the user code references types missing from the + * classpath. + */ + return new Annotation[0]; + } + } +} diff --git a/test/hotspot/jtreg/ProblemList-graal.txt b/test/hotspot/jtreg/ProblemList-graal.txt index 57ff1c1d6cc..24e6ab201d1 100644 --- a/test/hotspot/jtreg/ProblemList-graal.txt +++ b/test/hotspot/jtreg/ProblemList-graal.txt @@ -226,4 +226,3 @@ org.graalvm.compiler.core.test.deopt.CompiledMethodTest 8202955 org.graalvm.compiler.hotspot.test.ReservedStackAccessTest 8213567 windows-all -org.graalvm.compiler.hotspot.test.CheckGraalIntrinsics 8218074 diff --git a/test/hotspot/jtreg/compiler/graalunit/EA9Test.java b/test/hotspot/jtreg/compiler/graalunit/EA9Test.java new file mode 100644 index 00000000000..03b4a73dcaa --- /dev/null +++ b/test/hotspot/jtreg/compiler/graalunit/EA9Test.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @summary + * @requires vm.opt.final.EnableJVMCI == true + * + * @modules jdk.internal.vm.compiler + * + * @library /test/lib /compiler/graalunit / + * + * @build compiler.graalunit.common.GraalUnitTestLauncher + * + * @run driver jdk.test.lib.FileInstaller ../../ProblemList-graal.txt ExcludeList.txt + * + * @run main/othervm compiler.graalunit.common.GraalUnitTestLauncher -prefix org.graalvm.compiler.core.jdk9.test.ea -exclude ExcludeList.txt + */ diff --git a/test/hotspot/jtreg/compiler/graalunit/HotspotJdk9Test.java b/test/hotspot/jtreg/compiler/graalunit/HotspotJdk9Test.java new file mode 100644 index 00000000000..19e2861eae7 --- /dev/null +++ b/test/hotspot/jtreg/compiler/graalunit/HotspotJdk9Test.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @summary + * @requires vm.opt.final.EnableJVMCI == true + * + * @modules jdk.internal.vm.compiler + * + * @library /test/lib /compiler/graalunit / + * + * @build compiler.graalunit.common.GraalUnitTestLauncher + * + * @run driver jdk.test.lib.FileInstaller ../../ProblemList-graal.txt ExcludeList.txt + * + * @run main/othervm compiler.graalunit.common.GraalUnitTestLauncher -prefix org.graalvm.compiler.hotspot.jdk9.test -exclude ExcludeList.txt + */ diff --git a/test/hotspot/jtreg/compiler/graalunit/TestPackages.txt b/test/hotspot/jtreg/compiler/graalunit/TestPackages.txt index d4e99587faa..c39c36148f1 100644 --- a/test/hotspot/jtreg/compiler/graalunit/TestPackages.txt +++ b/test/hotspot/jtreg/compiler/graalunit/TestPackages.txt @@ -11,6 +11,7 @@ EA9 org.graalvm.compiler.core.jdk9.test.ea Debug org.graalvm.compiler.debug.test Graph org.graalvm.compiler.graph.test @requires vm.graal.enabled HotspotAmd64 org.graalvm.compiler.hotspot.amd64.test +HotspotJdk9 org.graalvm.compiler.hotspot.jdk9.test HotspotSparc org.graalvm.compiler.hotspot.sparc.test @requires vm.simpleArch == "sparcv9" HotspotLir org.graalvm.compiler.hotspot.lir.test Hotspot org.graalvm.compiler.hotspot.test