From 526dba1a2942e444bf11d03d8eaf014b5ef20ccf Mon Sep 17 00:00:00 2001 From: Emanuel Peter Date: Wed, 28 Jun 2023 05:52:26 +0000 Subject: [PATCH] 8310130: C2: assert(false) failed: scalar_input is neither phi nor a matchin reduction Reviewed-by: kvn, chagedorn --- src/hotspot/share/opto/loopopts.cpp | 8 +- .../superword/TestUnorderedReduction.java | 44 ++++++++- ...norderedReductionPartialVectorization.java | 97 +++++++++++++++++++ 3 files changed, 145 insertions(+), 4 deletions(-) create mode 100644 test/hotspot/jtreg/compiler/loopopts/superword/TestUnorderedReductionPartialVectorization.java diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index acdaf578a8a..e9cbdb22457 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -4224,7 +4224,8 @@ void PhaseIdealLoop::move_unordered_reduction_out_of_loop(IdealLoopTree* loop) { if (use != phi && ctrl_or_self(use) == cl) { DEBUG_ONLY( current->dump(-1); ) assert(false, "reduction has use inside loop"); - break; // Chain traversal fails. + // Should not be allowed by SuperWord::mark_reductions + return; // bail out of optimization } } } else { @@ -4245,8 +4246,9 @@ void PhaseIdealLoop::move_unordered_reduction_out_of_loop(IdealLoopTree* loop) { current = nullptr; break; // Success. } else { - DEBUG_ONLY( current->dump(1); ) - assert(false, "scalar_input is neither phi nor a matchin reduction"); + // scalar_input is neither phi nor a matching reduction + // Can for example be scalar reduction when we have + // partial vectorization. break; // Chain traversal fails. } } diff --git a/test/hotspot/jtreg/compiler/loopopts/superword/TestUnorderedReduction.java b/test/hotspot/jtreg/compiler/loopopts/superword/TestUnorderedReduction.java index d50dc96ffb3..7fb11d1f808 100644 --- a/test/hotspot/jtreg/compiler/loopopts/superword/TestUnorderedReduction.java +++ b/test/hotspot/jtreg/compiler/loopopts/superword/TestUnorderedReduction.java @@ -43,7 +43,7 @@ public class TestUnorderedReduction { "-XX:MaxVectorSize=16"); } - @Run(test = {"test1", "test2"}) + @Run(test = {"test1", "test2", "test3"}) @Warmup(0) public void runTests() throws Exception { int[] data = new int[RANGE]; @@ -64,6 +64,14 @@ public class TestUnorderedReduction { throw new RuntimeException("Wrong result test2: " + r1 + " != " + r2); } } + + for (int i = 0; i < ITER; i++) { + int r1 = test3(data, i); + int r2 = ref3(data, i); + if (r1 != r2) { + throw new RuntimeException("Wrong result test3: " + r1 + " != " + r2); + } + } } @Test @@ -140,6 +148,40 @@ public class TestUnorderedReduction { return sum; } + @Test + @IR(counts = {IRNode.LOAD_VECTOR, "> 0", + IRNode.MUL_VI, "> 0", + IRNode.ADD_VI, "= 0", // reduction not moved out of loop + IRNode.ADD_REDUCTION_VI, "> 0",}, + applyIfCPUFeatureOr = {"sse4.1", "true", "asimd", "true"}) + static int test3(int[] data, int sum) { + for (int i = 0; i < RANGE; i+=8) { + // Partial vectorization of reduction chain -> cannot move out of loop + sum += 11 * data[i+0]; // vec 1 + sum += 13 & data[i+1]; // ---------- breaks vec 1 -> scalar reductions + sum += 11 * data[i+2]; + sum += 11 * data[i+3]; + sum += 11 * data[i+4]; // vec 2 -> vectorizes -> vector reduction + sum += 11 * data[i+5]; + sum += 11 * data[i+6]; + sum += 11 * data[i+7]; + } + return sum; + } + + static int ref3(int[] data, int sum) { + for (int i = 0; i < RANGE; i+=8) { + sum += 11 * data[i+0]; + sum += 13 & data[i+1]; + sum += 11 * data[i+2]; + sum += 11 * data[i+3]; + sum += 11 * data[i+4]; + sum += 11 * data[i+5]; + sum += 11 * data[i+6]; + sum += 11 * data[i+7]; + } + return sum; + } static void init(int[] data) { for (int i = 0; i < RANGE; i++) { diff --git a/test/hotspot/jtreg/compiler/loopopts/superword/TestUnorderedReductionPartialVectorization.java b/test/hotspot/jtreg/compiler/loopopts/superword/TestUnorderedReductionPartialVectorization.java new file mode 100644 index 00000000000..9d851a3688b --- /dev/null +++ b/test/hotspot/jtreg/compiler/loopopts/superword/TestUnorderedReductionPartialVectorization.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @test + * @bug JDK-8310130 + * @summary Special test cases for PhaseIdealLoop::move_unordered_reduction_out_of_loop + * Here a case with partial vectorization of the reduction. + * @requires vm.bits == "64" + * @library /test/lib / + * @run driver compiler.loopopts.superword.TestUnorderedReductionPartialVectorization + */ + +package compiler.loopopts.superword; + +import compiler.lib.ir_framework.*; + +public class TestUnorderedReductionPartialVectorization { + static final int RANGE = 1024; + static final int ITER = 10; + + public static void main(String[] args) { + TestFramework.run(); + } + + @Run(test = {"test1"}) + @Warmup(0) + public void runTests() throws Exception { + int[] data = new int[RANGE]; + + init(data); + for (int i = 0; i < ITER; i++) { + long r1 = test1(data, i); + long r2 = ref1(data, i); + if (r1 != r2) { + throw new RuntimeException("Wrong result test1: " + r1 + " != " + r2); + } + } + } + + @Test + @IR(counts = {IRNode.LOAD_VECTOR, "> 0", + IRNode.OR_REDUCTION_V, "> 0",}, + applyIfCPUFeatureOr = {"avx2", "true"}) + static long test1(int[] data, long sum) { + for (int i = 0; i < data.length; i++) { + // Mixing int and long ops means we only end up allowing half of the int + // loads in one pack, and we have two int packs. The first pack has one + // of the pairs missing because of the store, which creates a dependency. + // The first pack is rejected and left as scalar, the second pack succeeds + // with vectorization. That means we have a mixed scalar/vector reduction + // chain. This way it is possible that a vector-reduction has a scalar + // reduction as input, which is neigher a phi nor a vector reduction. + // In such a case, we must bail out of the optimization in + // PhaseIdealLoop::move_unordered_reduction_out_of_loop + int v = data[i]; // int read + data[0] = 0; // ruin the first pack + sum |= v; // long reduction + } + return sum; + } + + static long ref1(int[] data, long sum) { + for (int i = 0; i < data.length; i++) { + int v = data[i]; + data[0] = 0; + sum |= v; + } + return sum; + } + + static void init(int[] data) { + for (int i = 0; i < RANGE; i++) { + data[i] = i + 1; + } + } +}