8279622: C2: miscompilation of map pattern as a vector reduction

Reviewed-by: roland, kvn, thartmann

src/hotspot/share/opto/loopTransform.cpp

@@ -1607,6 +1607,15 @@ void PhaseIdealLoop::insert_pre_post_loops(IdealLoopTree *loop, Node_List &old_n
   set_idom(new_pre_exit, pre_end, dd_main_head);
   set_loop(new_pre_exit, outer_loop->_parent);
 
+  if (peel_only) {
+    // Nodes in the peeled iteration that were marked as reductions within the
+    // original loop might not be reductions within their new outer loop.
+    for (uint i = 0; i < loop->_body.size(); i++) {
+      Node* n = old_new[loop->_body[i]->_idx];
+      n->remove_flag(Node::Flag_is_reduction);
+    }
+  }
+
   // Step B2: Build a zero-trip guard for the main-loop.  After leaving the
   // pre-loop, the main-loop may not execute at all.  Later in life this
   // zero-trip guard will become the minimum-trip guard when we unroll

src/hotspot/share/opto/loopnode.cpp

@@ -3886,6 +3886,17 @@ uint IdealLoopTree::est_loop_flow_merge_sz() const {
   return 0;
 }
 
+#ifdef ASSERT
+bool IdealLoopTree::has_reduction_nodes() const {
+  for (uint i = 0; i < _body.size(); i++) {
+    if (_body[i]->is_reduction()) {
+      return true;
+    }
+  }
+  return false;
+}
+#endif // ASSERT
+
 #ifndef PRODUCT
 //------------------------------dump_head--------------------------------------
 // Dump 1 liner for loop header info

src/hotspot/share/opto/loopnode.hpp

@@ -778,6 +778,11 @@ public:
 
   void remove_main_post_loops(CountedLoopNode *cl, PhaseIdealLoop *phase);
 
+#ifdef ASSERT
+  // Tell whether the body contains nodes marked as reductions.
+  bool has_reduction_nodes() const;
+#endif // ASSERT
+
 #ifndef PRODUCT
   void dump_head() const;       // Dump loop head only
   void dump() const;            // Dump this loop recursively

src/hotspot/share/opto/superword.cpp

@@ -110,6 +110,8 @@ bool SuperWord::transform_loop(IdealLoopTree* lpt, bool do_optimization) {
     return false; // skip malformed counted loop
   }
 
+  assert(!lpt->has_reduction_nodes() || cl->is_reduction_loop(),
+         "non-reduction loop contains reduction nodes");
   if (cl->is_rce_post_loop() && cl->is_reduction_loop()) {
     // Post loop vectorization doesn't support reductions
     return false;

test/hotspot/jtreg/compiler/loopopts/superword/TestPeeledReductionNode.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 8279622
+ * @summary Test that reduction nodes peeled out of an inner loop are not
+ *          vectorized as reductions within the outer loop.
+ * @library /test/lib
+ * @comment The test is run with -XX:LoopUnrollLimit=32 to prevent unrolling
+ *          from fully replacing vectorization.
+ * @run main/othervm -Xbatch -XX:LoopUnrollLimit=32
+ *      compiler.loopopts.superword.TestPeeledReductionNode
+ */
+package compiler.loopopts.superword;
+
+import jdk.test.lib.Asserts;
+
+public class TestPeeledReductionNode {
+    static final int N = 32;
+    static final int M = 65; // Must be odd and >= 65 to trigger the failure.
+    static final int INPUT = 0b0000_0000_0000_0000_0000_0000_0000_0001;
+    static final int MASK  = 0b0000_0000_1000_0000_0000_0000_0000_0000;
+    static final int EXPECTED = (M % 2 == 0 ? INPUT : INPUT ^ MASK);
+    static int mask = 0;
+    public static void main(String[] args) {
+        int r[] = new int[N];
+        for (int i = 0; i < N; i++) {
+            r[i] = INPUT;
+        }
+        // Trigger the relevant OSR compilation and set
+        // TestPeeledReductionNode.mask to MASK.
+        for (int k = 0; k < MASK; k++) {
+            TestPeeledReductionNode.mask++;
+        }
+        for (int i = 0; i < N; i++) {
+            for (int j = 0; j < M; j++) {
+                // Before the fix, this reduction is peeled out of its loop and
+                // wrongly remains marked as a reduction within the outer loop.
+                r[i] ^= TestPeeledReductionNode.mask;
+            }
+        }
+        for (int i = 0; i < N; i++) {
+            Asserts.assertEquals(r[i], EXPECTED);
+        }
+        return;
+    }
+}