8290529: C2: assert(BoolTest(btest).is_canonical()) failure

Reviewed-by: kvn, thartmann

src/hotspot/share/opto/subnode.cpp

@@ -1620,14 +1620,15 @@ Node *BoolNode::Ideal(PhaseGVN *phase, bool can_reshape) {
   }
 
   // Change x u< 1 or x u<= 0 to x == 0
+  // and    x u> 0 or u>= 1   to x != 0
   if (cop == Op_CmpU &&
       cmp1_op != Op_LoadRange &&
-      ((_test._test == BoolTest::lt &&
+      (((_test._test == BoolTest::lt || _test._test == BoolTest::ge) &&
         cmp2->find_int_con(-1) == 1) ||
-       (_test._test == BoolTest::le &&
+       ((_test._test == BoolTest::le || _test._test == BoolTest::gt) &&
         cmp2->find_int_con(-1) == 0))) {
     Node* ncmp = phase->transform(new CmpINode(cmp1, phase->intcon(0)));
-    return new BoolNode(ncmp, BoolTest::eq);
+    return new BoolNode(ncmp, _test.is_less() ? BoolTest::eq : BoolTest::ne);
   }
 
   // Change (arraylength <= 0) or (arraylength == 0)

test/hotspot/jtreg/compiler/c2/TestUnsignedCompareIntoEqualityNotCanonical.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2022, Red Hat, Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8290529
+ * @summary C2: assert(BoolTest(btest).is_canonical()) failure
+ * @run main/othervm -XX:-BackgroundCompilation -XX:-UseOnStackReplacement -XX:-TieredCompilation TestUnsignedCompareIntoEqualityNotCanonical
+ */
+
+
+public class TestUnsignedCompareIntoEqualityNotCanonical {
+    public static void main(String[] args) {
+        for (int i = 0; i < 20_000; i++) {
+            test(0);
+            test(1);
+        }
+    }
+
+    private static int test(int x) {
+        if (Integer.compareUnsigned(0, x) >= 0) {
+            return 42;
+        }
+        return -42;
+    }
+}

test/hotspot/jtreg/compiler/c2/irTests/CmpUWithZero.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2022, Red Hat, Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package compiler.c2.irTests;
+
+import compiler.lib.ir_framework.*;
+
+/*
+ * @test
+ * bug 8290529
+ * @summary verify that x <u 1 is transformed to x == 0
+ * @requires os.arch=="amd64" | os.arch=="x86_64"
+ * @library /test/lib /
+ * @requires vm.compiler2.enabled
+ * @run driver compiler.c2.irTests.CmpUWithZero
+ */
+
+public class CmpUWithZero {
+    static volatile boolean field;
+
+    public static void main(String[] args) {
+        TestFramework.run();
+    }
+
+    @Test
+    @IR(counts = { IRNode.CMP_I, "1" })
+    @IR(failOn = { IRNode.CMP_U})
+    public static void test(int x) {
+        if (Integer.compareUnsigned(x, 1) < 0) {
+            field = true;
+        } else {
+            field = false;
+        }
+    }
+
+    @Run(test = "test")
+    private void testRunner() {
+        test(0);
+        test(42);
+    }
+
+}

test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java

@@ -176,6 +176,7 @@ public class IRNode {
     public static final String CMP_UL = START + "CmpUL" + MID + END;
     public static final String CMP_U3 = START + "CmpU3" + MID + END;
     public static final String CMP_UL3 = START + "CmpUL3" + MID + END;
+    public static final String CMP_I = START + "CmpI" + MID + END;
     public static final String MUL = START + "Mul(I|L|F|D)" + MID + END;
     public static final String MUL_I = START + "MulI" + MID + END;
     public static final String MUL_L = START + "MulL" + MID + END;