8248655: Support supplementary characters in String case insensitive operations

8248434: some newly added locale cannot parse uppercased date string

Reviewed-by: jlaskey, joehw, rriggs, bchristi
This commit is contained in:
Naoto Sato 2020-07-23 08:46:31 -07:00
parent dc80e63811
commit 1f63603288
7 changed files with 272 additions and 87 deletions
src/java.base/share/classes/java/lang
test
jdk/java
micro/org/openjdk/bench/java/lang

@ -43,7 +43,6 @@ import java.util.Optional;
import java.util.Spliterator;
import java.util.StringJoiner;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import java.util.stream.Collectors;
@ -1134,16 +1133,16 @@ public final class String
/**
* Compares this {@code String} to another {@code String}, ignoring case
* considerations. Two strings are considered equal ignoring case if they
* are of the same length and corresponding characters in the two strings
* are equal ignoring case.
* are of the same length and corresponding Unicode code points in the two
* strings are equal ignoring case.
*
* <p> Two characters {@code c1} and {@code c2} are considered the same
* <p> Two Unicode code points are considered the same
* ignoring case if at least one of the following is true:
* <ul>
* <li> The two characters are the same (as compared by the
* <li> The two Unicode code points are the same (as compared by the
* {@code ==} operator)
* <li> Calling {@code Character.toLowerCase(Character.toUpperCase(char))}
* on each character produces the same result
* <li> Calling {@code Character.toLowerCase(Character.toUpperCase(int))}
* on each Unicode code point produces the same result
* </ul>
*
* <p>Note that this method does <em>not</em> take locale into account, and
@ -1158,6 +1157,7 @@ public final class String
* false} otherwise
*
* @see #equals(Object)
* @see #codePoints()
*/
public boolean equalsIgnoreCase(String anotherString) {
return (this == anotherString) ? true
@ -1224,7 +1224,8 @@ public final class String
/**
* A Comparator that orders {@code String} objects as by
* {@code compareToIgnoreCase}. This comparator is serializable.
* {@link #compareToIgnoreCase(String) compareToIgnoreCase}.
* This comparator is serializable.
* <p>
* Note that this Comparator does <em>not</em> take locale into account,
* and will result in an unsatisfactory ordering for certain locales.
@ -1261,10 +1262,10 @@ public final class String
/**
* Compares two strings lexicographically, ignoring case
* differences. This method returns an integer whose sign is that of
* calling {@code compareTo} with normalized versions of the strings
* calling {@code compareTo} with case folded versions of the strings
* where case differences have been eliminated by calling
* {@code Character.toLowerCase(Character.toUpperCase(character))} on
* each character.
* {@code Character.toLowerCase(Character.toUpperCase(int))} on
* each Unicode code point.
* <p>
* Note that this method does <em>not</em> take locale into account,
* and will result in an unsatisfactory ordering for certain locales.
@ -1275,6 +1276,7 @@ public final class String
* specified String is greater than, equal to, or less
* than this String, ignoring case considerations.
* @see java.text.Collator
* @see #codePoints()
* @since 1.2
*/
public int compareToIgnoreCase(String str) {
@ -1362,30 +1364,26 @@ public final class String
* <p>
* A substring of this {@code String} object is compared to a substring
* of the argument {@code other}. The result is {@code true} if these
* substrings represent character sequences that are the same, ignoring
* case if and only if {@code ignoreCase} is true. The substring of
* this {@code String} object to be compared begins at index
* {@code toffset} and has length {@code len}. The substring of
* {@code other} to be compared begins at index {@code ooffset} and
* has length {@code len}. The result is {@code false} if and only if
* at least one of the following is true:
* <ul><li>{@code toffset} is negative.
* <li>{@code ooffset} is negative.
* <li>{@code toffset+len} is greater than the length of this
* substrings represent Unicode code point sequences that are the same,
* ignoring case if and only if {@code ignoreCase} is true.
* The sequences {@code tsequence} and {@code osequence} are compared,
* where {@code tsequence} is the sequence produced as if by calling
* {@code this.substring(toffset, len).codePoints()} and {@code osequence}
* is the sequence produced as if by calling
* {@code other.substring(ooffset, len).codePoints()}.
* The result is {@code true} if and only if all of the following
* are true:
* <ul><li>{@code toffset} is non-negative.
* <li>{@code ooffset} is non-negative.
* <li>{@code toffset+len} is less than or equal to the length of this
* {@code String} object.
* <li>{@code ooffset+len} is greater than the length of the other
* <li>{@code ooffset+len} is less than or equal to the length of the other
* argument.
* <li>{@code ignoreCase} is {@code false} and there is some nonnegative
* integer <i>k</i> less than {@code len} such that:
* <blockquote><pre>
* this.charAt(toffset+k) != other.charAt(ooffset+k)
* </pre></blockquote>
* <li>{@code ignoreCase} is {@code true} and there is some nonnegative
* integer <i>k</i> less than {@code len} such that:
* <blockquote><pre>
* Character.toLowerCase(Character.toUpperCase(this.charAt(toffset+k))) !=
* Character.toLowerCase(Character.toUpperCase(other.charAt(ooffset+k)))
* </pre></blockquote>
* <li>if {@code ignoreCase} is {@code false}, all pairs of corresponding Unicode
* code points are equal integer values; or if {@code ignoreCase} is {@code true},
* {@link Character#toLowerCase(int) Character.toLowerCase(}
* {@link Character#toUpperCase(int)}{@code )} on all pairs of Unicode code points
* results in equal integer values.
* </ul>
*
* <p>Note that this method does <em>not</em> take locale into account,
@ -1400,12 +1398,14 @@ public final class String
* @param other the string argument.
* @param ooffset the starting offset of the subregion in the string
* argument.
* @param len the number of characters to compare.
* @param len the number of characters (Unicode code units -
* 16bit {@code char} value) to compare.
* @return {@code true} if the specified subregion of this string
* matches the specified subregion of the string argument;
* {@code false} otherwise. Whether the matching is exact
* or case insensitive depends on the {@code ignoreCase}
* argument.
* @see #codePoints()
*/
public boolean regionMatches(boolean ignoreCase, int toffset,
String other, int ooffset, int len) {

@ -319,25 +319,92 @@ final class StringUTF16 {
}
public static int compareToCI(byte[] value, byte[] other) {
int len1 = length(value);
int len2 = length(other);
int lim = Math.min(len1, len2);
for (int k = 0; k < lim; k++) {
char c1 = getChar(value, k);
char c2 = getChar(other, k);
if (c1 != c2) {
c1 = Character.toUpperCase(c1);
c2 = Character.toUpperCase(c2);
if (c1 != c2) {
c1 = Character.toLowerCase(c1);
c2 = Character.toLowerCase(c2);
if (c1 != c2) {
return c1 - c2;
}
}
return compareToCIImpl(value, 0, length(value), other, 0, length(other));
}
private static int compareToCIImpl(byte[] value, int toffset, int tlen,
byte[] other, int ooffset, int olen) {
int tlast = toffset + tlen;
int olast = ooffset + olen;
assert toffset >= 0 && ooffset >= 0;
assert tlast <= length(value);
assert olast <= length(other);
for (int k1 = toffset, k2 = ooffset; k1 < tlast && k2 < olast; k1++, k2++) {
int cp1 = (int)getChar(value, k1);
int cp2 = (int)getChar(other, k2);
if (cp1 == cp2 || compareCodePointCI(cp1, cp2) == 0) {
continue;
}
// Check for supplementary characters case
cp1 = codePointIncluding(value, cp1, k1, toffset, tlast);
if (cp1 < 0) {
k1++;
cp1 -= cp1;
}
cp2 = codePointIncluding(other, cp2, k2, ooffset, olast);
if (cp2 < 0) {
k2++;
cp2 -= cp2;
}
int diff = compareCodePointCI(cp1, cp2);
if (diff != 0) {
return diff;
}
}
return len1 - len2;
return tlen - olen;
}
// Case insensitive comparison of two code points
private static int compareCodePointCI(int cp1, int cp2) {
// try converting both characters to uppercase.
// If the results match, then the comparison scan should
// continue.
cp1 = Character.toUpperCase(cp1);
cp2 = Character.toUpperCase(cp2);
if (cp1 != cp2) {
// Unfortunately, conversion to uppercase does not work properly
// for the Georgian alphabet, which has strange rules about case
// conversion. So we need to make one last check before
// exiting.
cp1 = Character.toLowerCase(cp1);
cp2 = Character.toLowerCase(cp2);
if (cp1 != cp2) {
return cp1 - cp2;
}
}
return 0;
}
// Returns a code point from the code unit pointed by "index". If it is
// not a surrogate or an unpaired surrogate, then the code unit is
// returned as is. Otherwise, it is combined with the code unit before
// or after, depending on the type of the surrogate at index, to make a
// supplementary code point. The return value will be negated if the code
// unit pointed by index is a high surrogate, and index + 1 is a low surrogate.
private static int codePointIncluding(byte[] ba, int cp, int index, int start, int end) {
// fast check
if (!Character.isSurrogate((char)cp)) {
return cp;
}
if (Character.isLowSurrogate((char)cp)) {
if (index > start) {
char c = getChar(ba, index - 1);
if (Character.isHighSurrogate(c)) {
return Character.toCodePoint(c, (char)cp);
}
}
} else if (index + 1 < end) { // cp == high surrogate
char c = getChar(ba, index + 1);
if (Character.isLowSurrogate(c)) {
// negate the code point
return - Character.toCodePoint((char)cp, c);
}
}
return cp;
}
public static int compareToCI_Latin1(byte[] value, byte[] other) {
@ -716,34 +783,7 @@ final class StringUTF16 {
public static boolean regionMatchesCI(byte[] value, int toffset,
byte[] other, int ooffset, int len) {
int last = toffset + len;
assert toffset >= 0 && ooffset >= 0;
assert ooffset + len <= length(other);
assert last <= length(value);
while (toffset < last) {
char c1 = getChar(value, toffset++);
char c2 = getChar(other, ooffset++);
if (c1 == c2) {
continue;
}
// try converting both characters to uppercase.
// If the results match, then the comparison scan should
// continue.
char u1 = Character.toUpperCase(c1);
char u2 = Character.toUpperCase(c2);
if (u1 == u2) {
continue;
}
// Unfortunately, conversion to uppercase does not work properly
// for the Georgian alphabet, which has strange rules about case
// conversion. So we need to make one last check before
// exiting.
if (Character.toLowerCase(u1) == Character.toLowerCase(u2)) {
continue;
}
return false;
}
return true;
return compareToCIImpl(value, toffset, len, other, ooffset, len) == 0;
}
public static boolean regionMatchesCI_Latin1(byte[] value, int toffset,

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -28,7 +28,7 @@ import static org.testng.Assert.assertEquals;
/*
* @test
* @bug 8077559
* @bug 8077559 8248655
* @summary Tests Compact String. This one is for String.compareToIgnoreCase.
* @run testng/othervm -XX:+CompactStrings CompareToIgnoreCase
* @run testng/othervm -XX:-CompactStrings CompareToIgnoreCase
@ -39,7 +39,6 @@ public class CompareToIgnoreCase extends CompactString {
@DataProvider
public Object[][] provider() {
return new Object[][] {
new Object[] { STRING_EMPTY, "A", -1 },
new Object[] { STRING_L1, "a", 0 },
new Object[] { STRING_L1, "A", 0 },
@ -67,7 +66,10 @@ public class CompareToIgnoreCase extends CompactString {
new Object[] { STRING_M12, "\uFF41a", 0 },
new Object[] { STRING_M12, "\uFF41\uFF42", -65249 },
new Object[] { STRING_M11, "a\uFF41", 0 },
new Object[] { STRING_M11, "a\uFF42", -1 }, };
new Object[] { STRING_M11, "a\uFF42", -1 },
new Object[] { STRING_SUPPLEMENTARY, STRING_SUPPLEMENTARY_LOWERCASE, 0 },
new Object[] { STRING_SUPPLEMENTARY, "\uD801\uDC28\uD801\uDC27\uFF41a", -38 },
};
}
@Test(dataProvider = "provider")

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -28,7 +28,7 @@ import static org.testng.Assert.assertEquals;
/*
* @test
* @bug 8077559
* @bug 8077559 8248655
* @summary Tests Compact String. This one is for String.equalsIgnoreCase.
* @run testng/othervm -XX:+CompactStrings EqualsIgnoreCase
* @run testng/othervm -XX:-CompactStrings EqualsIgnoreCase
@ -54,6 +54,7 @@ public class EqualsIgnoreCase extends CompactString {
new Object[] { STRING_M12, "\uFF21A", true },
new Object[] { STRING_M11, "a\uFF41", true },
new Object[] { STRING_M11, "A\uFF21", true },
new Object[] { STRING_SUPPLEMENTARY, STRING_SUPPLEMENTARY_LOWERCASE, true },
};
}

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -28,7 +28,7 @@ import static org.testng.Assert.assertEquals;
/*
* @test
* @bug 8077559
* @bug 8077559 8248655
* @summary Tests Compact String. This one is for String.regionMatches.
* @run testng/othervm -XX:+CompactStrings RegionMatches
* @run testng/othervm -XX:-CompactStrings RegionMatches
@ -66,15 +66,23 @@ public class RegionMatches extends CompactString {
3, true },
new Object[] { STRING_MDUPLICATE1, false, 0, "\uFF21a\uFF21",
0, 3, false },
new Object[] { STRING_SUPPLEMENTARY, true, 0, "\uD801\uDC28\uD801\uDC29",
0, 4, true },
new Object[] { STRING_SUPPLEMENTARY, true, 1, "\uDC00\uD801",
0, 2, true },
new Object[] { STRING_SUPPLEMENTARY, true, 1, "\uDC28",
0, 1, false },
new Object[] { STRING_SUPPLEMENTARY, true, 4, "\uFF21", 0, 1,
true },
new Object[] { STRING_SUPPLEMENTARY, true, 5, "A", 0, 1, true },
new Object[] { STRING_SUPPLEMENTARY_LOWERCASE, false, 0,
"\uD801\uDC28\uD801\uDC29", 0, 4, true },
new Object[] { STRING_SUPPLEMENTARY_LOWERCASE, true, 0,
"\uD801\uDC00\uD801\uDC01", 0, 4, true },
new Object[] { STRING_SUPPLEMENTARY_LOWERCASE, true, 1,
"\uDC28\uD801", 0, 2, true },
new Object[] { STRING_SUPPLEMENTARY_LOWERCASE, true, 1,
"\uDC00", 0, 1, false },
new Object[] { STRING_SUPPLEMENTARY_LOWERCASE, true, 1,
"\uDC00\uD801", 0, 2, false },
new Object[] { STRING_SUPPLEMENTARY_LOWERCASE, true, 4,

@ -0,0 +1,72 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
/**
* @test
* @bug 8248434
* @modules jdk.localedata
* @run testng/othervm CaseInsensitiveParseTest
* @summary Checks format/parse round trip in case-insensitive manner.
*/
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.Locale;
import java.util.stream.Stream;
import static org.testng.Assert.assertEquals;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
public class CaseInsensitiveParseTest {
private final static String PATTERN = "GGGG/yyyy/MMMM/dddd/hhhh/mmmm/ss/aaaa";
private final static Date EPOCH = new Date(0L);
@DataProvider
private Object[][] locales() {
return (Object[][])Arrays.stream(DateFormat.getAvailableLocales())
.map(Stream::of)
.map(Stream::toArray)
.toArray(Object[][]::new);
}
@Test(dataProvider = "locales")
public void testUpperCase(Locale loc) throws ParseException {
SimpleDateFormat sdf = new SimpleDateFormat(PATTERN, loc);
String formatted = sdf.format(EPOCH);
assertEquals(sdf.parse(formatted.toUpperCase(Locale.ROOT)), EPOCH,
"roundtrip failed for string '" + formatted + "', locale: " + loc);
}
@Test(dataProvider = "locales")
public void testLowerCase(Locale loc) throws ParseException {
SimpleDateFormat sdf = new SimpleDateFormat(PATTERN, loc);
String formatted = sdf.format(EPOCH);
assertEquals(sdf.parse(formatted.toLowerCase(Locale.ROOT)), EPOCH,
"roundtrip failed for string '" + formatted + "', locale: " + loc);
}
}

@ -0,0 +1,62 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package org.openjdk.bench.java.lang;
import org.openjdk.jmh.annotations.*;
import java.util.concurrent.TimeUnit;
/*
* This benchmark naively explores String::compareToIgnoreCase performance
*/
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
@State(Scope.Benchmark)
public class StringCompareToIgnoreCase {
public String upper = new String("\u0100\u0102\u0104\u0106\u0108");
public String upperLower = new String("\u0100\u0102\u0104\u0106\u0109");
public String lower = new String("\u0101\u0103\u0105\u0107\u0109");
public String supUpper = new String("\ud801\udc00\ud801\udc01\ud801\udc02\ud801\udc03\ud801\udc04");
public String supUpperLower = new String("\ud801\udc00\ud801\udc01\ud801\udc02\ud801\udc03\ud801\udc2c");
public String supLower = new String("\ud801\udc28\ud801\udc29\ud801\udc2a\ud801\udc2b\ud801\udc2c");
@Benchmark
public int upperLower() {
return upper.compareToIgnoreCase(upperLower);
}
@Benchmark
public int lower() {
return upper.compareToIgnoreCase(lower);
}
@Benchmark
public int supUpperLower() {
return supUpper.compareToIgnoreCase(supUpperLower);
}
@Benchmark
public int supLower() {
return supUpper.compareToIgnoreCase(supLower);
}
}