/*
* Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
/* @test
@bug 4691554 6221056 6380723 6404504 6419565 6529796
@summary Test the supported New I/O coders
@modules jdk.charsets
*/
import java.io.*;
import java.nio.*;
import java.nio.charset.*;
import java.util.regex.*;
public class CoderTest {
private static final int BUFSIZ = 8192; // Initial buffer size
private static final int MAXERRS = 10; // Errors reported per test
private static final String testRootDir
= System.getProperty("test.src", ".");
private static final PrintStream log = System.out;
// Set by -v on the command line
private static boolean verbose = false;
// Test modes
private static final int ROUNDTRIP = 0;
private static final int ENCODE = 1;
private static final int DECODE = 2;
private static boolean shiftHackDBCS = false;
// File extensions, indexed by test mode
private static final String[] extension
= new String[] { ".b2c",
".c2b-irreversible",
".b2c-irreversible" };
// Utilities
private static ByteBuffer expand(ByteBuffer bb) {
ByteBuffer nbb = ByteBuffer.allocate(bb.capacity() * 2);
bb.flip();
nbb.put(bb);
return nbb;
}
private static CharBuffer expand(CharBuffer cb) {
CharBuffer ncb = CharBuffer.allocate(cb.capacity() * 2);
cb.flip();
ncb.put(cb);
return ncb;
}
private static byte[] parseBytes(String s) {
int nb = s.length() / 2;
byte[] bs = new byte[nb];
for (int i = 0; i < nb; i++) {
int j = i * 2;
if (j + 2 > s.length())
throw new RuntimeException("Malformed byte string: " + s);
bs[i] = (byte)Integer.parseInt(s.substring(j, j + 2), 16);
}
return bs;
}
private static String printBytes(byte[] bs) {
StringBuffer sb = new StringBuffer();
for (int i = 0; i < bs.length; i++) {
sb.append(Integer.toHexString((bs[i] >> 4) & 0xf));
sb.append(Integer.toHexString((bs[i] >> 0) & 0xf));
}
return sb.toString();
}
private static String printCodePoint(int cp) {
StringBuffer sb = new StringBuffer();
sb.append("U+");
if (cp > 0xffff)
sb.append(Integer.toHexString((cp >> 16) & 0xf));
sb.append(Integer.toHexString((cp >> 12) & 0xf));
sb.append(Integer.toHexString((cp >> 8) & 0xf));
sb.append(Integer.toHexString((cp >> 4) & 0xf));
sb.append(Integer.toHexString((cp >> 0) & 0xf));
return sb.toString();
}
private static int getCodePoint(CharBuffer cb) {
char c = cb.get();
if (Character.isHighSurrogate(c))
return Character.toCodePoint(c, cb.get());
else
return c;
}
private static String plural(int n) {
return (n == 1 ? "" : "s");
}
static class Entry {
byte[] bb;
int cp;
int cp2;
}
public static class Parser {
static Pattern p = Pattern.compile("(0[xX])?(00)?([0-9a-fA-F]+)\\s+(0[xX])?([0-9a-fA-F]+)(\\+0x([0-9a-fA-F]+))?\\s*");
static final int gBS = 1;
static final int gCP = 2;
static final int gCP2 = 3;
boolean isStateful = false;
BufferedReader reader;
boolean closed;
Matcher matcher;
public Parser (InputStream in)
throws IOException
{
this.reader = new BufferedReader(new InputStreamReader(in));
this.closed = false;
this.matcher = p.matcher("");
}
public boolean isStateful() {
return isStateful;
}
protected boolean isDirective(String line) {
// Stateful DBCS encodings need special treatment
if (line.startsWith("#STATEFUL")) {
return isStateful = true;
}
return line.startsWith("#");
}
protected Entry parse(Matcher m, Entry e) {
e.bb = parseBytes(m.group(3));
e.cp = Integer.parseInt(m.group(5), 16);
if (m.group(7) != null)
e.cp2 = Integer.parseInt(m.group(7), 16);
else
e.cp2 = 0;
return e;
}
public Entry next() throws Exception {
return next(new Entry());
}
// returns null and closes the input stream if the eof has beenreached.
public Entry next(Entry mapping) throws Exception {
if (closed)
return null;
String line;
while ((line = reader.readLine()) != null) {
if (isDirective(line))
continue;
matcher.reset(line);
if (!matcher.lookingAt()) {
//System.out.println("Missed: " + line);
continue;
}
return parse(matcher, mapping);
}
reader.close();
closed = true;
return null;
}
}
// CoderTest
private String encoding;
private Charset cs;
private CharsetDecoder decoder = null;
private CharsetEncoder encoder = null;
private CoderTest(String enc) throws Exception {
encoding = enc;
cs = Charset.forName(enc);
decoder = cs.newDecoder();
encoder = cs.newEncoder();
}
private class Test {
// An instance of this class tests all mappings for
// a particular bytesPerChar value
private int bytesPerChar;
// Reference data from .b2c file
private ByteBuffer refBytes = ByteBuffer.allocate(BUFSIZ);
private CharBuffer refChars = CharBuffer.allocate(BUFSIZ);
private ByteBuffer dRefBytes = ByteBuffer.allocateDirect(BUFSIZ);
private CharBuffer dRefChars = ByteBuffer.allocateDirect(BUFSIZ*2).asCharBuffer();
private Test(int bpc) {
bytesPerChar = bpc;
}
private void put(byte[] bs, char[] cc) {
if (bs.length != bytesPerChar)
throw new IllegalArgumentException(bs.length
+ " != "
+ bytesPerChar);
if (refBytes.remaining() < bytesPerChar)
refBytes = expand(refBytes);
refBytes.put(bs);
if (refChars.remaining() < cc.length)
refChars = expand(refChars);
refChars.put(cc);
}
private boolean decode(ByteBuffer refByte, CharBuffer refChars)
throws Exception {
log.println(" decode" + (refByte.isDirect()?" (direct)":""));
CharBuffer out = decoder.decode(refBytes);
refBytes.rewind();
byte[] bs = new byte[bytesPerChar];
int e = 0;
while (refBytes.hasRemaining()) {
refBytes.get(bs);
int rcp = getCodePoint(refChars);
int ocp = getCodePoint(out);
if (rcp != ocp) {
log.println(" Error: "
+ printBytes(bs)
+ " --> "
+ printCodePoint(ocp)
+ ", expected "
+ printCodePoint(rcp));
if (++e >= MAXERRS) {
log.println(" Too many errors, giving up");
break;
}
}
if (verbose) {
log.println(" "
+ printBytes(bs)
+ " --> "
+ printCodePoint(rcp));
}
}
if (e == 0 && (refChars.hasRemaining() || out.hasRemaining())) {
// Paranoia: Didn't consume everything
throw new IllegalStateException();
}
refBytes.rewind();
refChars.rewind();
return (e == 0);
}
private boolean encode(ByteBuffer refByte, CharBuffer refChars)
throws Exception {
log.println(" encode" + (refByte.isDirect()?" (direct)":""));
ByteBuffer out = encoder.encode(refChars);
refChars.rewind();
// Stateful b2c files have leading and trailing
// shift bytes for each mapping. However when
// block encoded the output will consist of a single
// leadByte followed by the raw DBCS byte values and
// a final trail byte. The state variable shiftHackDBCS
// which is true for stateful DBCS encodings is used
// to conditionally strip away per-mapping shift bytes
// from the comparison of expected versus actual encoded
// byte values. This hack can be eliminated in Mustang
// when sun.io converters and their associated tests are
// removed.
boolean boundaryBytes = false;
int bytesPC = bytesPerChar;
if (shiftHackDBCS && bytesPerChar==4) {
bytesPC = 2;
boundaryBytes = true;
if ((out.get()) != (byte)0x0e) {
log.println("Missing lead byte");
return(false);
}
}
byte[] rbs = new byte[bytesPC];
byte[] obs = new byte[bytesPC];
int e = 0;
while (refChars.hasRemaining()) {
int cp = getCodePoint(refChars);
// Skip lead shift ref byte for stateful encoding tests
if (shiftHackDBCS && bytesPC == 2)
refBytes.get();
refBytes.get(rbs);
out.get(obs);
boolean eq = true;
for (int i = 0; i < bytesPC; i++)
eq &= rbs[i] == obs[i];
if (!eq) {
log.println(" Error: "
+ printCodePoint(cp)
+ " --> "
+ printBytes(obs)
+ ", expected "
+ printBytes(rbs));
if (++e >= MAXERRS) {
log.println(" Too many errors, giving up");
break;
}
}
if (verbose) {
log.println(" "
+ printCodePoint(cp)
+ " --> "
+ printBytes(rbs));
}
// For stateful encodings ignore/exclude per-mapping
// trail bytes from byte comparison
if (shiftHackDBCS && bytesPC == 2)
refBytes.get();
}
if (shiftHackDBCS && boundaryBytes) {
if ((out.get()) != (byte)0x0f) {
log.println("Missing trail byte");
return(false);
}
}
if (e == 0 && (refBytes.hasRemaining() || out.hasRemaining())) {
// Paranoia: Didn't consume everything
throw new IllegalStateException();
}
refBytes.rewind();
refChars.rewind();
return (e == 0);
}
private boolean run(int mode) throws Exception {
log.println(" " + bytesPerChar
+ " byte" + plural(bytesPerChar) + "/char");
if (dRefBytes.capacity() < refBytes.capacity()) {
dRefBytes = ByteBuffer.allocateDirect(refBytes.capacity());
}
if (dRefChars.capacity() < refChars.capacity()) {
dRefChars = ByteBuffer.allocateDirect(refChars.capacity()*2)
.asCharBuffer();
}
refBytes.flip();
refChars.flip();
dRefBytes.clear();
dRefChars.clear();
dRefBytes.put(refBytes).flip();
dRefChars.put(refChars).flip();
refBytes.flip();
refChars.flip();
boolean rv = true;
if (mode != ENCODE) {
rv &= decode(refBytes, refChars);
rv &= decode(dRefBytes, dRefChars);
}
if (mode != DECODE) {
rv &= encode(refBytes, refChars);
rv &= encode(dRefBytes, dRefChars);
}
return rv;
}
}
// Maximum bytes/char being tested
private int maxBytesPerChar = 0;
// Tests, indexed by bytesPerChar - 1
private Test[] tests;
private void clearTests() {
maxBytesPerChar = 0;
tests = new Test[0];
}
// Find the test for the given bytes/char value,
// expanding the test array if needed
//
private Test testFor(int bpc) {
if (bpc > maxBytesPerChar) {
Test[] ts = new Test[bpc];
System.arraycopy(tests, 0, ts, 0, maxBytesPerChar);
for (int i = maxBytesPerChar; i < bpc; i++)
ts[i] = new Test(i + 1);
tests = ts;
maxBytesPerChar = bpc;
}
return tests[bpc - 1];
}
// Compute the name of the test file for the given encoding and mode. If
// the file exists then return its name, otherwise return null.
//
private File testFile(String encoding, int mode) {
File f = new File(testRootDir, encoding + extension[mode]);
if (!f.exists())
return null;
return f;
}
// Parse the given b2c file and load up the required test objects
//
private void loadTests(File f)
throws Exception
{
clearTests();
FileInputStream in = new FileInputStream(f);
try {
Parser p = new Parser(in);
Entry e = new Entry();
while ((e = (Entry)p.next(e)) != null) {
if (e.cp2 != 0)
continue; // skip composite (base+cc) for now
byte[] bs = e.bb;
char[] cc = Character.toChars(e.cp);
testFor(bs.length).put(bs, cc);
}
shiftHackDBCS = p.isStateful();
} finally {
in.close();
}
}
private boolean run() throws Exception {
encoder
.onUnmappableCharacter(CodingErrorAction.REPLACE)
.onMalformedInput(CodingErrorAction.REPLACE);
decoder.onUnmappableCharacter(CodingErrorAction.REPLACE)
.onMalformedInput(CodingErrorAction.REPLACE);
boolean rv = true;
log.println();
log.println(cs.name() + " (" + encoding + ")");
// Outer loop runs three passes: roundtrip, irreversible encodings,
// and then irreversible decodings
for (int mode = ROUNDTRIP; mode <= DECODE; mode++) {
File f = testFile(encoding, mode);
if (f == null)
continue;
loadTests(f);
for (int i = 0; i < maxBytesPerChar; i++)
rv &= tests[i].run(mode);
}
return rv;
}
// For debugging: java CoderTest [-v] foo.b2c bar.b2c ...
//
public static void main(String args[])
throws Exception
{
File d = new File(System.getProperty("test.src", "."));
String[] av = (args.length != 0) ? args : d.list();
int errors = 0;
int tested = 0;
int skipped = 0;
for (int i = 0; i < av.length; i++) {
String a = av[i];
if (a.equals("-v")) {
verbose = true;
continue;
}
if (a.endsWith(".b2c")) {
String encoding = a.substring(0, a.length() - 4);
if (!Charset.isSupported(encoding)) {
log.println();
log.println("Not supported: " + encoding);
skipped++;
continue;
}
tested++;
if (!new CoderTest(encoding).run())
errors++;
}
}
log.println();
log.println(tested + " charset" + plural(tested) + " tested, "
+ skipped + " not supported");
log.println();
if (errors > 0)
throw new Exception("Errors detected in "
+ errors + " charset" + plural(errors));
}
}