stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

4206909: want java.util.zip to work for interactive use (Z_SYNC_FLUSH)

Browse Source 
Add sync_flush option into Deflater/DefalterOutputStream

Reviewed-by: martin, alanb
This commit is contained in:
Xueming Shen 2009-10-21 11:50:25 -07:00
parent e5745ffc8c
commit 30e3c9dc22
4 changed files with 481 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

132
jdk/src/share/classes/java/util/zip/Deflater.java
View File

@ -122,6 +122,33 @@ class Deflater {
*/
public static final int DEFAULT_STRATEGY = 0;
/**
* Compression flush mode used to achieve best compression result.
*
* @see Deflater#deflate(byte[], int, int, int)
* @since 1.7
*/
public static final int NO_FLUSH = 0;
/**
* Compression flush mode used to flush out all pending output; may
* degrade compression for some compression algorithms.
*
* @see Deflater#deflate(byte[], int, int, int)
* @since 1.7
*/
public static final int SYNC_FLUSH = 2;
/**
* Compression flush mode used to flush out all pending output and
* reset the deflater. Using this mode too often can seriously degrade
* compression.
*
* @see Deflater#deflate(byte[], int, int, int)
* @since 1.7
*/
public static final int FULL_FLUSH = 3;
static {
/* Zip library is loaded from System.initializeSystemClass */
initIDs();
@ -289,35 +316,100 @@ class Deflater {
}
/**
* Fills specified buffer with compressed data. Returns actual number
* of bytes of compressed data. A return value of 0 indicates that
* needsInput() should be called in order to determine if more input
* data is required.
* Compresses the input data and fills specified buffer with compressed
* data. Returns actual number of bytes of compressed data. A return value
* of 0 indicates that {@link needsInput() needsInput} should be called
* in order to determine if more input data is required.
*
* <p>This method uses {@link #NO_FLUSH} as its compression flush mode.
* An invocation of this method of the form {@code deflater.deflate(b, off, len)}
* yields the same result as the invocation of
* {@code deflater.deflate(b, off, len, Deflater.NO_FLUSH)}.
*
* @param b the buffer for the compressed data
* @param off the start offset of the data
* @param len the maximum number of bytes of compressed data
* @return the actual number of bytes of compressed data
* @return the actual number of bytes of compressed data written to the
* output buffer
*/
public synchronized int deflate(byte[] b, int off, int len) {
public int deflate(byte[] b, int off, int len) {
return deflateBytes(b, off, len, NO_FLUSH);
}
/**
* Compresses the input data and fills specified buffer with compressed
* data. Returns actual number of bytes of compressed data. A return value
* of 0 indicates that {@link needsInput() needsInput} should be called
* in order to determine if more input data is required.
*
* <p>This method uses {@link #NO_FLUSH} as its compression flush mode.
* An invocation of this method of the form {@code deflater.deflate(b)}
* yields the same result as the invocation of
* {@code deflater.deflate(b, 0, b.length, Deflater.NO_FLUSH)}.
*
* @param b the buffer for the compressed data
* @return the actual number of bytes of compressed data written to the
* output buffer
*/
public int deflate(byte[] b) {
return deflate(b, 0, b.length, NO_FLUSH);
}
/**
* Compresses the input data and fills the specified buffer with compressed
* data. Returns actual number of bytes of data compressed.
*
* <p>Compression flush mode is one of the following three modes:
*
* <ul>
* <li>{@link #NO_FLUSH}: allows the deflater to decide how much data
* to accumulate, before producing output, in order to achieve the best
* compression (should be used in normal use scenario). A return value
* of 0 in this flush mode indicates that {@link #needsInput()} should
* be called in order to determine if more input data is required.
*
* <li>{@link #SYNC_FLUSH}: all pending output in the deflater is flushed,
* to the specified output buffer, so that an inflater that works on
* compressed data can get all input data available so far (In particular
* the {@link #needsInput()} returns {@code true} after this invocation
* if enough output space is provided). Flushing with {@link #SYNC_FLUSH}
* may degrade compression for some compression algorithms and so it
* should be used only when necessary.
*
* <li>{@link #FULL_FLUSH}: all pending output is flushed out as with
* {@link #SYNC_FLUSH}. The compression state is reset so that the inflater
* that works on the compressed output data can restart from this point
* if previous compressed data has been damaged or if random access is
* desired. Using {@link #FULL_FLUSH} too often can seriously degrade
* compression.
* </ul>
*
* <p>In the case of {@link #FULL_FLUSH} or {@link #SYNC_FLUSH}, if
* the return value is {@code len}, the space available in output
* buffer {@code b}, this method should be invoked again with the same
* {@code flush} parameter and more output space.
*
* @param b the buffer for the compressed data
* @param off the start offset of the data
* @param len the maximum number of bytes of compressed data
* @param flush the compression flush mode
* @return the actual number of bytes of compressed data written to
* the output buffer
*
* @throws IllegalArgumentException if the flush mode is invalid
* @since 1.7
*/
public synchronized int deflate(byte[] b, int off, int len, int flush) {
if (b == null) {
throw new NullPointerException();
}
if (off < 0 || len < 0 || off > b.length - len) {
throw new ArrayIndexOutOfBoundsException();
}
return deflateBytes(b, off, len);
}
/**
* Fills specified buffer with compressed data. Returns actual number
* of bytes of compressed data. A return value of 0 indicates that
* needsInput() should be called in order to determine if more input
* data is required.
* @param b the buffer for the compressed data
* @return the actual number of bytes of compressed data
*/
public int deflate(byte[] b) {
return deflate(b, 0, b.length);
if (flush == NO_FLUSH || flush == SYNC_FLUSH ||
flush == FULL_FLUSH)
return deflateBytes(b, off, len, flush);
throw new IllegalArgumentException();
}
/**
@ -420,7 +512,7 @@ class Deflater {
private native static long init(int level, int strategy, boolean nowrap);
private native static void setDictionary(long strm, byte[] b, int off,
int len);
private native int deflateBytes(byte[] b, int off, int len);
private native int deflateBytes(byte[] b, int off, int len, int flush);
private native static int getAdler(long strm);
private native static long getBytesRead(long strm);
private native static long getBytesWritten(long strm);

124
jdk/src/share/classes/java/util/zip/DeflaterOutputStream.java
View File

@ -1,5 +1,5 @@
/*
* Copyright 1996-2006 Sun Microsystems, Inc. All Rights Reserved.
* Copyright 1996-2009 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -56,15 +56,29 @@ class DeflaterOutputStream extends FilterOutputStream {
private boolean closed = false;
private final boolean syncFlush;
/**
* Creates a new output stream with the specified compressor and
* buffer size.
* Creates a new output stream with the specified compressor,
* buffer size and flush mode.
* @param out the output stream
* @param def the compressor ("deflater")
* @param size the output buffer size
* @exception IllegalArgumentException if size is <= 0
* @param syncFlush
* if {@code true} the {@link flush()} method of this
* instance flushes the compressor with flush mode
* {@link Deflater#SYNC_FLUSH} before flushing the output
* stream, otherwise only flushes the output stream
*
* @throws IllegalArgumentException if size is <= 0
*
* @since 1.7
*/
public DeflaterOutputStream(OutputStream out, Deflater def, int size) {
public DeflaterOutputStream(OutputStream out,
Deflater def,
int size,
boolean syncFlush) {
super(out);
if (out == null || def == null) {
throw new NullPointerException();
@ -72,27 +86,93 @@ class DeflaterOutputStream extends FilterOutputStream {
throw new IllegalArgumentException("buffer size <= 0");
}
this.def = def;
buf = new byte[size];
this.buf = new byte[size];
this.syncFlush = syncFlush;
}
/**
* Creates a new output stream with the specified compressor and
* buffer size.
*
* <p>The new output stream instance is created as if by invoking
* the 4-argument constructor DeflaterOutputStream(out, def, size, false).
*
* @param out the output stream
* @param def the compressor ("deflater")
* @param size the output buffer size
* @exception IllegalArgumentException if size is <= 0
*/
public DeflaterOutputStream(OutputStream out, Deflater def, int size) {
this(out, def, size, false);
}
/**
* Creates a new output stream with the specified compressor, flush
* mode and a default buffer size.
*
* @param out the output stream
* @param def the compressor ("deflater")
* @param syncFlush
* if {@code true} the {@link flush()} method of this
* instance flushes the compressor with flush mode
* {@link Deflater#SYNC_FLUSH} before flushing the output
* stream, otherwise only flushes the output stream
*
* @since 1.7
*/
public DeflaterOutputStream(OutputStream out,
Deflater def,
boolean syncFlush) {
this(out, def, 512, syncFlush);
}
/**
* Creates a new output stream with the specified compressor and
* a default buffer size.
*
* <p>The new output stream instance is created as if by invoking
* the 3-argument constructor DeflaterOutputStream(out, def, false).
*
* @param out the output stream
* @param def the compressor ("deflater")
*/
public DeflaterOutputStream(OutputStream out, Deflater def) {
this(out, def, 512);
this(out, def, 512, false);
}
boolean usesDefaultDeflater = false;
/**
* Creates a new output stream with a default compressor, a default
* buffer size and the specified flush mode.
*
* @param out the output stream
* @param syncFlush
* if {@code true} the {@link flush()} method of this
* instance flushes the compressor with flush mode
* {@link Deflater#SYNC_FLUSH} before flushing the output
* stream, otherwise only flushes the output stream
*
* @since 1.7
*/
public DeflaterOutputStream(OutputStream out, boolean syncFlush) {
this(out, new Deflater(), 512, syncFlush);
usesDefaultDeflater = true;
}
/**
* Creates a new output stream with a default compressor and buffer size.
*
* <p>The new output stream instance is created as if by invoking
* the 2-argument constructor DeflaterOutputStream(out, false).
*
* @param out the output stream
*/
public DeflaterOutputStream(OutputStream out) {
this(out, new Deflater());
this(out, false);
usesDefaultDeflater = true;
}
@ -178,4 +258,32 @@ class DeflaterOutputStream extends FilterOutputStream {
out.write(buf, 0, len);
}
}
/**
* Flushes the compressed output stream.
*
* If {@link DeflaterOutputStream(OutputStream, Deflater, int, boolean)
* syncFlush} is {@code true} when this compressed output stream is
* constructed this method flushes the underlying {@code compressor}
* first with the flush mode {@link Deflater#SYNC_FLUSH} to force
* all pending data to be flushed out to the output stream and then
* flushes the output stream. Otherwise this method only flushes the
* output stream without flushing the {@code compressor}.
*
* @throws IOException if an I/O error has occurred
*
* @since 1.7
*/
public void flush() throws IOException {
if (syncFlush && !def.finished()) {
int len = 0;
while ((len = def.deflate(buf, 0, buf.length, Deflater.SYNC_FLUSH)) > 0)
{
out.write(buf, 0, len);
if (len < buf.length)
break;
}
}
out.flush();
}
}

4
jdk/src/share/native/java/util/zip/Deflater.c
View File

@ -118,7 +118,7 @@ Java_java_util_zip_Deflater_setDictionary(JNIEnv *env, jclass cls, jlong strm,
JNIEXPORT jint JNICALL
Java_java_util_zip_Deflater_deflateBytes(JNIEnv *env, jobject this,
jarray b, jint off, jint len)
jarray b, jint off, jint len, jint flush)
{
z_stream *strm = jlong_to_ptr((*env)->GetLongField(env, this, strmID));
@ -197,7 +197,7 @@ Java_java_util_zip_Deflater_deflateBytes(JNIEnv *env, jobject this,
strm->next_out = (Bytef *) out_buf;
strm->avail_in = this_len;
strm->avail_out = len;
res = deflate(strm, finish ? Z_FINISH : Z_NO_FLUSH);
res = deflate(strm, finish ? Z_FINISH : flush);
if (res == Z_STREAM_END || res == Z_OK) {
(*env)->SetByteArrayRegion(env, b, off, len - strm->avail_out, out_buf);

251
jdk/test/java/util/zip/InflateIn_DeflateOut.java Normal file
View File

@ -0,0 +1,251 @@
/*
* Copyright 2009 Google, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
* CA 95054 USA or visit www.sun.com if you need additional information or
* have any questions.
*/
/**
* @test
* @bug 4206909
* @summary Test basic functionality of DeflaterOutputStream and InflaterInputStream including flush
*/
import java.io.*;
import java.util.*;
import java.util.zip.*;
public class InflateIn_DeflateOut {
private static class PairedInputStream extends ByteArrayInputStream {
private PairedOutputStream out = null;
private Random random;
public PairedInputStream() {
// The ByteArrayInputStream needs to start with a buffer, but we
// need to set it to have no data
super(new byte[1]);
count = 0;
pos = 0;
random = new Random(new Date().getTime());
}
public void setPairedOutputStream(PairedOutputStream out) {
this.out = out;
}
private void maybeFlushPair() {
if (random.nextInt(100) < 10) {
out.flush();
}
}
public int read() {
maybeFlushPair();
return super.read();
}
public int read(byte b[], int off, int len) {
maybeFlushPair();
return super.read(b, off, len);
}
public void addBytes(byte[] bytes, int len) {
int oldavail = count - pos;
int newcount = oldavail + len;
byte[] newbuf = new byte[newcount];
System.arraycopy(buf, pos, newbuf, 0, oldavail);
System.arraycopy(bytes, 0, newbuf, oldavail, len);
pos = 0;
count = newcount;
buf = newbuf;
}
}
private static class PairedOutputStream extends ByteArrayOutputStream {
private PairedInputStream pairedStream = null;
public PairedOutputStream(PairedInputStream inputPair) {
super();
this.pairedStream = inputPair;
}
public void flush() {
if (count > 0) {
pairedStream.addBytes(buf, count);
reset();
}
}
public void close() {
flush();
}
}
private static boolean readFully(InputStream in, byte[] buf, int length)
throws IOException {
int pos = 0;
int n;
while ((n = in.read(buf, pos, length - pos)) > 0) {
pos += n;
if (pos == length) return true;
}
return false;
}
private static boolean readLineIfAvailable(InputStream in, StringBuilder sb)
throws IOException {
try {
while (in.available() > 0) {
int i = in.read();
if (i < 0) break;
char c = (char) (((byte) i) & 0xff);
sb.append(c);
if (c == '\n') return true;
}
} catch (EOFException e) {
// empty
}
return false;
}
/** Check that written, closed and read */
private static void WriteCloseRead() throws Throwable {
Random random = new Random(new Date().getTime());
PairedInputStream pis = new PairedInputStream();
InflaterInputStream iis = new InflaterInputStream(pis);
PairedOutputStream pos = new PairedOutputStream(pis);
pis.setPairedOutputStream(pos);
DeflaterOutputStream dos = new DeflaterOutputStream(pos, true);
byte[] data = new byte[random.nextInt(1024 * 1024)];
byte[] buf = new byte[data.length];
random.nextBytes(data);
dos.write(data);
dos.close();
check(readFully(iis, buf, buf.length));
check(Arrays.equals(data, buf));
}
/** Check that written, flushed and read */
private static void WriteFlushRead() throws Throwable {
Random random = new Random(new Date().getTime());
PairedInputStream pis = new PairedInputStream();
InflaterInputStream iis = new InflaterInputStream(pis);
PairedOutputStream pos = new PairedOutputStream(pis);
pis.setPairedOutputStream(pos);
DeflaterOutputStream dos = new DeflaterOutputStream(pos, true);
// Large writes
for (int x = 0; x < 200 ; x++) {
// byte[] data = new byte[random.nextInt(1024 * 1024)];
byte[] data = new byte[1024];
byte[] buf = new byte[data.length];
random.nextBytes(data);
dos.write(data);
dos.flush();
check(readFully(iis, buf, buf.length));
check(Arrays.equals(data, buf));
}
// Small writes
for (int x = 0; x < 2000 ; x++) {
byte[] data = new byte[random.nextInt(20) + 10];
byte[] buf = new byte[data.length];
random.nextBytes(data);
dos.write(data);
dos.flush();
if (!readFully(iis, buf, buf.length)) {
fail("Didn't read full buffer of " + buf.length);
}
check(Arrays.equals(data, buf));
}
String quit = "QUIT\r\n";
// Close it out
dos.write(quit.getBytes());
dos.close();
StringBuilder sb = new StringBuilder();
check(readLineIfAvailable(iis, sb));
equal(sb.toString(), quit);
}
/** Validate that we need to use flush at least once on a line
* oriented protocol */
private static void LineOrientedProtocol() throws Throwable {
PairedInputStream pis = new PairedInputStream();
InflaterInputStream iis = new InflaterInputStream(pis);
PairedOutputStream pos = new PairedOutputStream(pis);
pis.setPairedOutputStream(pos);
DeflaterOutputStream dos = new DeflaterOutputStream(pos, true);
boolean flushed = false;
int count = 0;
// Do at least a certain number of lines, but too many without a
// flush means this test isn't testing anything
while ((count < 10 && flushed) || (count < 1000 && !flushed)) {
String command = "PING " + count + "\r\n";
dos.write(command.getBytes());
StringBuilder buf = new StringBuilder();
if (!readLineIfAvailable(iis, buf)) {
flushed = true;
dos.flush();
check(readLineIfAvailable(iis, buf));
}
equal(buf.toString(), command);
count++;
}
check(flushed);
}
public static void realMain(String[] args) throws Throwable {
WriteCloseRead();
WriteFlushRead();
LineOrientedProtocol();
}
//--------------------- Infrastructure ---------------------------
static volatile int passed = 0, failed = 0;
static void pass() {passed++;}
static void fail() {failed++; Thread.dumpStack();}
static void fail(String msg) {System.out.println(msg); fail();}
static void unexpected(Throwable t) {failed++; t.printStackTrace();}
static void check(boolean cond) {if (cond) pass(); else fail();}
static void equal(Object x, Object y) {
if (x == null ? y == null : x.equals(y)) pass();
else fail(x + " not equal to " + y);}
public static void main(String[] args) throws Throwable {
try {realMain(args);} catch (Throwable t) {unexpected(t);}
System.out.println("\nPassed = " + passed + " failed = " + failed);
if (failed > 0) throw new AssertionError("Some tests failed");}
}