Skip to content

Commit a1e9a0b

Browse files
mruppanercpovirk
mruppaner
authored and
cpovirk
committed
Upgraded ByteStreams#copy(InputStream, OutputStream) to use the faster FileChannel if possible.
See also https://medium.com/@xunnan.xu/its-all-about-buffers-zero-copy-mmap-and-java-nio-50f2a1bfc05c for some background. RELNOTES=`io`: Upgraded `ByteStreams#copy(InputStream, OutputStream)` to use the faster `FileChannel` if possible. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=316177487
1 parent 14dd8fe commit a1e9a0b

File tree

6 files changed

+202
-24
lines changed

6 files changed

+202
-24
lines changed

‎android/guava-tests/test/com/google/common/io/ByteStreamsTest.java‎

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import java.io.ByteArrayOutputStream;
2424
import java.io.EOFException;
2525
import java.io.File;
26+
import java.io.FileInputStream;
2627
import java.io.FileOutputStream;
2728
import java.io.FilterInputStream;
2829
import java.io.IOException;
@@ -41,7 +42,7 @@
4142
*/
4243
public class ByteStreamsTest extends IoTestCase {
4344

44-
public void testCopyChannel() throws IOException {
45+
public void testCopy_channel() throws IOException {
4546
byte[] expected = newPreFilledByteArray(100);
4647
ByteArrayOutputStream out = new ByteArrayOutputStream();
4748
WritableByteChannel outChannel = Channels.newChannel(out);
@@ -51,7 +52,7 @@ public void testCopyChannel() throws IOException {
5152
assertThat(out.toByteArray()).isEqualTo(expected);
5253
}
5354

54-
public void testCopyFileChannel() throws IOException {
55+
public void testCopy_channel_fromFile() throws IOException {
5556
final int chunkSize = 14407; // Random prime, unlikely to match any internal chunk size
5657
ByteArrayOutputStream out = new ByteArrayOutputStream();
5758
WritableByteChannel outChannel = Channels.newChannel(out);
@@ -72,6 +73,68 @@ public void testCopyFileChannel() throws IOException {
7273
}
7374
}
7475

76+
public void testCopy_stream() throws IOException {
77+
byte[] expected = newPreFilledByteArray(100);
78+
ByteArrayOutputStream out = new ByteArrayOutputStream();
79+
80+
ByteStreams.copy(new ByteArrayInputStream(expected), out);
81+
82+
assertThat(out.toByteArray()).isEqualTo(expected);
83+
}
84+
85+
public void testCopy_stream_files_emptyDestination() throws IOException {
86+
byte[] expected = new byte[] {0, 1, 2};
87+
File inputFile = createTempFile(expected);
88+
File outputFile = createTempFile();
89+
90+
try (FileInputStream inputStream = new FileInputStream(inputFile);
91+
FileOutputStream outputStream = new FileOutputStream(outputFile)) {
92+
ByteStreams.copy(inputStream, outputStream);
93+
}
94+
95+
assertThat(Files.asByteSource(outputFile).read()).isEqualTo(expected);
96+
}
97+
98+
public void testCopy_stream_files_appendDestination() throws IOException {
99+
File inputFile = createTempFile(new byte[] {3, 4, 5});
100+
File outputFile = createTempFile(new byte[] {0, 1, 2});
101+
102+
try (FileInputStream inputStream = new FileInputStream(inputFile);
103+
FileOutputStream outputStream = new FileOutputStream(outputFile, /* append= */ true)) {
104+
ByteStreams.copy(inputStream, outputStream);
105+
}
106+
107+
assertThat(Files.asByteSource(outputFile).read()).isEqualTo(new byte[] {0, 1, 2, 3, 4, 5});
108+
}
109+
110+
public void testCopy_stream_files_additionalWrites_emptyDestination() throws IOException {
111+
File inputFile = createTempFile(new byte[] {0, 1, 2});
112+
File outputFile = createTempFile();
113+
114+
try (FileInputStream inputStream = new FileInputStream(inputFile);
115+
FileOutputStream outputStream = new FileOutputStream(outputFile)) {
116+
outputStream.write(new byte[] {0, 0});
117+
ByteStreams.copy(inputStream, outputStream);
118+
outputStream.write(new byte[] {2, 2});
119+
}
120+
121+
assertThat(Files.asByteSource(outputFile).read()).isEqualTo(new byte[] {0, 0, 0, 1, 2, 2, 2});
122+
}
123+
124+
public void testCopy_stream_files_additionalWrites_appendDestination() throws IOException {
125+
File inputFile = createTempFile(new byte[] {0, 1, 2});
126+
File outputFile = createTempFile(new byte[] {0});
127+
128+
try (FileInputStream inputStream = new FileInputStream(inputFile);
129+
FileOutputStream outputStream = new FileOutputStream(outputFile, /* append= */ true)) {
130+
outputStream.write(new byte[] {0});
131+
ByteStreams.copy(inputStream, outputStream);
132+
outputStream.write(new byte[] {2, 2});
133+
}
134+
135+
assertThat(Files.asByteSource(outputFile).read()).isEqualTo(new byte[] {0, 0, 0, 1, 2, 2, 2});
136+
}
137+
75138
public void testReadFully() throws IOException {
76139
byte[] b = new byte[10];
77140

‎android/guava-tests/test/com/google/common/io/IoTestCase.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,18 @@ protected final File createTempFile() throws IOException {
139139
return File.createTempFile("test", null, getTempDir());
140140
}
141141

142+
/**
143+
* Creates a new temp file in the temp directory returned by {@link #getTempDir()}. The file will
144+
* be deleted in the tear-down for this test.
145+
*
146+
* @param content which should be written to the file
147+
*/
148+
protected final File createTempFile(byte[] content) throws IOException {
149+
File file = File.createTempFile("test", null, getTempDir());
150+
Files.write(content, file);
151+
return file;
152+
}
153+
142154
/** Returns a byte array of length size that has values 0 .. size - 1. */
143155
static byte[] newPreFilledByteArray(int size) {
144156
return newPreFilledByteArray(0, size);

‎android/guava/src/com/google/common/io/ByteStreams.java‎

Lines changed: 24 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
import java.io.DataOutput;
3131
import java.io.DataOutputStream;
3232
import java.io.EOFException;
33+
import java.io.FileInputStream;
34+
import java.io.FileOutputStream;
3335
import java.io.FilterInputStream;
3436
import java.io.IOException;
3537
import java.io.InputStream;
@@ -103,6 +105,15 @@ private ByteStreams() {}
103105
public static long copy(InputStream from, OutputStream to) throws IOException {
104106
checkNotNull(from);
105107
checkNotNull(to);
108+
109+
// Use java.nio.channels in case we're copying from file to file.
110+
// Copying through channels happens ideally in the kernel space and therefore faster.
111+
if (from instanceof FileInputStream && to instanceof FileOutputStream) {
112+
FileChannel fromChannel = ((FileInputStream) from).getChannel();
113+
FileChannel toChannel = ((FileOutputStream) to).getChannel();
114+
return copyFileChannel(fromChannel, toChannel);
115+
}
116+
106117
byte[] buf = createBuffer();
107118
long total = 0;
108119
while (true) {
@@ -130,16 +141,7 @@ public static long copy(ReadableByteChannel from, WritableByteChannel to) throws
130141
checkNotNull(from);
131142
checkNotNull(to);
132143
if (from instanceof FileChannel) {
133-
FileChannel sourceChannel = (FileChannel) from;
134-
long oldPosition = sourceChannel.position();
135-
long position = oldPosition;
136-
long copied;
137-
do {
138-
copied = sourceChannel.transferTo(position, ZERO_COPY_CHUNK_SIZE, to);
139-
position += copied;
140-
sourceChannel.position(position);
141-
} while (copied > 0 || position < sourceChannel.size());
142-
return position - oldPosition;
144+
return copyFileChannel((FileChannel) from, to);
143145
}
144146

145147
ByteBuffer buf = ByteBuffer.wrap(createBuffer());
@@ -154,6 +156,18 @@ public static long copy(ReadableByteChannel from, WritableByteChannel to) throws
154156
return total;
155157
}
156158

159+
private static long copyFileChannel(FileChannel from, WritableByteChannel to) throws IOException {
160+
long oldPosition = from.position();
161+
long position = oldPosition;
162+
long copied;
163+
do {
164+
copied = from.transferTo(position, ZERO_COPY_CHUNK_SIZE, to);
165+
position += copied;
166+
from.position(position);
167+
} while (copied > 0 || position < from.size());
168+
return position - oldPosition;
169+
}
170+
157171
/** Max array length on JVM. */
158172
private static final int MAX_ARRAY_LEN = Integer.MAX_VALUE - 8;
159173

‎guava-tests/test/com/google/common/io/ByteStreamsTest.java‎

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import java.io.ByteArrayOutputStream;
2424
import java.io.EOFException;
2525
import java.io.File;
26+
import java.io.FileInputStream;
2627
import java.io.FileOutputStream;
2728
import java.io.FilterInputStream;
2829
import java.io.IOException;
@@ -41,7 +42,7 @@
4142
*/
4243
public class ByteStreamsTest extends IoTestCase {
4344

44-
public void testCopyChannel() throws IOException {
45+
public void testCopy_channel() throws IOException {
4546
byte[] expected = newPreFilledByteArray(100);
4647
ByteArrayOutputStream out = new ByteArrayOutputStream();
4748
WritableByteChannel outChannel = Channels.newChannel(out);
@@ -51,7 +52,7 @@ public void testCopyChannel() throws IOException {
5152
assertThat(out.toByteArray()).isEqualTo(expected);
5253
}
5354

54-
public void testCopyFileChannel() throws IOException {
55+
public void testCopy_channel_fromFile() throws IOException {
5556
final int chunkSize = 14407; // Random prime, unlikely to match any internal chunk size
5657
ByteArrayOutputStream out = new ByteArrayOutputStream();
5758
WritableByteChannel outChannel = Channels.newChannel(out);
@@ -72,6 +73,68 @@ public void testCopyFileChannel() throws IOException {
7273
}
7374
}
7475

76+
public void testCopy_stream() throws IOException {
77+
byte[] expected = newPreFilledByteArray(100);
78+
ByteArrayOutputStream out = new ByteArrayOutputStream();
79+
80+
ByteStreams.copy(new ByteArrayInputStream(expected), out);
81+
82+
assertThat(out.toByteArray()).isEqualTo(expected);
83+
}
84+
85+
public void testCopy_stream_files_emptyDestination() throws IOException {
86+
byte[] expected = new byte[] {0, 1, 2};
87+
File inputFile = createTempFile(expected);
88+
File outputFile = createTempFile();
89+
90+
try (FileInputStream inputStream = new FileInputStream(inputFile);
91+
FileOutputStream outputStream = new FileOutputStream(outputFile)) {
92+
ByteStreams.copy(inputStream, outputStream);
93+
}
94+
95+
assertThat(Files.asByteSource(outputFile).read()).isEqualTo(expected);
96+
}
97+
98+
public void testCopy_stream_files_appendDestination() throws IOException {
99+
File inputFile = createTempFile(new byte[] {3, 4, 5});
100+
File outputFile = createTempFile(new byte[] {0, 1, 2});
101+
102+
try (FileInputStream inputStream = new FileInputStream(inputFile);
103+
FileOutputStream outputStream = new FileOutputStream(outputFile, /* append= */ true)) {
104+
ByteStreams.copy(inputStream, outputStream);
105+
}
106+
107+
assertThat(Files.asByteSource(outputFile).read()).isEqualTo(new byte[] {0, 1, 2, 3, 4, 5});
108+
}
109+
110+
public void testCopy_stream_files_additionalWrites_emptyDestination() throws IOException {
111+
File inputFile = createTempFile(new byte[] {0, 1, 2});
112+
File outputFile = createTempFile();
113+
114+
try (FileInputStream inputStream = new FileInputStream(inputFile);
115+
FileOutputStream outputStream = new FileOutputStream(outputFile)) {
116+
outputStream.write(new byte[] {0, 0});
117+
ByteStreams.copy(inputStream, outputStream);
118+
outputStream.write(new byte[] {2, 2});
119+
}
120+
121+
assertThat(Files.asByteSource(outputFile).read()).isEqualTo(new byte[] {0, 0, 0, 1, 2, 2, 2});
122+
}
123+
124+
public void testCopy_stream_files_additionalWrites_appendDestination() throws IOException {
125+
File inputFile = createTempFile(new byte[] {0, 1, 2});
126+
File outputFile = createTempFile(new byte[] {0});
127+
128+
try (FileInputStream inputStream = new FileInputStream(inputFile);
129+
FileOutputStream outputStream = new FileOutputStream(outputFile, /* append= */ true)) {
130+
outputStream.write(new byte[] {0});
131+
ByteStreams.copy(inputStream, outputStream);
132+
outputStream.write(new byte[] {2, 2});
133+
}
134+
135+
assertThat(Files.asByteSource(outputFile).read()).isEqualTo(new byte[] {0, 0, 0, 1, 2, 2, 2});
136+
}
137+
75138
public void testReadFully() throws IOException {
76139
byte[] b = new byte[10];
77140

‎guava-tests/test/com/google/common/io/IoTestCase.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,18 @@ protected final File createTempFile() throws IOException {
139139
return File.createTempFile("test", null, getTempDir());
140140
}
141141

142+
/**
143+
* Creates a new temp file in the temp directory returned by {@link #getTempDir()}. The file will
144+
* be deleted in the tear-down for this test.
145+
*
146+
* @param content which should be written to the file
147+
*/
148+
protected final File createTempFile(byte[] content) throws IOException {
149+
File file = File.createTempFile("test", null, getTempDir());
150+
Files.write(content, file);
151+
return file;
152+
}
153+
142154
/** Returns a byte array of length size that has values 0 .. size - 1. */
143155
static byte[] newPreFilledByteArray(int size) {
144156
return newPreFilledByteArray(0, size);

‎guava/src/com/google/common/io/ByteStreams.java‎

Lines changed: 24 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
import java.io.DataOutput;
3131
import java.io.DataOutputStream;
3232
import java.io.EOFException;
33+
import java.io.FileInputStream;
34+
import java.io.FileOutputStream;
3335
import java.io.FilterInputStream;
3436
import java.io.IOException;
3537
import java.io.InputStream;
@@ -103,6 +105,15 @@ private ByteStreams() {}
103105
public static long copy(InputStream from, OutputStream to) throws IOException {
104106
checkNotNull(from);
105107
checkNotNull(to);
108+
109+
// Use java.nio.channels in case we're copying from file to file.
110+
// Copying through channels happens ideally in the kernel space and therefore faster.
111+
if (from instanceof FileInputStream && to instanceof FileOutputStream) {
112+
FileChannel fromChannel = ((FileInputStream) from).getChannel();
113+
FileChannel toChannel = ((FileOutputStream) to).getChannel();
114+
return copyFileChannel(fromChannel, toChannel);
115+
}
116+
106117
byte[] buf = createBuffer();
107118
long total = 0;
108119
while (true) {
@@ -130,16 +141,7 @@ public static long copy(ReadableByteChannel from, WritableByteChannel to) throws
130141
checkNotNull(from);
131142
checkNotNull(to);
132143
if (from instanceof FileChannel) {
133-
FileChannel sourceChannel = (FileChannel) from;
134-
long oldPosition = sourceChannel.position();
135-
long position = oldPosition;
136-
long copied;
137-
do {
138-
copied = sourceChannel.transferTo(position, ZERO_COPY_CHUNK_SIZE, to);
139-
position += copied;
140-
sourceChannel.position(position);
141-
} while (copied > 0 || position < sourceChannel.size());
142-
return position - oldPosition;
144+
return copyFileChannel((FileChannel) from, to);
143145
}
144146

145147
ByteBuffer buf = ByteBuffer.wrap(createBuffer());
@@ -154,6 +156,18 @@ public static long copy(ReadableByteChannel from, WritableByteChannel to) throws
154156
return total;
155157
}
156158

159+
private static long copyFileChannel(FileChannel from, WritableByteChannel to) throws IOException {
160+
long oldPosition = from.position();
161+
long position = oldPosition;
162+
long copied;
163+
do {
164+
copied = from.transferTo(position, ZERO_COPY_CHUNK_SIZE, to);
165+
position += copied;
166+
from.position(position);
167+
} while (copied > 0 || position < from.size());
168+
return position - oldPosition;
169+
}
170+
157171
/** Max array length on JVM. */
158172
private static final int MAX_ARRAY_LEN = Integer.MAX_VALUE - 8;
159173

0 commit comments

Comments
 (0)