在处理多个千兆字节文件时,我注意到了一些奇怪的事情:似乎使用文件通道从文件读取到分配有allocateDirect的重复使用的ByteBuffer对象中,比从MappedByteBuffer中读取要慢得多,实际上,它甚至比读取字节中的记录还要慢。使用常规读取调用的数组!
我期望它(几乎)与从mapedbytebuffers读取的速度一样快,因为我的ByteBuffer是使用allocateDirect分配的,因此读取应该直接结束于我的字节缓冲区,而没有任何中间副本。
我现在的问题是:我做错了什么?还是字节缓冲区+文件通道确实比常规的io / mmap慢?
在下面的示例代码中,我还添加了一些代码,可将读取的内容转换为长值,因为这是我的真实代码经常执行的操作。我希望ByteBuffer getLong()方法比我自己的字节shuffeler快得多。
测试结果:mmap:3.828字节缓冲区:55.097常规I / O:38.175
import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.channels.FileChannel.MapMode; import java.nio.MappedByteBuffer; class testbb { static final int size = 536870904, n = size / 24; static public long byteArrayToLong(byte [] in, int offset) { return ((((((((long)(in[offset + 0] & 0xff) << 8) | (long)(in[offset + 1] & 0xff)) << 8 | (long)(in[offset + 2] & 0xff)) << 8 | (long)(in[offset + 3] & 0xff)) << 8 | (long)(in[offset + 4] & 0xff)) << 8 | (long)(in[offset + 5] & 0xff)) << 8 | (long)(in[offset + 6] & 0xff)) << 8 | (long)(in[offset + 7] & 0xff); } public static void main(String [] args) throws IOException { long start; RandomAccessFile fileHandle; FileChannel fileChannel; // create file fileHandle = new RandomAccessFile("file.dat", "rw"); byte [] buffer = new byte[24]; for(int index=0; index<n; index++) fileHandle.write(buffer); fileChannel = fileHandle.getChannel(); // mmap() MappedByteBuffer mbb = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, size); byte [] buffer1 = new byte[24]; start = System.currentTimeMillis(); for(int index=0; index<n; index++) { mbb.position(index * 24); mbb.get(buffer1, 0, 24); long dummy1 = byteArrayToLong(buffer1, 0); long dummy2 = byteArrayToLong(buffer1, 8); long dummy3 = byteArrayToLong(buffer1, 16); } System.out.println("mmap: " + (System.currentTimeMillis() - start) / 1000.0); // bytebuffer ByteBuffer buffer2 = ByteBuffer.allocateDirect(24); start = System.currentTimeMillis(); for(int index=0; index<n; index++) { buffer2.rewind(); fileChannel.read(buffer2, index * 24); buffer2.rewind(); // need to rewind it to be able to use it long dummy1 = buffer2.getLong(); long dummy2 = buffer2.getLong(); long dummy3 = buffer2.getLong(); } System.out.println("bytebuffer: " + (System.currentTimeMillis() - start) / 1000.0); // regular i/o byte [] buffer3 = new byte[24]; start = System.currentTimeMillis(); for(int index=0; index<n; index++) { fileHandle.seek(index * 24); fileHandle.read(buffer3); long dummy1 = byteArrayToLong(buffer1, 0); long dummy2 = byteArrayToLong(buffer1, 8); long dummy3 = byteArrayToLong(buffer1, 16); } System.out.println("regular i/o: " + (System.currentTimeMillis() - start) / 1000.0); } }
由于装入大块然后进行处理不是一个选择(我将在各处读取数据),我认为我应该坚持使用MappedByteBuffer。谢谢大家的建议。
我相信你只是在做微优化,这 可能只是 无关紧要 (www.codinghorror.com) 。
下面是一个具有较大缓冲区并删除了多余seek/ setPosition调用的版本。
seek
setPosition
mmap: 1.358 bytebuffer: 0.922 regular i/o: 1.387
mmap: 1.336 bytebuffer: 1.62 regular i/o: 1.467
mmap: 3.262 bytebuffer: 106.676 regular i/o: 90.903
这是代码:
import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; import java.nio.channels.FileChannel.MapMode; import java.nio.MappedByteBuffer; class Testbb2 { /** Buffer a whole lot of long values at the same time. */ static final int BUFFSIZE = 0x800 * 8; // 8192 static final int DATASIZE = 0x8000 * BUFFSIZE; static public long byteArrayToLong(byte [] in, int offset) { return ((((((((long)(in[offset + 0] & 0xff) << 8) | (long)(in[offset + 1] & 0xff)) << 8 | (long)(in[offset + 2] & 0xff)) << 8 | (long)(in[offset + 3] & 0xff)) << 8 | (long)(in[offset + 4] & 0xff)) << 8 | (long)(in[offset + 5] & 0xff)) << 8 | (long)(in[offset + 6] & 0xff)) << 8 | (long)(in[offset + 7] & 0xff); } public static void main(String [] args) throws IOException { long start; RandomAccessFile fileHandle; FileChannel fileChannel; // Sanity check - this way the convert-to-long loops don't need extra bookkeeping like BUFFSIZE / 8. if ((DATASIZE % BUFFSIZE) > 0 || (DATASIZE % 8) > 0) { throw new IllegalStateException("DATASIZE should be a multiple of 8 and BUFFSIZE!"); } int pos; int nDone; // create file File testFile = new File("file.dat"); fileHandle = new RandomAccessFile("file.dat", "rw"); if (testFile.exists() && testFile.length() >= DATASIZE) { System.out.println("File exists"); } else { testFile.delete(); System.out.println("Preparing file"); byte [] buffer = new byte[BUFFSIZE]; pos = 0; nDone = 0; while (pos < DATASIZE) { fileHandle.write(buffer); pos += buffer.length; } System.out.println("File prepared"); } fileChannel = fileHandle.getChannel(); // mmap() MappedByteBuffer mbb = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, DATASIZE); byte [] buffer1 = new byte[BUFFSIZE]; mbb.position(0); start = System.currentTimeMillis(); pos = 0; while (pos < DATASIZE) { mbb.get(buffer1, 0, BUFFSIZE); // This assumes BUFFSIZE is a multiple of 8. for (int i = 0; i < BUFFSIZE; i += 8) { long dummy = byteArrayToLong(buffer1, i); } pos += BUFFSIZE; } System.out.println("mmap: " + (System.currentTimeMillis() - start) / 1000.0); // bytebuffer ByteBuffer buffer2 = ByteBuffer.allocateDirect(BUFFSIZE); // buffer2.order(ByteOrder.nativeOrder()); buffer2.order(); fileChannel.position(0); start = System.currentTimeMillis(); pos = 0; nDone = 0; while (pos < DATASIZE) { buffer2.rewind(); fileChannel.read(buffer2); buffer2.rewind(); // need to rewind it to be able to use it // This assumes BUFFSIZE is a multiple of 8. for (int i = 0; i < BUFFSIZE; i += 8) { long dummy = buffer2.getLong(); } pos += BUFFSIZE; } System.out.println("bytebuffer: " + (System.currentTimeMillis() - start) / 1000.0); // regular i/o fileHandle.seek(0); byte [] buffer3 = new byte[BUFFSIZE]; start = System.currentTimeMillis(); pos = 0; while (pos < DATASIZE && nDone != -1) { nDone = 0; while (nDone != -1 && nDone < BUFFSIZE) { nDone = fileHandle.read(buffer3, nDone, BUFFSIZE - nDone); } // This assumes BUFFSIZE is a multiple of 8. for (int i = 0; i < BUFFSIZE; i += 8) { long dummy = byteArrayToLong(buffer3, i); } pos += nDone; } System.out.println("regular i/o: " + (System.currentTimeMillis() - start) / 1000.0); } }