/** * */ public static byte[] readBytes(File file) throws IOException { if (file != null) { long fileSize = file.length(); if (fileSize > MAX_ARRAY_SIZE) { throw new IOException("file is too large"); } try (SeekableByteChannel byteChannel = Files.newByteChannel(file.toPath()); InputStream inputStream = Channels.newInputStream(byteChannel)) { int available = inputStream.available(); if (available > 0) { return readBytes(inputStream,available > fileSize ? available : (int)fileSize); } } } return EMPTY_BYTE_ARRAY; }
public static void main(String[] args) { Path file = Paths.get(System.getProperty("user.home") + "/test/myfile.txt"); try (SeekableByteChannel sbc = Files.newByteChannel(file)) { // |\longremark{newByteChannel默认返回只读的Channel}| ByteBuffer buf = ByteBuffer.allocate(10); // |\longremark{allocate创建一个指定字节的ByteBuffer,本例中,sbc这个Channel每次读取10个字节}| String encoding = System.getProperty("file.encoding"); // |\longremark{获得当前系统文件编码方式,以便读取文件字节后解码}| while (sbc.read(buf) > 0) { // |\longremark{从通道读数据到缓冲区}| buf.flip(); // |\longremark{切换缓冲区为读模式}| System.out.print(Charset.forName(encoding).decode(buf)); buf.clear(); // |\longremark{清空缓冲区,准备写入下一轮数据}| } } catch (IOException x) { System.out.println("caught exception: " + x); } }
private static byte[] hash(Path path) throws IOException { TlData tlData = InputFile.tlData.get(); MessageDigest digest = tlData.digest; ByteBuffer buffer = tlData.buffer; buffer.clear(); try (SeekableByteChannel channel = Files.newByteChannel(path)) { while (channel.read(buffer) != -1) { buffer.flip(); digest.update(buffer); buffer.clear(); } } return digest.digest(); }
/** * Reads an LC style string from a channel, which is a int32 length * plus a UTF-8 encoded string possibly ends with \0. * @throws IOException if there is a format error * @throws BufferUnderflowException if goes beyond the end */ private static String readStringWithLength(SeekableByteChannel chan) throws IOException { ByteBuffer bb = ByteBuffer.allocate(4); bb.order(ByteOrder.nativeOrder()); chan.read(bb); bb.flip(); int len = bb.getInt(); if (len > 1024) { // Memory attack? The string should be fairly short. throw new IOException("Invalid string length"); } bb = ByteBuffer.allocate(len); if (chan.read(bb) != len) { throw new IOException("Not enough string"); } byte[] data = bb.array(); return (data[len-1] == 0)? new String(data, 0, len-1, StandardCharsets.UTF_8): new String(data, StandardCharsets.UTF_8); }
@Override protected ZipFile newZipFile(final SeekableByteChannel sbc) throws IOException { final RaesReadOnlyChannel rroc = RaesReadOnlyChannel.create( raesParameters, new OneTimeSource(sbc)); try { if (rroc.size() < AUTHENTICATION_TRIGGER) // heuristic rroc.authenticate(); return new ZipFile(rroc); } catch (final Throwable ex) { try { rroc.close(); } catch (final IOException ex2) { ex.addSuppressed(ex2); } throw ex; } }
@Override @SuppressWarnings("ResultOfObjectAllocationIgnored") protected ZipFile newZipFile( final SeekableByteChannel channel, final Charset charset) throws IOException { Objects.requireNonNull(charset); new String(new byte[0], charset); // may throw UnsupportedEncodingExceoption! final RaesReadOnlyChannel rroc = RaesReadOnlyChannel.create( raesParameters, new OneTimeSource(channel)); try { if (rroc.size() < AUTHENTICATION_TRIGGER) // heuristic rroc.authenticate(); return new ZipFile(rroc, charset); } catch (final Throwable ex) { try { rroc.close(); } catch (final IOException ex2) { ex.addSuppressed(ex2); } throw ex; } }
private static void expunge(Path p, KerberosTime currTime) throws IOException { Path p2 = Files.createTempFile(p.getParent(), "rcache", null); try (SeekableByteChannel oldChan = Files.newByteChannel(p); SeekableByteChannel newChan = createNoClose(p2)) { long timeLimit = currTime.getSeconds() - readHeader(oldChan); while (true) { try { AuthTime at = AuthTime.readFrom(oldChan); if (at.ctime > timeLimit) { ByteBuffer bb = ByteBuffer.wrap(at.encode(true)); newChan.write(bb); } } catch (BufferUnderflowException e) { break; } } } makeMine(p2); Files.move(p2, p, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); }
/** * Creates a new RAES read-only channel. * * @param param the {@link RaesParameters} required to access the RAES * type actually found in the file. * If the class of this parameter does not match the required * parameter interface according to the RAES type found in the * file, but is an instance of the {@link RaesParametersProvider} * interface, then it gets queried to find the required RAES * parameters. * This algorithm gets recursively applied. * @param channel the channel for reading the RAES file from. * @return A new RAES read-only channel. * @throws RaesParametersException If no RAES parameter can be found which * match the type of RAES file in the given channel. * @throws RaesException If the source data is not RAES compatible. * @throws EOFException on unexpected end-of-file. * @throws IOException on any I/O error. */ @CreatesObligation private static RaesReadOnlyChannel create( final RaesParameters param, final @WillCloseWhenClosed SeekableByteChannel channel) throws RaesParametersException, RaesException, EOFException, IOException { final PowerBuffer header = PowerBuffer .allocate(HEADER_MIN_LEN) .littleEndian() .load(channel.position(0)); if (SIGNATURE != header.getUInt()) throw new RaesException("No RAES signature!"); final int type = header.getUByte(); switch (type) { case 0: return new Type0RaesReadOnlyChannel( parameters(Type0RaesParameters.class, param), channel); default: throw new RaesException("Unknown RAES type: " + type); } }
@Override public SeekableByteChannel channel(final OutputSocket<? extends Entry> peer) throws IOException { final IoBuffer buffer = entry.getPool().allocate(); try { IoSockets.copy(entry.input(), buffer.output()); } catch (final Throwable ex) { try { buffer.release(); } catch (final Throwable ex2) { ex.addSuppressed(ex2); } throw ex; } final class BufferReadOnlyChannel extends ReadOnlyChannel { boolean closed; @CreatesObligation BufferReadOnlyChannel() throws IOException { super(buffer.input().channel(peer)); // or .channel(null) } @Override public void close() throws IOException { if (!closed) { channel.close(); closed = true; buffer.release(); } } } return new BufferReadOnlyChannel(); }
private void parseDatHeader() throws IOException { if (parsed) return; try (SeekableByteChannel channel = Files.newByteChannel(datFile)) { channel.position(Math.abs(position)); CountingStream countingStream = new CountingStream(new BufferedInputStream(Channels.newInputStream(channel)), Long.MAX_VALUE); DataInputStream stream = new DataInputStream(countingStream); String status = readString(stream); String size = readString(stream); String msg = readString(stream); String contentType = readString(stream); String lastModified = readString(stream); String etag = readString(stream); for (;;) { String line = readString(stream); if (line.equals("HTS")) break; if (line.equals("SD")) { readString(stream); // supplementary data } } dataLen = Long.parseLong(readString(stream)); headerLen = countingStream.count; parsed = true; } }
ByteString read(CheckedFunction<Path, Path, IOException> resolver, long offset, long length) throws IOException { flush(); if (offset + length > getSize()) { throw new IOException("Failed to read: offset (=" + offset + " + length (=" + length + ") > size = " + getSize() + ", path=" + getRelativePath()); } try(final SeekableByteChannel in = Files.newByteChannel( resolver.apply(getRelativePath()), StandardOpenOption.READ)) { final ByteBuffer buffer = ByteBuffer.allocateDirect(FileStoreCommon.getChunkSize(length)); in.position(offset).read(buffer); buffer.flip(); return ByteString.copyFrom(buffer); } }
public static ByteBuffer ioResourceToByteBuffer(String resource, int bufferSize) throws IOException { ByteBuffer buffer; Path path = Paths.get(resource); if(Files.isReadable(path)) { try(SeekableByteChannel fc = Files.newByteChannel(path)){ buffer = BufferUtils.createByteBuffer((int)fc.size() + 1); while(fc.read(buffer) != -1); } }else{ try(InputStream source = Thread.currentThread().getContextClassLoader().getResourceAsStream(resource); ReadableByteChannel rbc = Channels.newChannel(source)){ buffer = BufferUtils.createByteBuffer(bufferSize); while(true){ int bytes = rbc.read(buffer); if(bytes == -1) break; if (buffer.remaining() == 0) buffer = resizeBuffer(buffer, buffer.capacity() * 2); } } } buffer.flip(); return buffer; }
public static List<BinaryTrade> readFromFile(Path p) throws IOException { System.out.println("Reading binary data to file ... "); List<BinaryTrade> trades = new ArrayList<>(); long start = System.nanoTime(); try (SeekableByteChannel channel = Files.newByteChannel(p, READ)) { ByteBuffer byteBuffer = ByteBuffer.allocateDirect(batchSize); int nRead; while ((nRead = channel.read(byteBuffer)) != -1) { if (nRead == 0) continue; byteBuffer.position(0); byteBuffer.limit(nRead); while (byteBuffer.hasRemaining()) { byte[] bytes = new byte[40]; byteBuffer.get(bytes, 0, 40); trades.add(new BinaryTrade(bytes)); } byteBuffer.clear(); } } System.out.printf("\tTime to read %,.1f million elements: %,.2f seconds%n%n", trades.size() / 1e6, (System.nanoTime() - start) / 1e9); return trades; }
public LineReader(final ReadableByteChannel channel) throws IOException { buf = ByteBuffer.allocate(BUF_SIZE); buf.flip(); boolean removeLine = false; // If we are not at the beginning of a line, we should ignore the current line. if (channel instanceof SeekableByteChannel) { SeekableByteChannel seekChannel = (SeekableByteChannel) channel; if (seekChannel.position() > 0) { // Start from one character back and read till we find a new line. seekChannel.position(seekChannel.position() - 1); removeLine = true; } nextLineStart = seekChannel.position(); } this.channel = channel; if (removeLine) { nextLineStart += readNextLine(new ByteArrayOutputStream()); } }
private static int csize(int p) throws Exception { try (SeekableByteChannel chan = Files.newByteChannel( Paths.get(cwd, dfl(p)), StandardOpenOption.READ)) { chan.position(6); int cc = 0; while (true) { try { if (AuthTime.readFrom(chan) != null) cc++; } catch (BufferUnderflowException e) { break; } } return cc; } catch (IOException ioe) { return 0; } }
private static int csize(int p) throws Exception { try (SeekableByteChannel chan = Files.newByteChannel( Paths.get(dfl(p)), StandardOpenOption.READ)) { chan.position(6); int cc = 0; while (true) { try { if (AuthTime.readFrom(chan) != null) cc++; } catch (BufferUnderflowException e) { break; } } return cc; } catch (IOException ioe) { return 0; } }
@Override public int available() throws IOException { if (channel instanceof SeekableByteChannel) { SeekableByteChannel sbc = (SeekableByteChannel) channel; long remainder = Math.max(0, sbc.size() - sbc.position()); return (remainder > Integer.MAX_VALUE) ? Integer.MAX_VALUE : (int)(remainder + buffer.remaining()); } return buffer.remaining(); }
public void testDeleteSnapshotWithCorruptedSnapshotFile() throws Exception { Client client = client(); Path repo = randomRepoPath(); logger.info("--> creating repository at {}", repo.toAbsolutePath()); assertAcked(client.admin().cluster().preparePutRepository("test-repo") .setType("fs").setSettings(Settings.builder() .put("location", repo) .put("compress", false) .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES))); createIndex("test-idx-1", "test-idx-2"); logger.info("--> indexing some data"); indexRandom(true, client().prepareIndex("test-idx-1", "doc").setSource("foo", "bar"), client().prepareIndex("test-idx-2", "doc").setSource("foo", "bar")); logger.info("--> creating snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setWaitForCompletion(true).setIndices("test-idx-*").get(); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); logger.info("--> truncate snapshot file to make it unreadable"); Path snapshotPath = repo.resolve("snap-" + createSnapshotResponse.getSnapshotInfo().snapshotId().getUUID() + ".dat"); try(SeekableByteChannel outChan = Files.newByteChannel(snapshotPath, StandardOpenOption.WRITE)) { outChan.truncate(randomInt(10)); } logger.info("--> delete snapshot"); client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap-1").get(); logger.info("--> make sure snapshot doesn't exist"); assertThrows(client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("test-snap-1"), SnapshotMissingException.class); logger.info("--> make sure that we can create the snapshot again"); createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setWaitForCompletion(true).setIndices("test-idx-*").get(); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); }
public static byte[] reverseRead(final Path path,Charset charset,long size) throws IOException { try (SeekableByteChannel sbc = Files.newByteChannel(path); InputStream in = Channels.newInputStream(sbc)) { long startIndex = sbc.size() - size; if (startIndex < 0) { startIndex = 0; } in.skip(startIndex); if (size > (long) MAX_BUFFER_SIZE) throw new OutOfMemoryError("Required array size too large"); return read(in, (int) size); } }
@Override public SeekableByteChannel newByteChannel(Path file, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException { triggerEx(file, "newByteChannel"); return Files.newByteChannel(unwrap(file), options, attrs); }
private RaesReadOnlyChannel newRaesReadOnlyChannel(final Path file) throws IOException { return RaesReadOnlyChannel.create( raesParameters, new AbstractSource() { @Override public SeekableByteChannel channel() throws IOException { return newByteChannel(file); } }); }
@CreatesObligation public ThrowingSeekableChannel( final @WillCloseWhenClosed SeekableByteChannel channel, final @CheckForNull FsThrowManager control) { super(channel); this.control = null != control ? control : FsTestConfig.get().getThrowControl(); }
@Override protected ZipFile newZipFile(SeekableByteChannel file) throws IOException { final ZipFile zf = new ZipFile(file); zf.setCryptoParameters(new WinZipAesCryptoParameters()); return zf.recoverLostEntries(); }
@Override protected ZipFile newZipFile( SeekableByteChannel file, Charset charset) throws IOException { final ZipFile zf = new ZipFile(file, charset); zf.setCryptoParameters(new WinZipAesCryptoParameters()); return zf.recoverLostEntries(); }
public static ByteBuffer ioResourceToByteBuffer(String resource, int bufferSize) throws IOException { ByteBuffer buffer; Path path = Paths.get(resource); if (Files.isReadable(path)) { try (SeekableByteChannel fc = Files.newByteChannel(path)) { buffer = BufferUtils.createByteBuffer((int) fc.size() + 1); while (fc.read(buffer) != -1) ; } } else { try ( InputStream source = Util.class.getResourceAsStream(resource); ReadableByteChannel rbc = Channels.newChannel(source)) { buffer = createByteBuffer(bufferSize); while (true) { int bytes = rbc.read(buffer); if (bytes == -1) { break; } if (buffer.remaining() == 0) { buffer = resizeBuffer(buffer, buffer.capacity() * 2); } } } } buffer.flip(); return buffer; }
@Override public SeekableByteChannel truncate(long size) throws IOException { try { channel.truncate(size); return this; } catch (IOException ex) { throw exception = ex; } }
@Override public SeekableByteChannel channel(final InputSocket<? extends Entry> peer) throws IOException { final FileNode buffer = begin(); final class Channel extends IOExceptionSeekableChannel { boolean closed; Channel() throws IOException { super(newByteChannel(buffer.getPath(), optionSet())); } @Override public void close() throws IOException { if (closed) return; super.close(); closed = true; FileOutputSocket.this.close(buffer, null == exception); } } // Channel try { append(buffer); return new Channel(); } catch (IOException ex) { throw release(ex, buffer); } }
/** * Constructs a new cipher read-only channel. * * @param cipher the initialized, seekable block cipher. * @param channel the seekable byte channel. */ @CreatesObligation public CipherReadOnlyChannel( final SeekableBlockCipher cipher, final @WillCloseWhenClosed SeekableByteChannel channel) { this(cipher, channel, Streams.BUFFER_SIZE); }
@Override public SeekableByteChannel position(final long pos) throws IOException { if (0 > pos) throw new IllegalArgumentException(); checkOpen(); this.pos = pos; return this; }
private void checkZipFileSignature(final SeekableByteChannel channel) throws IOException { final long sig = MutableBuffer .allocate(4) .littleEndian() .load(channel.position(preamble)) .getUInt(); // Constraint: A ZIP file must start with a Local File Header // or a (ZIP64) End Of Central Directory Record iff it's emtpy. if (LFH_SIG != sig && ZIP64_EOCDR_SIG != sig && EOCDR_SIG != sig) throw new ZipException( "Expected local file header or (ZIP64) end of central directory record!"); }
/** * Closes the file. * This closes any allocated input streams reading from this ZIP file. * * @throws IOException if an error occurs closing the file. */ @Override @DischargesObligation public void close() throws IOException { final SeekableByteChannel channel = this.channel; if (null != channel) { channel.close(); this.channel = null; } }
@CreatesObligation SafeBufferedReadOnlyChannel( final @WillCloseWhenClosed SeekableByteChannel channel, final long size) { super(channel); this.size = size; }
@Override public SeekableByteChannel newByteChannel( Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException { return promote(path).newByteChannel(options, attrs); }
/** * * @param resource * @param bufferSize * @return * @throws IOException */ public static ByteBuffer ioResourceToByteBuffer(String resource, int bufferSize) throws IOException { ByteBuffer buffer; Path path = Paths.get(resource); if (Files.isReadable(path)) { try (SeekableByteChannel fc = Files.newByteChannel(path)) { buffer = createByteBuffer((int)fc.size() + 1); while (fc.read(buffer) != -1) { ; } } } else { try ( InputStream source = IOUtil.class.getClassLoader().getResourceAsStream(resource); ReadableByteChannel rbc = Channels.newChannel(source) ) { buffer = createByteBuffer(bufferSize); while (true) { int bytes = rbc.read(buffer); if (bytes == -1) { break; } if (buffer.remaining() == 0) { buffer = resizeBuffer(buffer, buffer.capacity() * 2); } } } } buffer.flip(); return buffer; }
@Override public InputStream openStream() throws IOException { parseDatHeader(); SeekableByteChannel channel = Files.newByteChannel(datFile); channel.position(Math.abs(position) + headerLen); return new CountingStream(Channels.newInputStream(channel), dataLen); }
@Override public byte[] read() throws IOException { try (SeekableByteChannel channel = Files.newByteChannel(path, options)) { return com.google.common.io.Files.readFile( Channels.newInputStream(channel), channel.size()); } }
public static ByteBuffer ioResourceToByteBuffer(String resource, int bufferSize) throws IOException { ByteBuffer buffer; Path path = Paths.get(resource); if (Files.isReadable(path)) { try (SeekableByteChannel fc = Files.newByteChannel(path)) { buffer = BufferUtils.createByteBuffer((int)fc.size() + 1); while (fc.read(buffer) != -1) { ; } } } else { try ( InputStream source = Thread.class.getClassLoader().getResourceAsStream(resource); ReadableByteChannel rbc = Channels.newChannel(source) ) { buffer = BufferUtils.createByteBuffer(bufferSize); while (true) { int bytes = rbc.read(buffer); if (bytes == -1) { break; } if (buffer.remaining() == 0) { buffer = resizeBuffer(buffer, buffer.capacity() * 2); } } } } buffer.flip(); return buffer; }