private static long getApkFileChecksum(Context context) { String apkPath = context.getPackageCodePath(); Long chksum = null; try { // Open the file and build a CRC32 checksum. FileInputStream fis = new FileInputStream(new File(apkPath)); CRC32 chk = new CRC32(); CheckedInputStream cis = new CheckedInputStream(fis, chk); byte[] buff = new byte[80]; while (cis.read(buff) >= 0) ; chksum = chk.getValue(); } catch (Exception e) { e.printStackTrace(); } return chksum; }
private long calculateChecksum(URL layer) { if (layer == null) { return -1; } try { InputStream is = layer.openStream(); try { CheckedInputStream cis = new CheckedInputStream(is, new CRC32()); // Compute the CRC32 checksum byte[] buf = new byte[1024]; while (cis.read(buf) >= 0) { } cis.close(); return cis.getChecksum().getValue(); } finally { is.close(); } } catch (IOException e) { return -1; } }
/** return if checksum matches for a snapshot **/ private boolean getCheckSum(FileSnap snap, File snapFile) throws IOException { DataTree dt = new DataTree(); Map<Long, Integer> sessions = new ConcurrentHashMap<Long, Integer>(); InputStream snapIS = new BufferedInputStream(new FileInputStream( snapFile)); CheckedInputStream crcIn = new CheckedInputStream(snapIS, new Adler32()); InputArchive ia = BinaryInputArchive.getArchive(crcIn); try { snap.deserialize(dt, sessions, ia); } catch (IOException ie) { // we failed on the most recent snapshot // must be incomplete // try reading the next one // after corrupting snapIS.close(); crcIn.close(); throw ie; } long checksum = crcIn.getChecksum().getValue(); long val = ia.readLong("val"); snapIS.close(); crcIn.close(); return (val != checksum); }
public static long getCRC32(String File) { try { FileInputStream fis = new FileInputStream(File); CRC32 crc = new CRC32(); CheckedInputStream cis = new CheckedInputStream(fis, crc); while (cis.read(THROWAWAY_BUFFER, 0, THROWAWAY_BUFFER.length) != -1) ; cis.close(); return crc.getValue(); } catch (Exception e) { e.printStackTrace(); return -1; } }
/** * Construct the reader * @param in The stream to read from. * @param logVersion The version of the data coming from the stream. */ public Reader(DataInputStream in, StreamLimiter limiter, int logVersion) { this.logVersion = logVersion; if (NameNodeLayoutVersion.supports( LayoutVersion.Feature.EDITS_CHESKUM, logVersion)) { this.checksum = DataChecksum.newCrc32(); } else { this.checksum = null; } // It is possible that the logVersion is actually a future layoutversion // during the rolling upgrade (e.g., the NN gets upgraded first). We // assume future layout will also support length of editlog op. this.supportEditLogLength = NameNodeLayoutVersion.supports( NameNodeLayoutVersion.Feature.EDITLOG_LENGTH, logVersion) || logVersion < NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION; if (this.checksum != null) { this.in = new DataInputStream( new CheckedInputStream(in, this.checksum)); } else { this.in = in; } this.limiter = limiter; this.cache = new OpInstanceCache(); this.maxOpSize = DFSConfigKeys.DFS_NAMENODE_MAX_OP_SIZE_DEFAULT; }
private static void validateFileWithChecksum(FileSystem fs, Path filePath, BackupFileInfo backupFileInfo) throws IOException { final CheckedInputStream cin = new CheckedInputStream(fs.open(filePath), new CRC32()); final BufferedReader reader = new BufferedReader(new InputStreamReader(cin)); final ObjectMapper objectMapper = new ObjectMapper(); String line; long records = 0; // parse records just to make sure formatting is correct while ((line = reader.readLine()) != null) { objectMapper.readValue(line, BackupRecord.class); ++records; } cin.close(); long found = cin.getChecksum().getValue(); if (backupFileInfo.getChecksum() != found) { throw new IOException(format("Corrupt backup data file %s. Expected checksum %x, found %x", filePath, backupFileInfo.getChecksum(), found)); } if (backupFileInfo.getRecords() != records) { throw new IOException(format("Corrupt backup data file %s. Expected records %x, found %x", filePath, backupFileInfo.getRecords(), records)); } }
public static String createPackageChecksum(String fp) throws IOException { Path path = Paths.get(fp); try (CheckedInputStream is = new CheckedInputStream(Files.newInputStream(path), new Adler32())) { byte[] buf = new byte[1024*1024]; int total = 0; int c = 0; while (total < 100*1024*1024 && (c = is.read(buf)) >= 0) { total += c; } ByteBuffer bb = ByteBuffer.allocate(Long.BYTES); bb.putLong(path.toFile().length()); buf = bb.array(); is.getChecksum().update(buf, 0, buf.length); return Long.toHexString(is.getChecksum().getValue()); } }
private SFVInfo getSFVFile(Slave slave, String path) throws IOException { BufferedReader reader = null; CRC32 checksum = null; try { File file = slave.getRoots().getFile(path); checksum = new CRC32(); reader = new BufferedReader(new InputStreamReader(new CheckedInputStream(new FileInputStream(file), checksum))); SFVInfo sfvInfo = SFVInfo.importSFVInfoFromFile(reader); sfvInfo.setSFVFileName(file.getName()); sfvInfo.setChecksum(checksum.getValue()); return sfvInfo; } finally { if (reader != null) { reader.close(); } } }
private static void uncompress() { FileInputStream fis; try { fis = new FileInputStream("./dir/demo.zip"); CheckedInputStream cis = new CheckedInputStream(fis,new Adler32()); ZipInputStream zis = new ZipInputStream(cis); BufferedInputStream bis = new BufferedInputStream(zis); ZipEntry zipENtry ; while ((zipENtry = zis.getNextEntry()) != null){ System.err.println("read file--->" + zipENtry); int x; while ((x = bis.read()) != -1){ System.err.println(x); } } } catch (Exception e) { e.printStackTrace(); } }
/** * Construct the reader * @param in The stream to read from. * @param logVersion The version of the data coming from the stream. */ @SuppressWarnings("deprecation") public Reader(DataInputStream in, int logVersion) { this.logVersion = logVersion; if (LayoutVersion.supports(Feature.EDITS_CHESKUM, logVersion)) { this.checksum = FSEditLog.getChecksumForRead(); } else { this.checksum = null; } if (this.checksum != null) { this.in = new DataInputStream( new CheckedInputStream(in, this.checksum)); } else { this.in = in; } }
/** * Construct the reader * @param in The stream to read from. * @param logVersion The version of the data coming from the stream. */ @SuppressWarnings("deprecation") public Reader(DataInputStream in, StreamLimiter limiter, int logVersion) { this.logVersion = logVersion; if (LayoutVersion.supports(Feature.EDITS_CHESKUM, logVersion)) { this.checksum = new PureJavaCrc32(); } else { this.checksum = null; } if (this.checksum != null) { this.in = new DataInputStream( new CheckedInputStream(in, this.checksum)); } else { this.in = in; } this.limiter = limiter; this.cache = new OpInstanceCache(); this.maxOpSize = DFSConfigKeys.DFS_NAMENODE_MAX_OP_SIZE_DEFAULT; }
public void copyFileToCache(Path inFilePath, Object cache_key) throws IOException { int inBufferSize = 32*1024; int copyBufferSize = 128*1024; try ( CheckedInputStream is = new CheckedInputStream(new BufferedInputStream(Files.newInputStream(inFilePath),inBufferSize),new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new EhcacheOutputStream(cache, cache_key),new CRC32()) ) { System.out.println("============ testCopyFileToCacheWithBuffer ===================="); long start = System.nanoTime();; pipeStreamsWithBuffer(is, os, copyBufferSize); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("============================================"); } }
public void copyCacheToFile(Object cache_key, Path outFilePath) throws IOException { int copyBufferSize = 512 * 1024; //copy buffer size try ( CheckedInputStream is = new CheckedInputStream(new EhcacheInputStream(cache, cache_key),new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new BufferedOutputStream(Files.newOutputStream(outFilePath)), new CRC32()) ) { System.out.println("============ copyCacheToFileUsingStreamDefaultBuffers ===================="); long start = System.nanoTime();; pipeStreamsWithBuffer(is, os, copyBufferSize); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double) (end - start) / 1000000) + " millis"); System.out.println("============================================"); } }
@Before public void copyFileToCache() throws Exception { int inBufferSize = 32 * 1024; int copyBufferSize = 128 * 1024; try ( CheckedInputStream is = new CheckedInputStream(new BufferedInputStream(Files.newInputStream(IN_FILE_PATH),inBufferSize),new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new EhcacheOutputStream(cache, cache_key),new CRC32()) ) { System.out.println("============ copyFileToCache ===================="); long start = System.nanoTime();; pipeStreamsWithBuffer(is, os, copyBufferSize); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("============================================"); this.fileCheckSum = is.getChecksum().getValue(); Assert.assertEquals(is.getChecksum().getValue(), os.getChecksum().getValue()); } }
@Test public void copyCacheToFileUsingStreamSmallerCopyBuffer() throws Exception { int inBufferSize = 128 * 1024; //ehcache input stream internal buffer int outBufferSize = 128 * 1024; int copyBufferSize = 64 * 1024; //copy buffer size *smaller* than ehcache input stream internal buffer to make sure it works that way try ( CheckedInputStream is = new CheckedInputStream(new EhcacheInputStream(cache, cache_key, inBufferSize),new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new BufferedOutputStream(Files.newOutputStream(OUT_FILE_PATH),outBufferSize), new CRC32()) ) { System.out.println("============ copyCacheToFileUsingStreamSmallerCopyBuffer ===================="); long start = System.nanoTime();; pipeStreamsWithBuffer(is, os, copyBufferSize); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("============================================"); Assert.assertEquals(fileCheckSum, os.getChecksum().getValue()); Assert.assertEquals(is.getChecksum().getValue(), fileCheckSum); Assert.assertEquals(is.getChecksum().getValue(), os.getChecksum().getValue()); } }
@Test public void copyCacheToFileUsingStreamLargerCopyBuffer() throws Exception { int inBufferSize = 128 * 1024; //ehcache input stream internal buffer int outBufferSize = 128 * 1024; int copyBufferSize = 357 * 1024; //copy buffer size *larger* than ehcache input stream internal buffer to make sure it works that way try ( CheckedInputStream is = new CheckedInputStream(new EhcacheInputStream(cache, cache_key, inBufferSize),new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new BufferedOutputStream(Files.newOutputStream(OUT_FILE_PATH),outBufferSize), new CRC32()) ) { System.out.println("============ copyCacheToFileUsingStreamLargerCopyBuffer ===================="); long start = System.nanoTime();; pipeStreamsWithBuffer(is, os, copyBufferSize); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("============================================"); Assert.assertEquals(fileCheckSum, os.getChecksum().getValue()); Assert.assertEquals(is.getChecksum().getValue(), fileCheckSum); Assert.assertEquals(is.getChecksum().getValue(), os.getChecksum().getValue()); } }
@Test public void copyCacheToFileUsingStreamDefaultBuffers() throws Exception { int copyBufferSize = 512 * 1024; //copy buffer size try ( CheckedInputStream is = new CheckedInputStream(new EhcacheInputStream(cache, cache_key),new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new BufferedOutputStream(Files.newOutputStream(OUT_FILE_PATH)), new CRC32()) ) { System.out.println("============ copyCacheToFileUsingStreamDefaultBuffers ===================="); long start = System.nanoTime();; pipeStreamsWithBuffer(is, os, copyBufferSize); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("============================================"); Assert.assertEquals(fileCheckSum, os.getChecksum().getValue()); Assert.assertEquals(is.getChecksum().getValue(), fileCheckSum); Assert.assertEquals(is.getChecksum().getValue(), os.getChecksum().getValue()); } }
@Test public void copyCacheToFileUsingStreamDefaultBuffersByteByByte() throws Exception { try ( CheckedInputStream is = new CheckedInputStream(new EhcacheInputStream(cache, cache_key),new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new BufferedOutputStream(Files.newOutputStream(OUT_FILE_PATH)), new CRC32()) ) { System.out.println("============ copyCacheToFileUsingStreamDefaultBuffersByteByByte ===================="); long start = System.nanoTime();; pipeStreamsByteByByte(is, os); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("============================================"); Assert.assertEquals(fileCheckSum, os.getChecksum().getValue()); Assert.assertEquals(is.getChecksum().getValue(), fileCheckSum); Assert.assertEquals(is.getChecksum().getValue(), os.getChecksum().getValue()); } }
@Test public void copyCacheToFileNoCacheKey() throws Exception { int copyBufferSize = 512 * 1024; //copy buffer size final String cacheKey = "something-else"; try ( CheckedInputStream is = new CheckedInputStream(new EhcacheInputStream(cache, cacheKey),new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new BufferedOutputStream(Files.newOutputStream(OUT_FILE_PATH)), new CRC32()) ) { System.out.println("============ copyCacheToFileNoCacheKey ===================="); long start = System.nanoTime();; pipeStreamsWithBuffer(is, os, copyBufferSize); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("============================================"); Assert.assertEquals(0, os.getChecksum().getValue()); Assert.assertEquals(is.getChecksum().getValue(), 0); Assert.assertEquals(is.getChecksum().getValue(), os.getChecksum().getValue()); } }
@Test public void testCopyFileToFile() throws Exception { try ( CheckedInputStream is = new CheckedInputStream(new BufferedInputStream(Files.newInputStream(IN_FILE_PATH)), new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new BufferedOutputStream(Files.newOutputStream(OUT_FILE_PATH)), new CRC32()) ) { System.out.println("============ testCopyFileToFile ===================="); int copyBufferSize = 32*1024; long start = System.nanoTime();; pipeStreamsWithBuffer(is, os, copyBufferSize); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("============================================"); Assert.assertEquals(is.getChecksum().getValue(), os.getChecksum().getValue()); } }
@Test public void testCopyFileToCacheByteByByte() throws IOException { int inBufferSize = 32*1024; try ( CheckedInputStream is = new CheckedInputStream(new BufferedInputStream(Files.newInputStream(IN_FILE_PATH),inBufferSize),new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new EhcacheOutputStream(cache, cache_key),new CRC32()) ) { System.out.println("============ testCopyFileToCacheByteByByte ===================="); long start = System.nanoTime();; pipeStreamsByteByByte(is, os); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("============================================"); Assert.assertEquals(is.getChecksum().getValue(), os.getChecksum().getValue()); } }
@Test public void testCopyFileToCacheWithBuffer() throws IOException { int inBufferSize = 32*1024; int copyBufferSize = 128*1024; try ( CheckedInputStream is = new CheckedInputStream(new BufferedInputStream(Files.newInputStream(IN_FILE_PATH),inBufferSize),new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new EhcacheOutputStream(cache, cache_key),new CRC32()) ) { System.out.println("============ testCopyFileToCacheWithBuffer ===================="); long start = System.nanoTime();; pipeStreamsWithBuffer(is, os, copyBufferSize); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("============================================"); Assert.assertEquals(is.getChecksum().getValue(), os.getChecksum().getValue()); } }
@Test public void testCopyFileToCacheInOneShot() throws IOException { int inBufferSize = 32*1024; try ( CheckedInputStream is = new CheckedInputStream(new BufferedInputStream(Files.newInputStream(IN_FILE_PATH),inBufferSize),new CRC32()); CheckedOutputStream os = new CheckedOutputStream(new EhcacheOutputStream(cache, cache_key),new CRC32()); ByteArrayOutputStream bos = new ByteArrayOutputStream(new Double(IN_FILE_SIZE * 1.2).intValue()) ) { System.out.println("============ testCopyFileToCacheInOneShot ===================="); //first stream from file to ByteArrayOutputStream pipeStreamsByteByByte(is, bos); byte[] fullBytes = bos.toByteArray(); long start = System.nanoTime();; os.write(fullBytes); long end = System.nanoTime();; System.out.println("Execution Time = " + formatD.format((double)(end - start) / 1000000) + " millis"); System.out.println("============================================"); Assert.assertEquals(is.getChecksum().getValue(), os.getChecksum().getValue()); } }
/** * Find the Zip CRC32 checksum in the desired base. * Mediaflux uses this algorithm with base 16 for setting its content checksum * This algorithm copes with large files * * @param f * @param radix * @return * @throws Throwable */ public static String getCRC32 (File f, int radix) throws Throwable { FileInputStream file = new FileInputStream(f); CheckedInputStream check = new CheckedInputStream(file, new CRC32()); BufferedInputStream in = new BufferedInputStream(check); while (in.read() != -1) { // Read file in completely } in.close(); long n = check.getChecksum().getValue(); check.close(); return Long.toString(n, radix); }
public void test_read() throws Exception { // testing that the return by skip is valid InputStream checkInput = Support_Resources .getStream("hyts_checkInput.txt"); CheckedInputStream checkIn = new CheckedInputStream(checkInput, new CRC32()); checkIn.read(); checkIn.close(); try { checkIn.read(); fail("IOException expected."); } catch (IOException ee) { // expected } checkInput.close(); }
public void test_read$byteII() throws Exception { // testing that the return by skip is valid InputStream checkInput = Support_Resources .getStream("hyts_checkInput.txt"); CheckedInputStream checkIn = new CheckedInputStream(checkInput, new CRC32()); byte buff[] = new byte[50]; checkIn.read(buff, 10, 5); checkIn.close(); try { checkIn.read(buff, 10, 5); fail("IOException expected."); } catch (IOException ee) { // expected } checkInput.close(); }
/** * Computes a value for a file content with the specified checksum algorithm. * @param file File object for which the value should be evaluated. * @return The value for that file */ @Override public String getValue(File file) { initChecksum(); if (file.canRead()) { checksum.reset(); try (CheckedInputStream check = new CheckedInputStream( new BufferedInputStream(Files.newInputStream(file.toPath())), checksum)) { // Read the file while (check.read() != -1) { } return Long.toString(check.getChecksum().getValue()); } catch (Exception ignored) { } } return null; }
