static FSEditLogLoader.EditLogValidation validateEditLog(File file) throws IOException { EditLogFileInputStream in; try { in = new EditLogFileInputStream(file); in.getVersion(true); // causes us to read the header } catch (LogHeaderCorruptException e) { // If the header is malformed or the wrong value, this indicates a corruption LOG.warn("Log file " + file + " has no valid header", e); return new FSEditLogLoader.EditLogValidation(0, HdfsConstants.INVALID_TXID, true); } try { return FSEditLogLoader.validateEditLog(in); } finally { IOUtils.closeStream(in); } }
/** * @param file File being scanned and validated. * @param maxTxIdToScan Maximum Tx ID to try to scan. * The scan returns after reading this or a higher * ID. The file portion beyond this ID is * potentially being updated. * @return Result of the validation * @throws IOException */ static FSEditLogLoader.EditLogValidation scanEditLog(File file, long maxTxIdToScan, boolean verifyVersion) throws IOException { EditLogFileInputStream in; try { in = new EditLogFileInputStream(file); // read the header, initialize the inputstream, but do not check the // layoutversion in.getVersion(verifyVersion); } catch (LogHeaderCorruptException e) { LOG.warn("Log file " + file + " has no valid header", e); return new FSEditLogLoader.EditLogValidation(0, HdfsServerConstants.INVALID_TXID, true); } try { return FSEditLogLoader.scanEditLog(in, maxTxIdToScan); } finally { IOUtils.closeStream(in); } }
@Test public void testValidateEditLogWithCorruptHeader() throws IOException { File testDir = new File(TEST_DIR, "testValidateEditLogWithCorruptHeader"); SortedMap<Long, Long> offsetToTxId = Maps.newTreeMap(); File logFile = prepareUnfinalizedTestEditLog(testDir, 2, offsetToTxId); RandomAccessFile rwf = new RandomAccessFile(logFile, "rw"); try { rwf.seek(0); rwf.writeLong(42); // corrupt header } finally { rwf.close(); } EditLogValidation validation = EditLogFileInputStream.scanEditLog(logFile, Long.MAX_VALUE, true); assertTrue(validation.hasCorruptHeader()); }
@Test public void testValidateEditLogWithCorruptHeader() throws IOException { File testDir = new File(TEST_DIR, "testValidateEditLogWithCorruptHeader"); SortedMap<Long, Long> offsetToTxId = Maps.newTreeMap(); File logFile = prepareUnfinalizedTestEditLog(testDir, 2, offsetToTxId); RandomAccessFile rwf = new RandomAccessFile(logFile, "rw"); try { rwf.seek(0); rwf.writeLong(42); // corrupt header } finally { rwf.close(); } EditLogValidation validation = EditLogFileInputStream.validateEditLog(logFile); assertTrue(validation.hasCorruptHeader()); }
@Test public void testValidateEmptyEditLog() throws IOException { File testDir = new File(TEST_DIR, "testValidateEmptyEditLog"); SortedMap<Long, Long> offsetToTxId = Maps.newTreeMap(); File logFile = prepareUnfinalizedTestEditLog(testDir, 0, offsetToTxId); // Truncate the file so that there is nothing except the header and // layout flags section. truncateFile(logFile, 8); EditLogValidation validation = EditLogFileInputStream.validateEditLog(logFile); assertTrue(!validation.hasCorruptHeader()); assertEquals(HdfsConstants.INVALID_TXID, validation.getEndTxId()); }
@Test public void testValidateEmptyEditLog() throws IOException { File testDir = new File(TEST_DIR, "testValidateEmptyEditLog"); SortedMap<Long, Long> offsetToTxId = Maps.newTreeMap(); File logFile = prepareUnfinalizedTestEditLog(testDir, 0, offsetToTxId); // Truncate the file so that there is nothing except the header and // layout flags section. truncateFile(logFile, 8); EditLogValidation validation = EditLogFileInputStream.scanEditLog(logFile, Long.MAX_VALUE, true); assertTrue(!validation.hasCorruptHeader()); assertEquals(HdfsServerConstants.INVALID_TXID, validation.getEndTxId()); }
@Test public void testPreallocation() throws IOException { Configuration conf = new Configuration(); MiniDFSCluster cluster = new MiniDFSCluster(conf, 0, true, null); StorageDirectory sd = cluster.getNameNode().getFSImage() .storage.getStorageDir(0); File editLog = NNStorage.getInProgressEditsFile(sd, 0); EditLogValidation validation = EditLogFileInputStream.validateEditLog(editLog); assertEquals("Edit log should contain a header as valid length", HEADER_LEN, validation.getValidLength()); assertEquals(1, validation.getNumTransactions()); assertEquals("Edit log should have 1MB pre-allocated", PREALLOCATION_LENGTH, editLog.length()); cluster.getFileSystem().mkdirs(new Path("/tmp"), new FsPermission((short)777)); long oldLength = validation.getValidLength(); validation = EditLogFileInputStream.validateEditLog(editLog); assertTrue("Edit log should have more valid data after writing a txn " + "(was: " + oldLength + " now: " + validation.getValidLength() + ")", validation.getValidLength() > oldLength); assertEquals(2, validation.getNumTransactions()); assertEquals("Edit log should be 1MB long, plus 4 bytes for the version number", PREALLOCATION_LENGTH, editLog.length()); }
@Test public void testValidateEmptyEditLog() throws IOException { File testDir = new File(TEST_DIR, "testValidateEmptyEditLog"); SortedMap<Long, Long> offsetToTxId = Maps.newTreeMap(); File logFile = prepareUnfinalizedTestEditLog(testDir, 0, offsetToTxId); // Truncate the file so that there is nothing except the header truncateFile(logFile, 4); EditLogValidation validation = EditLogFileInputStream.validateEditLog(logFile); assertTrue(!validation.hasCorruptHeader()); assertEquals(HdfsConstants.INVALID_TXID, validation.getEndTxId()); }
/** * Count the number of valid transactions in a log. * This will update the lastTxId of the EditLogFile or * mark it as corrupt if it is. */ public void validateLog() throws IOException { EditLogValidation val = EditLogFileInputStream.validateEditLog(file); if (val.getNumTransactions() == 0) { markCorrupt(); } else { this.lastTxId = val.getEndTxId(); } }
static FSEditLogLoader.EditLogValidation scanEditLog(File file) throws IOException { EditLogFileInputStream in; try { in = new EditLogFileInputStream(file); // read the header, initialize the inputstream, but do not check the // layoutversion in.getVersion(false); } catch (LogHeaderCorruptException e) { LOG.warn("Log file " + file + " has no valid header", e); return new FSEditLogLoader.EditLogValidation(0, HdfsConstants.INVALID_TXID, true); } long lastPos = 0; long lastTxId = HdfsConstants.INVALID_TXID; long numValid = 0; try { while (true) { long txid = HdfsConstants.INVALID_TXID; lastPos = in.getPosition(); try { if ((txid = in.scanNextOp()) == HdfsConstants.INVALID_TXID) { break; } } catch (Throwable t) { FSImage.LOG.warn("Caught exception after scanning through " + numValid + " ops from " + in + " while determining its valid length. Position was " + lastPos, t); in.resync(); FSImage.LOG.warn("After resync, position is " + in.getPosition()); continue; } if (lastTxId == HdfsConstants.INVALID_TXID || txid > lastTxId) { lastTxId = txid; } numValid++; } return new EditLogValidation(lastPos, lastTxId, false); } finally { IOUtils.closeStream(in); } }
/** * Find out where the edit log ends. * This will update the lastTxId of the EditLogFile or * mark it as corrupt if it is. */ public void validateLog() throws IOException { EditLogValidation val = EditLogFileInputStream.validateEditLog(file); this.lastTxId = val.getEndTxId(); this.hasCorruptHeader = val.hasCorruptHeader(); }
public void scanLog() throws IOException { EditLogValidation val = EditLogFileInputStream.scanEditLog(file); this.lastTxId = val.getEndTxId(); this.hasCorruptHeader = val.hasCorruptHeader(); }