@Override public void visitOp(FSEditLogOp op) throws IOException { try { op.outputToXml(contentHandler); } catch (SAXException e) { throw new IOException("SAX error: " + e.getMessage()); } }
/** * Construct BookKeeper edit log input stream. * Starts reading from firstBookKeeperEntry. This allows the stream * to take a shortcut during recovery, as it doesn't have to read * every edit log transaction to find out what the last one is. */ BookKeeperEditLogInputStream(LedgerHandle lh, EditLogLedgerMetadata metadata, long firstBookKeeperEntry) throws IOException { this.lh = lh; this.firstTxId = metadata.getFirstTxId(); this.lastTxId = metadata.getLastTxId(); this.logVersion = metadata.getDataLayoutVersion(); this.inProgress = metadata.isInProgress(); if (firstBookKeeperEntry < 0 || firstBookKeeperEntry > lh.getLastAddConfirmed()) { throw new IOException("Invalid first bk entry to read: " + firstBookKeeperEntry + ", LAC: " + lh.getLastAddConfirmed()); } BufferedInputStream bin = new BufferedInputStream( new LedgerInputStream(lh, firstBookKeeperEntry)); tracker = new FSEditLogLoader.PositionTrackingInputStream(bin); DataInputStream in = new DataInputStream(tracker); reader = new FSEditLogOp.Reader(in, tracker, logVersion); }
@Test public void testSimpleWrite() throws Exception { NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-simplewrite"), nsi); bkjm.format(nsi); EditLogOutputStream out = bkjm.startLogSegment(1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); for (long i = 1 ; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); op.setTransactionId(i); out.write(op); } out.close(); bkjm.finalizeLogSegment(1, 100); String zkpath = bkjm.finalizedLedgerZNode(1, 100); assertNotNull(zkc.exists(zkpath, false)); assertNull(zkc.exists(bkjm.inprogressZNode(1), false)); }
@Test public void testNumberOfTransactions() throws Exception { NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-txncount"), nsi); bkjm.format(nsi); EditLogOutputStream out = bkjm.startLogSegment(1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); for (long i = 1 ; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); op.setTransactionId(i); out.write(op); } out.close(); bkjm.finalizeLogSegment(1, 100); long numTrans = bkjm.getNumberOfTransactions(1, true); assertEquals(100, numTrans); }
private String startAndFinalizeLogSegment(BookKeeperJournalManager bkjm, int startTxid, int endTxid) throws IOException, KeeperException, InterruptedException { EditLogOutputStream out = bkjm.startLogSegment(startTxid, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); for (long i = startTxid; i <= endTxid; i++) { FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); op.setTransactionId(i); out.write(op); } out.close(); // finalize the inprogress_1 log segment. bkjm.finalizeLogSegment(startTxid, endTxid); String zkpath1 = bkjm.finalizedLedgerZNode(startTxid, endTxid); assertNotNull(zkc.exists(zkpath1, false)); assertNull(zkc.exists(bkjm.inprogressZNode(startTxid), false)); return zkpath1; }
/** * Construct BookKeeper edit log input stream. * Starts reading from firstBookKeeperEntry. This allows the stream * to take a shortcut during recovery, as it doesn't have to read * every edit log transaction to find out what the last one is. */ BookKeeperEditLogInputStream(LedgerHandle lh, EditLogLedgerMetadata metadata, long firstBookKeeperEntry) throws IOException { this.lh = lh; this.firstTxId = metadata.getFirstTxId(); this.lastTxId = metadata.getLastTxId(); this.logVersion = metadata.getDataLayoutVersion(); this.inProgress = metadata.isInProgress(); if (firstBookKeeperEntry < 0 || firstBookKeeperEntry > lh.getLastAddConfirmed()) { throw new IOException("Invalid first bk entry to read: " + firstBookKeeperEntry + ", LAC: " + lh.getLastAddConfirmed()); } BufferedInputStream bin = new BufferedInputStream( new LedgerInputStream(lh, firstBookKeeperEntry)); tracker = new FSEditLogLoader.PositionTrackingInputStream(bin); DataInputStream in = new DataInputStream(tracker); reader = FSEditLogOp.Reader.create(in, tracker, logVersion); }
/** * Converts a Transaction Log operation into a NamespaceNotification * object. * @param op the Transaction Log operation. * @return the NamespaceNotification object or null if the type of * the operation isn't supported to be transformed into a * NamespaceNotification. */ static NamespaceNotification createNotification(FSEditLogOp op) { switch (op.opCode) { case OP_ADD: return new NamespaceNotification(((AddOp)op).path, EventType.FILE_ADDED.getByteValue(), op.getTransactionId()); case OP_CLOSE: return new NamespaceNotification(((CloseOp)op).path, EventType.FILE_CLOSED.getByteValue(), op.getTransactionId()); case OP_DELETE: return new NamespaceNotification(((DeleteOp)op).path, EventType.NODE_DELETED.getByteValue(), op.getTransactionId()); case OP_MKDIR: return new NamespaceNotification(((MkdirOp)op).path, EventType.DIR_ADDED.getByteValue(), op.getTransactionId()); default: return null; } }
URLLogInputStream(AsyncLogger logger, long firstTxId, int httpTimeout) throws LogHeaderCorruptException, IOException { log = new URLLog(logger, firstTxId, httpTimeout); fStream = log.getInputStream(0); lastValidTxId = log.getLastValidTxId(); super.setIsInProgress(log.getIsInProgress()); BufferedInputStream bin = new BufferedInputStream(fStream); tracker = new FSEditLogLoader.PositionTrackingInputStream(bin); DataInputStream in = new DataInputStream(tracker); try { logVersion = readLogVersion(in); } catch (EOFException eofe) { throw new LogHeaderCorruptException("No header found in log"); } reader = new FSEditLogOp.Reader(in, logVersion); this.firstTxId = firstTxId; this.disabled = false; // set initial position (version is 4 bytes) this.currentPosition = tracker.getPos(); }
@Test public void testSimpleWrite() throws Exception { NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-simplewrite"), nsi); bkjm.format(nsi); EditLogOutputStream out = bkjm.startLogSegment(1); for (long i = 1 ; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); op.setTransactionId(i); out.write(op); } out.close(); bkjm.finalizeLogSegment(1, 100); String zkpath = bkjm.finalizedLedgerZNode(1, 100); assertNotNull(zkc.exists(zkpath, false)); assertNull(zkc.exists(bkjm.inprogressZNode(1), false)); }
@Test public void testNumberOfTransactions() throws Exception { NamespaceInfo nsi = newNSInfo(); BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, BKJMUtil.createJournalURI("/hdfsjournal-txncount"), nsi); bkjm.format(nsi); EditLogOutputStream out = bkjm.startLogSegment(1); for (long i = 1 ; i <= 100; i++) { FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); op.setTransactionId(i); out.write(op); } out.close(); bkjm.finalizeLogSegment(1, 100); long numTrans = bkjm.getNumberOfTransactions(1, true); assertEquals(100, numTrans); }
private String startAndFinalizeLogSegment(BookKeeperJournalManager bkjm, int startTxid, int endTxid) throws IOException, KeeperException, InterruptedException { EditLogOutputStream out = bkjm.startLogSegment(startTxid); for (long i = startTxid; i <= endTxid; i++) { FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); op.setTransactionId(i); out.write(op); } out.close(); // finalize the inprogress_1 log segment. bkjm.finalizeLogSegment(startTxid, endTxid); String zkpath1 = bkjm.finalizedLedgerZNode(startTxid, endTxid); assertNotNull(zkc.exists(zkpath1, false)); assertNull(zkc.exists(bkjm.inprogressZNode(startTxid), false)); return zkpath1; }
public static byte[] createTxnData(int startTxn, int numTxns) throws Exception { DataOutputBuffer buf = new DataOutputBuffer(); FSEditLogOp.Writer writer = new FSEditLogOp.Writer(buf); for (long txid = startTxn; txid < startTxn + numTxns; txid++) { FSEditLogOp op = NameNodeAdapter.createMkdirOp("tx " + txid); op.setTransactionId(txid); writer.writeOp(op); } return Arrays.copyOf(buf.getData(), buf.getLength()); }
