Java 类org.apache.hadoop.hdfs.server.namenode.FSEditLogOp 实例源码

项目:hadoop    文件:XmlEditsVisitor.java   
@Override
public void visitOp(FSEditLogOp op) throws IOException {
  try {
    op.outputToXml(contentHandler);
  }
  catch (SAXException e) {
    throw new IOException("SAX error: " + e.getMessage());
  }
}
项目:hadoop    文件:BookKeeperEditLogInputStream.java   
/**
 * Construct BookKeeper edit log input stream. 
 * Starts reading from firstBookKeeperEntry. This allows the stream
 * to take a shortcut during recovery, as it doesn't have to read
 * every edit log transaction to find out what the last one is.
 */
BookKeeperEditLogInputStream(LedgerHandle lh, EditLogLedgerMetadata metadata,
                             long firstBookKeeperEntry) 
    throws IOException {
  this.lh = lh;
  this.firstTxId = metadata.getFirstTxId();
  this.lastTxId = metadata.getLastTxId();
  this.logVersion = metadata.getDataLayoutVersion();
  this.inProgress = metadata.isInProgress();

  if (firstBookKeeperEntry < 0
      || firstBookKeeperEntry > lh.getLastAddConfirmed()) {
    throw new IOException("Invalid first bk entry to read: "
        + firstBookKeeperEntry + ", LAC: " + lh.getLastAddConfirmed());
  }
  BufferedInputStream bin = new BufferedInputStream(
      new LedgerInputStream(lh, firstBookKeeperEntry));
  tracker = new FSEditLogLoader.PositionTrackingInputStream(bin);
  DataInputStream in = new DataInputStream(tracker);

  reader = new FSEditLogOp.Reader(in, tracker, logVersion);
}
项目:hadoop    文件:TestBookKeeperJournalManager.java   
@Test
public void testSimpleWrite() throws Exception {
  NamespaceInfo nsi = newNSInfo();
  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-simplewrite"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);

  String zkpath = bkjm.finalizedLedgerZNode(1, 100);

  assertNotNull(zkc.exists(zkpath, false));
  assertNull(zkc.exists(bkjm.inprogressZNode(1), false));
}
项目:hadoop    文件:TestBookKeeperJournalManager.java   
@Test
public void testNumberOfTransactions() throws Exception {
  NamespaceInfo nsi = newNSInfo();

  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-txncount"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);

  long numTrans = bkjm.getNumberOfTransactions(1, true);
  assertEquals(100, numTrans);
}
项目:hadoop    文件:TestBookKeeperJournalManager.java   
private String startAndFinalizeLogSegment(BookKeeperJournalManager bkjm,
    int startTxid, int endTxid) throws IOException, KeeperException,
    InterruptedException {
  EditLogOutputStream out = bkjm.startLogSegment(startTxid,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = startTxid; i <= endTxid; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  // finalize the inprogress_1 log segment.
  bkjm.finalizeLogSegment(startTxid, endTxid);
  String zkpath1 = bkjm.finalizedLedgerZNode(startTxid, endTxid);
  assertNotNull(zkc.exists(zkpath1, false));
  assertNull(zkc.exists(bkjm.inprogressZNode(startTxid), false));
  return zkpath1;
}
项目:aliyun-oss-hadoop-fs    文件:BookKeeperEditLogInputStream.java   
/**
 * Construct BookKeeper edit log input stream. 
 * Starts reading from firstBookKeeperEntry. This allows the stream
 * to take a shortcut during recovery, as it doesn't have to read
 * every edit log transaction to find out what the last one is.
 */
BookKeeperEditLogInputStream(LedgerHandle lh, EditLogLedgerMetadata metadata,
                             long firstBookKeeperEntry) 
    throws IOException {
  this.lh = lh;
  this.firstTxId = metadata.getFirstTxId();
  this.lastTxId = metadata.getLastTxId();
  this.logVersion = metadata.getDataLayoutVersion();
  this.inProgress = metadata.isInProgress();

  if (firstBookKeeperEntry < 0
      || firstBookKeeperEntry > lh.getLastAddConfirmed()) {
    throw new IOException("Invalid first bk entry to read: "
        + firstBookKeeperEntry + ", LAC: " + lh.getLastAddConfirmed());
  }
  BufferedInputStream bin = new BufferedInputStream(
      new LedgerInputStream(lh, firstBookKeeperEntry));
  tracker = new FSEditLogLoader.PositionTrackingInputStream(bin);
  DataInputStream in = new DataInputStream(tracker);

  reader = FSEditLogOp.Reader.create(in, tracker, logVersion);
}
项目:aliyun-oss-hadoop-fs    文件:TestBookKeeperJournalManager.java   
@Test
public void testSimpleWrite() throws Exception {
  NamespaceInfo nsi = newNSInfo();
  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-simplewrite"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);

  String zkpath = bkjm.finalizedLedgerZNode(1, 100);

  assertNotNull(zkc.exists(zkpath, false));
  assertNull(zkc.exists(bkjm.inprogressZNode(1), false));
}
项目:aliyun-oss-hadoop-fs    文件:TestBookKeeperJournalManager.java   
@Test
public void testNumberOfTransactions() throws Exception {
  NamespaceInfo nsi = newNSInfo();

  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-txncount"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);

  long numTrans = bkjm.getNumberOfTransactions(1, true);
  assertEquals(100, numTrans);
}
项目:aliyun-oss-hadoop-fs    文件:TestBookKeeperJournalManager.java   
private String startAndFinalizeLogSegment(BookKeeperJournalManager bkjm,
    int startTxid, int endTxid) throws IOException, KeeperException,
    InterruptedException {
  EditLogOutputStream out = bkjm.startLogSegment(startTxid,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = startTxid; i <= endTxid; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  // finalize the inprogress_1 log segment.
  bkjm.finalizeLogSegment(startTxid, endTxid);
  String zkpath1 = bkjm.finalizedLedgerZNode(startTxid, endTxid);
  assertNotNull(zkc.exists(zkpath1, false));
  assertNull(zkc.exists(bkjm.inprogressZNode(startTxid), false));
  return zkpath1;
}
项目:big-c    文件:BookKeeperEditLogInputStream.java   
/**
 * Construct BookKeeper edit log input stream. 
 * Starts reading from firstBookKeeperEntry. This allows the stream
 * to take a shortcut during recovery, as it doesn't have to read
 * every edit log transaction to find out what the last one is.
 */
BookKeeperEditLogInputStream(LedgerHandle lh, EditLogLedgerMetadata metadata,
                             long firstBookKeeperEntry) 
    throws IOException {
  this.lh = lh;
  this.firstTxId = metadata.getFirstTxId();
  this.lastTxId = metadata.getLastTxId();
  this.logVersion = metadata.getDataLayoutVersion();
  this.inProgress = metadata.isInProgress();

  if (firstBookKeeperEntry < 0
      || firstBookKeeperEntry > lh.getLastAddConfirmed()) {
    throw new IOException("Invalid first bk entry to read: "
        + firstBookKeeperEntry + ", LAC: " + lh.getLastAddConfirmed());
  }
  BufferedInputStream bin = new BufferedInputStream(
      new LedgerInputStream(lh, firstBookKeeperEntry));
  tracker = new FSEditLogLoader.PositionTrackingInputStream(bin);
  DataInputStream in = new DataInputStream(tracker);

  reader = new FSEditLogOp.Reader(in, tracker, logVersion);
}
项目:big-c    文件:TestBookKeeperJournalManager.java   
@Test
public void testSimpleWrite() throws Exception {
  NamespaceInfo nsi = newNSInfo();
  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-simplewrite"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);

  String zkpath = bkjm.finalizedLedgerZNode(1, 100);

  assertNotNull(zkc.exists(zkpath, false));
  assertNull(zkc.exists(bkjm.inprogressZNode(1), false));
}
项目:big-c    文件:TestBookKeeperJournalManager.java   
@Test
public void testNumberOfTransactions() throws Exception {
  NamespaceInfo nsi = newNSInfo();

  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-txncount"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);

  long numTrans = bkjm.getNumberOfTransactions(1, true);
  assertEquals(100, numTrans);
}
项目:big-c    文件:TestBookKeeperJournalManager.java   
private String startAndFinalizeLogSegment(BookKeeperJournalManager bkjm,
    int startTxid, int endTxid) throws IOException, KeeperException,
    InterruptedException {
  EditLogOutputStream out = bkjm.startLogSegment(startTxid,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = startTxid; i <= endTxid; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  // finalize the inprogress_1 log segment.
  bkjm.finalizeLogSegment(startTxid, endTxid);
  String zkpath1 = bkjm.finalizedLedgerZNode(startTxid, endTxid);
  assertNotNull(zkc.exists(zkpath1, false));
  assertNull(zkc.exists(bkjm.inprogressZNode(startTxid), false));
  return zkpath1;
}
项目:hadoop-2.6.0-cdh5.4.3    文件:BookKeeperEditLogInputStream.java   
/**
 * Construct BookKeeper edit log input stream. 
 * Starts reading from firstBookKeeperEntry. This allows the stream
 * to take a shortcut during recovery, as it doesn't have to read
 * every edit log transaction to find out what the last one is.
 */
BookKeeperEditLogInputStream(LedgerHandle lh, EditLogLedgerMetadata metadata,
                             long firstBookKeeperEntry) 
    throws IOException {
  this.lh = lh;
  this.firstTxId = metadata.getFirstTxId();
  this.lastTxId = metadata.getLastTxId();
  this.logVersion = metadata.getDataLayoutVersion();
  this.inProgress = metadata.isInProgress();

  if (firstBookKeeperEntry < 0
      || firstBookKeeperEntry > lh.getLastAddConfirmed()) {
    throw new IOException("Invalid first bk entry to read: "
        + firstBookKeeperEntry + ", LAC: " + lh.getLastAddConfirmed());
  }
  BufferedInputStream bin = new BufferedInputStream(
      new LedgerInputStream(lh, firstBookKeeperEntry));
  tracker = new FSEditLogLoader.PositionTrackingInputStream(bin);
  DataInputStream in = new DataInputStream(tracker);

  reader = new FSEditLogOp.Reader(in, tracker, logVersion);
}
项目:hadoop-2.6.0-cdh5.4.3    文件:TestBookKeeperJournalManager.java   
@Test
public void testSimpleWrite() throws Exception {
  NamespaceInfo nsi = newNSInfo();
  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-simplewrite"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);

  String zkpath = bkjm.finalizedLedgerZNode(1, 100);

  assertNotNull(zkc.exists(zkpath, false));
  assertNull(zkc.exists(bkjm.inprogressZNode(1), false));
}
项目:hadoop-2.6.0-cdh5.4.3    文件:TestBookKeeperJournalManager.java   
@Test
public void testNumberOfTransactions() throws Exception {
  NamespaceInfo nsi = newNSInfo();

  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-txncount"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);

  long numTrans = bkjm.getNumberOfTransactions(1, true);
  assertEquals(100, numTrans);
}
项目:hadoop-2.6.0-cdh5.4.3    文件:TestBookKeeperJournalManager.java   
private String startAndFinalizeLogSegment(BookKeeperJournalManager bkjm,
    int startTxid, int endTxid) throws IOException, KeeperException,
    InterruptedException {
  EditLogOutputStream out = bkjm.startLogSegment(startTxid,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = startTxid; i <= endTxid; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  // finalize the inprogress_1 log segment.
  bkjm.finalizeLogSegment(startTxid, endTxid);
  String zkpath1 = bkjm.finalizedLedgerZNode(startTxid, endTxid);
  assertNotNull(zkc.exists(zkpath1, false));
  assertNull(zkc.exists(bkjm.inprogressZNode(startTxid), false));
  return zkpath1;
}
项目:hadoop-EAR    文件:ServerLogReaderUtil.java   
/**
 * Converts a Transaction Log operation into a NamespaceNotification
 * object.
 * @param op the Transaction Log operation.
 * @return the NamespaceNotification object or null if the type of
 *         the operation isn't supported to be transformed into a
 *         NamespaceNotification.
 */
static NamespaceNotification createNotification(FSEditLogOp op) {
  switch (op.opCode) {
    case OP_ADD:
      return new NamespaceNotification(((AddOp)op).path,
          EventType.FILE_ADDED.getByteValue(), op.getTransactionId());

    case OP_CLOSE:
      return new NamespaceNotification(((CloseOp)op).path,
          EventType.FILE_CLOSED.getByteValue(), op.getTransactionId());

    case OP_DELETE:
      return new NamespaceNotification(((DeleteOp)op).path,
          EventType.NODE_DELETED.getByteValue(), op.getTransactionId());

    case OP_MKDIR:
      return new NamespaceNotification(((MkdirOp)op).path,
          EventType.DIR_ADDED.getByteValue(), op.getTransactionId());
    default:
      return null;
  }
}
项目:hadoop-EAR    文件:URLLogInputStream.java   
URLLogInputStream(AsyncLogger logger, long firstTxId, int httpTimeout)
    throws LogHeaderCorruptException, IOException {
  log = new URLLog(logger, firstTxId, httpTimeout);
  fStream = log.getInputStream(0);

  lastValidTxId = log.getLastValidTxId();
  super.setIsInProgress(log.getIsInProgress());

  BufferedInputStream bin = new BufferedInputStream(fStream);  
  tracker = new FSEditLogLoader.PositionTrackingInputStream(bin);  
  DataInputStream in = new DataInputStream(tracker);

  try {
    logVersion = readLogVersion(in);
  } catch (EOFException eofe) {
    throw new LogHeaderCorruptException("No header found in log");
  }
  reader = new FSEditLogOp.Reader(in, logVersion);
  this.firstTxId = firstTxId;
  this.disabled = false;
  // set initial position (version is 4 bytes)
  this.currentPosition = tracker.getPos();
}
项目:hadoop-plus    文件:BookKeeperEditLogInputStream.java   
/**
 * Construct BookKeeper edit log input stream. 
 * Starts reading from firstBookKeeperEntry. This allows the stream
 * to take a shortcut during recovery, as it doesn't have to read
 * every edit log transaction to find out what the last one is.
 */
BookKeeperEditLogInputStream(LedgerHandle lh, EditLogLedgerMetadata metadata,
                             long firstBookKeeperEntry) 
    throws IOException {
  this.lh = lh;
  this.firstTxId = metadata.getFirstTxId();
  this.lastTxId = metadata.getLastTxId();
  this.logVersion = metadata.getDataLayoutVersion();
  this.inProgress = metadata.isInProgress();

  if (firstBookKeeperEntry < 0
      || firstBookKeeperEntry > lh.getLastAddConfirmed()) {
    throw new IOException("Invalid first bk entry to read: "
        + firstBookKeeperEntry + ", LAC: " + lh.getLastAddConfirmed());
  }
  BufferedInputStream bin = new BufferedInputStream(
      new LedgerInputStream(lh, firstBookKeeperEntry));
  tracker = new FSEditLogLoader.PositionTrackingInputStream(bin);
  DataInputStream in = new DataInputStream(tracker);

  reader = new FSEditLogOp.Reader(in, tracker, logVersion);
}
项目:hadoop-plus    文件:TestBookKeeperJournalManager.java   
@Test
public void testSimpleWrite() throws Exception {
  NamespaceInfo nsi = newNSInfo();
  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-simplewrite"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);

  String zkpath = bkjm.finalizedLedgerZNode(1, 100);

  assertNotNull(zkc.exists(zkpath, false));
  assertNull(zkc.exists(bkjm.inprogressZNode(1), false));
}
项目:hadoop-plus    文件:TestBookKeeperJournalManager.java   
@Test
public void testNumberOfTransactions() throws Exception {
  NamespaceInfo nsi = newNSInfo();

  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-txncount"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);

  long numTrans = bkjm.getNumberOfTransactions(1, true);
  assertEquals(100, numTrans);
}
项目:hadoop-plus    文件:TestBookKeeperJournalManager.java   
private String startAndFinalizeLogSegment(BookKeeperJournalManager bkjm,
    int startTxid, int endTxid) throws IOException, KeeperException,
    InterruptedException {
  EditLogOutputStream out = bkjm.startLogSegment(startTxid);
  for (long i = startTxid; i <= endTxid; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  // finalize the inprogress_1 log segment.
  bkjm.finalizeLogSegment(startTxid, endTxid);
  String zkpath1 = bkjm.finalizedLedgerZNode(startTxid, endTxid);
  assertNotNull(zkc.exists(zkpath1, false));
  assertNull(zkc.exists(bkjm.inprogressZNode(startTxid), false));
  return zkpath1;
}
项目:FlexMap    文件:BookKeeperEditLogInputStream.java   
/**
 * Construct BookKeeper edit log input stream. 
 * Starts reading from firstBookKeeperEntry. This allows the stream
 * to take a shortcut during recovery, as it doesn't have to read
 * every edit log transaction to find out what the last one is.
 */
BookKeeperEditLogInputStream(LedgerHandle lh, EditLogLedgerMetadata metadata,
                             long firstBookKeeperEntry) 
    throws IOException {
  this.lh = lh;
  this.firstTxId = metadata.getFirstTxId();
  this.lastTxId = metadata.getLastTxId();
  this.logVersion = metadata.getDataLayoutVersion();
  this.inProgress = metadata.isInProgress();

  if (firstBookKeeperEntry < 0
      || firstBookKeeperEntry > lh.getLastAddConfirmed()) {
    throw new IOException("Invalid first bk entry to read: "
        + firstBookKeeperEntry + ", LAC: " + lh.getLastAddConfirmed());
  }
  BufferedInputStream bin = new BufferedInputStream(
      new LedgerInputStream(lh, firstBookKeeperEntry));
  tracker = new FSEditLogLoader.PositionTrackingInputStream(bin);
  DataInputStream in = new DataInputStream(tracker);

  reader = new FSEditLogOp.Reader(in, tracker, logVersion);
}
项目:FlexMap    文件:TestBookKeeperJournalManager.java   
@Test
public void testSimpleWrite() throws Exception {
  NamespaceInfo nsi = newNSInfo();
  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-simplewrite"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);

  String zkpath = bkjm.finalizedLedgerZNode(1, 100);

  assertNotNull(zkc.exists(zkpath, false));
  assertNull(zkc.exists(bkjm.inprogressZNode(1), false));
}
项目:FlexMap    文件:TestBookKeeperJournalManager.java   
@Test
public void testNumberOfTransactions() throws Exception {
  NamespaceInfo nsi = newNSInfo();

  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-txncount"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);

  long numTrans = bkjm.getNumberOfTransactions(1, true);
  assertEquals(100, numTrans);
}
项目:FlexMap    文件:TestBookKeeperJournalManager.java   
private String startAndFinalizeLogSegment(BookKeeperJournalManager bkjm,
    int startTxid, int endTxid) throws IOException, KeeperException,
    InterruptedException {
  EditLogOutputStream out = bkjm.startLogSegment(startTxid,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = startTxid; i <= endTxid; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  // finalize the inprogress_1 log segment.
  bkjm.finalizeLogSegment(startTxid, endTxid);
  String zkpath1 = bkjm.finalizedLedgerZNode(startTxid, endTxid);
  assertNotNull(zkc.exists(zkpath1, false));
  assertNull(zkc.exists(bkjm.inprogressZNode(startTxid), false));
  return zkpath1;
}
项目:hadoop-TCP    文件:BookKeeperEditLogInputStream.java   
/**
 * Construct BookKeeper edit log input stream. 
 * Starts reading from firstBookKeeperEntry. This allows the stream
 * to take a shortcut during recovery, as it doesn't have to read
 * every edit log transaction to find out what the last one is.
 */
BookKeeperEditLogInputStream(LedgerHandle lh, EditLogLedgerMetadata metadata,
                             long firstBookKeeperEntry) 
    throws IOException {
  this.lh = lh;
  this.firstTxId = metadata.getFirstTxId();
  this.lastTxId = metadata.getLastTxId();
  this.logVersion = metadata.getDataLayoutVersion();
  this.inProgress = metadata.isInProgress();

  if (firstBookKeeperEntry < 0
      || firstBookKeeperEntry > lh.getLastAddConfirmed()) {
    throw new IOException("Invalid first bk entry to read: "
        + firstBookKeeperEntry + ", LAC: " + lh.getLastAddConfirmed());
  }
  BufferedInputStream bin = new BufferedInputStream(
      new LedgerInputStream(lh, firstBookKeeperEntry));
  tracker = new FSEditLogLoader.PositionTrackingInputStream(bin);
  DataInputStream in = new DataInputStream(tracker);

  reader = new FSEditLogOp.Reader(in, tracker, logVersion);
}
项目:hadoop-TCP    文件:TestBookKeeperJournalManager.java   
@Test
public void testSimpleWrite() throws Exception {
  NamespaceInfo nsi = newNSInfo();
  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-simplewrite"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);

  String zkpath = bkjm.finalizedLedgerZNode(1, 100);

  assertNotNull(zkc.exists(zkpath, false));
  assertNull(zkc.exists(bkjm.inprogressZNode(1), false));
}
项目:hadoop-TCP    文件:TestBookKeeperJournalManager.java   
@Test
public void testNumberOfTransactions() throws Exception {
  NamespaceInfo nsi = newNSInfo();

  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-txncount"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);

  long numTrans = bkjm.getNumberOfTransactions(1, true);
  assertEquals(100, numTrans);
}
项目:hadoop-TCP    文件:TestBookKeeperJournalManager.java   
private String startAndFinalizeLogSegment(BookKeeperJournalManager bkjm,
    int startTxid, int endTxid) throws IOException, KeeperException,
    InterruptedException {
  EditLogOutputStream out = bkjm.startLogSegment(startTxid);
  for (long i = startTxid; i <= endTxid; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  // finalize the inprogress_1 log segment.
  bkjm.finalizeLogSegment(startTxid, endTxid);
  String zkpath1 = bkjm.finalizedLedgerZNode(startTxid, endTxid);
  assertNotNull(zkc.exists(zkpath1, false));
  assertNull(zkc.exists(bkjm.inprogressZNode(startTxid), false));
  return zkpath1;
}
项目:hardfs    文件:BookKeeperEditLogInputStream.java   
/**
 * Construct BookKeeper edit log input stream. 
 * Starts reading from firstBookKeeperEntry. This allows the stream
 * to take a shortcut during recovery, as it doesn't have to read
 * every edit log transaction to find out what the last one is.
 */
BookKeeperEditLogInputStream(LedgerHandle lh, EditLogLedgerMetadata metadata,
                             long firstBookKeeperEntry) 
    throws IOException {
  this.lh = lh;
  this.firstTxId = metadata.getFirstTxId();
  this.lastTxId = metadata.getLastTxId();
  this.logVersion = metadata.getDataLayoutVersion();
  this.inProgress = metadata.isInProgress();

  if (firstBookKeeperEntry < 0
      || firstBookKeeperEntry > lh.getLastAddConfirmed()) {
    throw new IOException("Invalid first bk entry to read: "
        + firstBookKeeperEntry + ", LAC: " + lh.getLastAddConfirmed());
  }
  BufferedInputStream bin = new BufferedInputStream(
      new LedgerInputStream(lh, firstBookKeeperEntry));
  tracker = new FSEditLogLoader.PositionTrackingInputStream(bin);
  DataInputStream in = new DataInputStream(tracker);

  reader = new FSEditLogOp.Reader(in, tracker, logVersion);
}
项目:hardfs    文件:TestBookKeeperJournalManager.java   
@Test
public void testSimpleWrite() throws Exception {
  NamespaceInfo nsi = newNSInfo();
  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-simplewrite"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);

  String zkpath = bkjm.finalizedLedgerZNode(1, 100);

  assertNotNull(zkc.exists(zkpath, false));
  assertNull(zkc.exists(bkjm.inprogressZNode(1), false));
}
项目:hardfs    文件:TestBookKeeperJournalManager.java   
@Test
public void testNumberOfTransactions() throws Exception {
  NamespaceInfo nsi = newNSInfo();

  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-txncount"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);

  long numTrans = bkjm.getNumberOfTransactions(1, true);
  assertEquals(100, numTrans);
}
项目:hardfs    文件:TestBookKeeperJournalManager.java   
private String startAndFinalizeLogSegment(BookKeeperJournalManager bkjm,
    int startTxid, int endTxid) throws IOException, KeeperException,
    InterruptedException {
  EditLogOutputStream out = bkjm.startLogSegment(startTxid);
  for (long i = startTxid; i <= endTxid; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  // finalize the inprogress_1 log segment.
  bkjm.finalizeLogSegment(startTxid, endTxid);
  String zkpath1 = bkjm.finalizedLedgerZNode(startTxid, endTxid);
  assertNotNull(zkc.exists(zkpath1, false));
  assertNull(zkc.exists(bkjm.inprogressZNode(startTxid), false));
  return zkpath1;
}
项目:hadoop-on-lustre2    文件:BookKeeperEditLogInputStream.java   
/**
 * Construct BookKeeper edit log input stream. 
 * Starts reading from firstBookKeeperEntry. This allows the stream
 * to take a shortcut during recovery, as it doesn't have to read
 * every edit log transaction to find out what the last one is.
 */
BookKeeperEditLogInputStream(LedgerHandle lh, EditLogLedgerMetadata metadata,
                             long firstBookKeeperEntry) 
    throws IOException {
  this.lh = lh;
  this.firstTxId = metadata.getFirstTxId();
  this.lastTxId = metadata.getLastTxId();
  this.logVersion = metadata.getDataLayoutVersion();
  this.inProgress = metadata.isInProgress();

  if (firstBookKeeperEntry < 0
      || firstBookKeeperEntry > lh.getLastAddConfirmed()) {
    throw new IOException("Invalid first bk entry to read: "
        + firstBookKeeperEntry + ", LAC: " + lh.getLastAddConfirmed());
  }
  BufferedInputStream bin = new BufferedInputStream(
      new LedgerInputStream(lh, firstBookKeeperEntry));
  tracker = new FSEditLogLoader.PositionTrackingInputStream(bin);
  DataInputStream in = new DataInputStream(tracker);

  reader = new FSEditLogOp.Reader(in, tracker, logVersion);
}
项目:hadoop-on-lustre2    文件:TestBookKeeperJournalManager.java   
@Test
public void testSimpleWrite() throws Exception {
  NamespaceInfo nsi = newNSInfo();
  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-simplewrite"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);

  String zkpath = bkjm.finalizedLedgerZNode(1, 100);

  assertNotNull(zkc.exists(zkpath, false));
  assertNull(zkc.exists(bkjm.inprogressZNode(1), false));
}
项目:hadoop-on-lustre2    文件:TestBookKeeperJournalManager.java   
@Test
public void testNumberOfTransactions() throws Exception {
  NamespaceInfo nsi = newNSInfo();

  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-txncount"), nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  bkjm.finalizeLogSegment(1, 100);

  long numTrans = bkjm.getNumberOfTransactions(1, true);
  assertEquals(100, numTrans);
}
项目:hadoop-on-lustre2    文件:TestBookKeeperJournalManager.java   
private String startAndFinalizeLogSegment(BookKeeperJournalManager bkjm,
    int startTxid, int endTxid) throws IOException, KeeperException,
    InterruptedException {
  EditLogOutputStream out = bkjm.startLogSegment(startTxid,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  for (long i = startTxid; i <= endTxid; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.close();
  // finalize the inprogress_1 log segment.
  bkjm.finalizeLogSegment(startTxid, endTxid);
  String zkpath1 = bkjm.finalizedLedgerZNode(startTxid, endTxid);
  assertNotNull(zkc.exists(zkpath1, false));
  assertNull(zkc.exists(bkjm.inprogressZNode(startTxid), false));
  return zkpath1;
}
项目:hadoop    文件:QJMTestUtil.java   
public static byte[] createTxnData(int startTxn, int numTxns) throws Exception {
  DataOutputBuffer buf = new DataOutputBuffer();
  FSEditLogOp.Writer writer = new FSEditLogOp.Writer(buf);

  for (long txid = startTxn; txid < startTxn + numTxns; txid++) {
    FSEditLogOp op = NameNodeAdapter.createMkdirOp("tx " + txid);
    op.setTransactionId(txid);
    writer.writeOp(op);
  }

  return Arrays.copyOf(buf.getData(), buf.getLength());
}