1. 包列表
  2. 类列表
  3. org.apache.hadoop.hdfs.server.protocol.RemoteEditLog

Java 类org.apache.hadoop.hdfs.server.protocol.RemoteEditLog 实例源码

项目：hadoop    文件：Checkpointer.java    
static void rollForwardByApplyingLogs(
    RemoteEditLogManifest manifest,
    FSImage dstImage,
    FSNamesystem dstNamesystem) throws IOException {
  NNStorage dstStorage = dstImage.getStorage();

  List<EditLogInputStream> editsStreams = Lists.newArrayList();    
  for (RemoteEditLog log : manifest.getLogs()) {
    if (log.getEndTxId() > dstImage.getLastAppliedTxId()) {
      File f = dstStorage.findFinalizedEditsFile(
          log.getStartTxId(), log.getEndTxId());
      editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), 
                                                  log.getEndTxId(), true));
    }
  }
  LOG.info("Checkpointer about to load edits from " +
      editsStreams.size() + " stream(s).");
  dstImage.loadEdits(editsStreams, dstNamesystem);
}
项目：hadoop    文件：Journal.java    
/**
 * @see QJournalProtocol#getEditLogManifest(String, long, boolean)
 */
public RemoteEditLogManifest getEditLogManifest(long sinceTxId,
    boolean inProgressOk) throws IOException {
  // No need to checkRequest() here - anyone may ask for the list
  // of segments.
  checkFormatted();

  List<RemoteEditLog> logs = fjm.getRemoteEditLogs(sinceTxId, inProgressOk);

  if (inProgressOk) {
    RemoteEditLog log = null;
    for (Iterator<RemoteEditLog> iter = logs.iterator(); iter.hasNext();) {
      log = iter.next();
      if (log.isInProgress()) {
        iter.remove();
        break;
      }
    }
    if (log != null && log.isInProgress()) {
      logs.add(new RemoteEditLog(log.getStartTxId(), getHighestWrittenTxId(),
          true));
    }
  }

  return new RemoteEditLogManifest(logs);
}
项目：aliyun-oss-hadoop-fs    文件：Checkpointer.java    
static void rollForwardByApplyingLogs(
    RemoteEditLogManifest manifest,
    FSImage dstImage,
    FSNamesystem dstNamesystem) throws IOException {
  NNStorage dstStorage = dstImage.getStorage();

  List<EditLogInputStream> editsStreams = Lists.newArrayList();    
  for (RemoteEditLog log : manifest.getLogs()) {
    if (log.getEndTxId() > dstImage.getLastAppliedTxId()) {
      File f = dstStorage.findFinalizedEditsFile(
          log.getStartTxId(), log.getEndTxId());
      editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), 
                                                  log.getEndTxId(), true));
    }
  }
  LOG.info("Checkpointer about to load edits from " +
      editsStreams.size() + " stream(s).");
  dstImage.loadEdits(editsStreams, dstNamesystem);
}
项目：aliyun-oss-hadoop-fs    文件：Journal.java    
/**
 * @see QJournalProtocol#getEditLogManifest(String, long, boolean)
 */
public RemoteEditLogManifest getEditLogManifest(long sinceTxId,
    boolean inProgressOk) throws IOException {
  // No need to checkRequest() here - anyone may ask for the list
  // of segments.
  checkFormatted();

  List<RemoteEditLog> logs = fjm.getRemoteEditLogs(sinceTxId, inProgressOk);

  if (inProgressOk) {
    RemoteEditLog log = null;
    for (Iterator<RemoteEditLog> iter = logs.iterator(); iter.hasNext();) {
      log = iter.next();
      if (log.isInProgress()) {
        iter.remove();
        break;
      }
    }
    if (log != null && log.isInProgress()) {
      logs.add(new RemoteEditLog(log.getStartTxId(), getHighestWrittenTxId(),
          true));
    }
  }

  return new RemoteEditLogManifest(logs);
}
项目：big-c    文件：Checkpointer.java    
static void rollForwardByApplyingLogs(
    RemoteEditLogManifest manifest,
    FSImage dstImage,
    FSNamesystem dstNamesystem) throws IOException {
  NNStorage dstStorage = dstImage.getStorage();

  List<EditLogInputStream> editsStreams = Lists.newArrayList();    
  for (RemoteEditLog log : manifest.getLogs()) {
    if (log.getEndTxId() > dstImage.getLastAppliedTxId()) {
      File f = dstStorage.findFinalizedEditsFile(
          log.getStartTxId(), log.getEndTxId());
      editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), 
                                                  log.getEndTxId(), true));
    }
  }
  LOG.info("Checkpointer about to load edits from " +
      editsStreams.size() + " stream(s).");
  dstImage.loadEdits(editsStreams, dstNamesystem);
}
项目：big-c    文件：Journal.java    
/**
 * @see QJournalProtocol#getEditLogManifest(String, long, boolean)
 */
public RemoteEditLogManifest getEditLogManifest(long sinceTxId,
    boolean inProgressOk) throws IOException {
  // No need to checkRequest() here - anyone may ask for the list
  // of segments.
  checkFormatted();

  List<RemoteEditLog> logs = fjm.getRemoteEditLogs(sinceTxId, inProgressOk);

  if (inProgressOk) {
    RemoteEditLog log = null;
    for (Iterator<RemoteEditLog> iter = logs.iterator(); iter.hasNext();) {
      log = iter.next();
      if (log.isInProgress()) {
        iter.remove();
        break;
      }
    }
    if (log != null && log.isInProgress()) {
      logs.add(new RemoteEditLog(log.getStartTxId(), getHighestWrittenTxId(),
          true));
    }
  }

  return new RemoteEditLogManifest(logs);
}
项目：hadoop-2.6.0-cdh5.4.3    文件：Checkpointer.java    
static void rollForwardByApplyingLogs(
    RemoteEditLogManifest manifest,
    FSImage dstImage,
    FSNamesystem dstNamesystem) throws IOException {
  NNStorage dstStorage = dstImage.getStorage();

  List<EditLogInputStream> editsStreams = Lists.newArrayList();    
  for (RemoteEditLog log : manifest.getLogs()) {
    if (log.getEndTxId() > dstImage.getLastAppliedTxId()) {
      File f = dstStorage.findFinalizedEditsFile(
          log.getStartTxId(), log.getEndTxId());
      editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), 
                                                  log.getEndTxId(), true));
    }
  }
  LOG.info("Checkpointer about to load edits from " +
      editsStreams.size() + " stream(s).");
  dstImage.loadEdits(editsStreams, dstNamesystem);
}
项目：hadoop-2.6.0-cdh5.4.3    文件：Journal.java    
/**
 * @see QJournalProtocol#getEditLogManifest(String, long, boolean)
 */
public RemoteEditLogManifest getEditLogManifest(long sinceTxId,
    boolean inProgressOk) throws IOException {
  // No need to checkRequest() here - anyone may ask for the list
  // of segments.
  checkFormatted();

  List<RemoteEditLog> logs = fjm.getRemoteEditLogs(sinceTxId, inProgressOk);

  if (inProgressOk) {
    RemoteEditLog log = null;
    for (Iterator<RemoteEditLog> iter = logs.iterator(); iter.hasNext();) {
      log = iter.next();
      if (log.isInProgress()) {
        iter.remove();
        break;
      }
    }
    if (log != null && log.isInProgress()) {
      logs.add(new RemoteEditLog(log.getStartTxId(), getHighestWrittenTxId(),
          true));
    }
  }

  return new RemoteEditLogManifest(logs);
}
项目：hadoop-EAR    文件：FSImage.java    
static void rollForwardByApplyingLogs(
    RemoteEditLogManifest manifest,
    FSImage dstImage) throws IOException {
  NNStorage dstStorage = dstImage.storage;

  List<EditLogInputStream> editsStreams = new ArrayList<EditLogInputStream>();    
  for (RemoteEditLog log : manifest.getLogs()) {
    if (log.inProgress())
      break;
    File f = dstStorage.findFinalizedEditsFile(
        log.getStartTxId(), log.getEndTxId());
    if (log.getStartTxId() > dstImage.getLastAppliedTxId()) {
      editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), 
                                                  log.getEndTxId(), false));
     }
  }
  dstImage.loadEdits(editsStreams);
}
项目：hadoop-plus    文件：Checkpointer.java    
static void rollForwardByApplyingLogs(
    RemoteEditLogManifest manifest,
    FSImage dstImage,
    FSNamesystem dstNamesystem) throws IOException {
  NNStorage dstStorage = dstImage.getStorage();

  List<EditLogInputStream> editsStreams = Lists.newArrayList();    
  for (RemoteEditLog log : manifest.getLogs()) {
    if (log.getEndTxId() > dstImage.getLastAppliedTxId()) {
      File f = dstStorage.findFinalizedEditsFile(
          log.getStartTxId(), log.getEndTxId());
      editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), 
                                                  log.getEndTxId(), true));
    }
  }
  LOG.info("Checkpointer about to load edits from " +
      editsStreams.size() + " stream(s).");
  dstImage.loadEdits(editsStreams, dstNamesystem, null);
}
项目：FlexMap    文件：Checkpointer.java    
static void rollForwardByApplyingLogs(
    RemoteEditLogManifest manifest,
    FSImage dstImage,
    FSNamesystem dstNamesystem) throws IOException {
  NNStorage dstStorage = dstImage.getStorage();

  List<EditLogInputStream> editsStreams = Lists.newArrayList();    
  for (RemoteEditLog log : manifest.getLogs()) {
    if (log.getEndTxId() > dstImage.getLastAppliedTxId()) {
      File f = dstStorage.findFinalizedEditsFile(
          log.getStartTxId(), log.getEndTxId());
      editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), 
                                                  log.getEndTxId(), true));
    }
  }
  LOG.info("Checkpointer about to load edits from " +
      editsStreams.size() + " stream(s).");
  dstImage.loadEdits(editsStreams, dstNamesystem);
}
项目：FlexMap    文件：Journal.java    
/**
 * @see QJournalProtocol#getEditLogManifest(String, long, boolean)
 */
public RemoteEditLogManifest getEditLogManifest(long sinceTxId,
    boolean inProgressOk) throws IOException {
  // No need to checkRequest() here - anyone may ask for the list
  // of segments.
  checkFormatted();

  List<RemoteEditLog> logs = fjm.getRemoteEditLogs(sinceTxId, inProgressOk);

  if (inProgressOk) {
    RemoteEditLog log = null;
    for (Iterator<RemoteEditLog> iter = logs.iterator(); iter.hasNext();) {
      log = iter.next();
      if (log.isInProgress()) {
        iter.remove();
        break;
      }
    }
    if (log != null && log.isInProgress()) {
      logs.add(new RemoteEditLog(log.getStartTxId(), getHighestWrittenTxId(),
          true));
    }
  }

  return new RemoteEditLogManifest(logs);
}
项目：hadoop-TCP    文件：Checkpointer.java    
static void rollForwardByApplyingLogs(
    RemoteEditLogManifest manifest,
    FSImage dstImage,
    FSNamesystem dstNamesystem) throws IOException {
  NNStorage dstStorage = dstImage.getStorage();

  List<EditLogInputStream> editsStreams = Lists.newArrayList();    
  for (RemoteEditLog log : manifest.getLogs()) {
    if (log.getEndTxId() > dstImage.getLastAppliedTxId()) {
      File f = dstStorage.findFinalizedEditsFile(
          log.getStartTxId(), log.getEndTxId());
      editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), 
                                                  log.getEndTxId(), true));
    }
  }
  LOG.info("Checkpointer about to load edits from " +
      editsStreams.size() + " stream(s).");
  dstImage.loadEdits(editsStreams, dstNamesystem, null);
}
项目：hardfs    文件：Checkpointer.java    
static void rollForwardByApplyingLogs(
    RemoteEditLogManifest manifest,
    FSImage dstImage,
    FSNamesystem dstNamesystem) throws IOException {
  NNStorage dstStorage = dstImage.getStorage();

  List<EditLogInputStream> editsStreams = Lists.newArrayList();    
  for (RemoteEditLog log : manifest.getLogs()) {
    if (log.getEndTxId() > dstImage.getLastAppliedTxId()) {
      File f = dstStorage.findFinalizedEditsFile(
          log.getStartTxId(), log.getEndTxId());
      editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), 
                                                  log.getEndTxId(), true));
    }
  }
  LOG.info("Checkpointer about to load edits from " +
      editsStreams.size() + " stream(s).");
  dstImage.loadEdits(editsStreams, dstNamesystem, null);
}
项目：hadoop-on-lustre2    文件：Checkpointer.java    
static void rollForwardByApplyingLogs(
    RemoteEditLogManifest manifest,
    FSImage dstImage,
    FSNamesystem dstNamesystem) throws IOException {
  NNStorage dstStorage = dstImage.getStorage();

  List<EditLogInputStream> editsStreams = Lists.newArrayList();    
  for (RemoteEditLog log : manifest.getLogs()) {
    if (log.getEndTxId() > dstImage.getLastAppliedTxId()) {
      File f = dstStorage.findFinalizedEditsFile(
          log.getStartTxId(), log.getEndTxId());
      editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), 
                                                  log.getEndTxId(), true));
    }
  }
  LOG.info("Checkpointer about to load edits from " +
      editsStreams.size() + " stream(s).");
  dstImage.loadEdits(editsStreams, dstNamesystem);
}
项目：hadoop-on-lustre2    文件：FileJournalManager.java    
/**
 * Find all editlog segments starting at or above the given txid.
 * @param fromTxId the txnid which to start looking
 * @param inProgressOk whether or not to include the in-progress edit log 
 *        segment       
 * @return a list of remote edit logs
 * @throws IOException if edit logs cannot be listed.
 */
public List<RemoteEditLog> getRemoteEditLogs(long firstTxId,
    boolean inProgressOk) throws IOException {
  File currentDir = sd.getCurrentDir();
  List<EditLogFile> allLogFiles = matchEditLogs(currentDir);
  List<RemoteEditLog> ret = Lists.newArrayListWithCapacity(
      allLogFiles.size());

  for (EditLogFile elf : allLogFiles) {
    if (elf.hasCorruptHeader() || (!inProgressOk && elf.isInProgress())) {
      continue;
    }
    if (elf.getFirstTxId() >= firstTxId) {
      ret.add(new RemoteEditLog(elf.firstTxId, elf.lastTxId));
    } else if (elf.getFirstTxId() < firstTxId && firstTxId <= elf.getLastTxId()) {
      // If the firstTxId is in the middle of an edit log segment. Return this
      // anyway and let the caller figure out whether it wants to use it.
      ret.add(new RemoteEditLog(elf.firstTxId, elf.lastTxId));
    }
  }

  Collections.sort(ret);

  return ret;
}
项目：hadoop-on-lustre2    文件：Journal.java    
/**
 * @see QJournalProtocol#getEditLogManifest(String, long, boolean)
 */
public RemoteEditLogManifest getEditLogManifest(long sinceTxId,
    boolean inProgressOk) throws IOException {
  // No need to checkRequest() here - anyone may ask for the list
  // of segments.
  checkFormatted();

  List<RemoteEditLog> logs = fjm.getRemoteEditLogs(sinceTxId, inProgressOk);

  if (inProgressOk) {
    RemoteEditLog log = null;
    for (Iterator<RemoteEditLog> iter = logs.iterator(); iter.hasNext();) {
      log = iter.next();
      if (log.isInProgress()) {
        iter.remove();
        break;
      }
    }
    if (log != null && log.isInProgress()) {
      logs.add(new RemoteEditLog(log.getStartTxId(), getHighestWrittenTxId()));
    }
  }

  return new RemoteEditLogManifest(logs);
}
项目：hadoop    文件：ImageServlet.java    
static String getParamStringForLog(RemoteEditLog log,
    StorageInfo remoteStorageInfo) {
  return "getedit=1&" + START_TXID_PARAM + "=" + log.getStartTxId()
      + "&" + END_TXID_PARAM + "=" + log.getEndTxId()
      + "&" + STORAGEINFO_PARAM + "=" +
        remoteStorageInfo.toColonSeparatedString();
}
项目：hadoop    文件：FileJournalManager.java    
/**
 * Find all editlog segments starting at or above the given txid.
 * @param firstTxId the txnid which to start looking
 * @param inProgressOk whether or not to include the in-progress edit log 
 *        segment       
 * @return a list of remote edit logs
 * @throws IOException if edit logs cannot be listed.
 */
public List<RemoteEditLog> getRemoteEditLogs(long firstTxId,
    boolean inProgressOk) throws IOException {
  File currentDir = sd.getCurrentDir();
  List<EditLogFile> allLogFiles = matchEditLogs(currentDir);
  List<RemoteEditLog> ret = Lists.newArrayListWithCapacity(
      allLogFiles.size());
  for (EditLogFile elf : allLogFiles) {
    if (elf.hasCorruptHeader() || (!inProgressOk && elf.isInProgress())) {
      continue;
    }
    if (elf.isInProgress()) {
      try {
        elf.validateLog();
      } catch (IOException e) {
        LOG.error("got IOException while trying to validate header of " +
            elf + ".  Skipping.", e);
        continue;
      }
    }
    if (elf.getFirstTxId() >= firstTxId) {
      ret.add(new RemoteEditLog(elf.firstTxId, elf.lastTxId,
          elf.isInProgress()));
    } else if (elf.getFirstTxId() < firstTxId && firstTxId <= elf.getLastTxId()) {
      // If the firstTxId is in the middle of an edit log segment. Return this
      // anyway and let the caller figure out whether it wants to use it.
      ret.add(new RemoteEditLog(elf.firstTxId, elf.lastTxId,
          elf.isInProgress()));
    }
  }

  Collections.sort(ret);

  return ret;
}
项目：hadoop    文件：PBHelper.java    
public static RemoteEditLogManifestProto convert(
    RemoteEditLogManifest manifest) {
  RemoteEditLogManifestProto.Builder builder = RemoteEditLogManifestProto
      .newBuilder();
  for (RemoteEditLog log : manifest.getLogs()) {
    builder.addLogs(convert(log));
  }
  return builder.build();
}
项目：hadoop    文件：PBHelper.java    
public static RemoteEditLogManifest convert(
    RemoteEditLogManifestProto manifest) {
  List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>(manifest
      .getLogsList().size());
  for (RemoteEditLogProto l : manifest.getLogsList()) {
    logs.add(convert(l));
  }
  return new RemoteEditLogManifest(logs);
}
项目：hadoop    文件：QuorumJournalManager.java    
@Override
public void selectInputStreams(Collection<EditLogInputStream> streams,
    long fromTxnId, boolean inProgressOk) throws IOException {

  QuorumCall<AsyncLogger, RemoteEditLogManifest> q =
      loggers.getEditLogManifest(fromTxnId, inProgressOk);
  Map<AsyncLogger, RemoteEditLogManifest> resps =
      loggers.waitForWriteQuorum(q, selectInputStreamsTimeoutMs,
          "selectInputStreams");

  LOG.debug("selectInputStream manifests:\n" +
      Joiner.on("\n").withKeyValueSeparator(": ").join(resps));

  final PriorityQueue<EditLogInputStream> allStreams = 
      new PriorityQueue<EditLogInputStream>(64,
          JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR);
  for (Map.Entry<AsyncLogger, RemoteEditLogManifest> e : resps.entrySet()) {
    AsyncLogger logger = e.getKey();
    RemoteEditLogManifest manifest = e.getValue();

    for (RemoteEditLog remoteLog : manifest.getLogs()) {
      URL url = logger.buildURLToFetchLogs(remoteLog.getStartTxId());

      EditLogInputStream elis = EditLogFileInputStream.fromUrl(
          connectionFactory, url, remoteLog.getStartTxId(),
          remoteLog.getEndTxId(), remoteLog.isInProgress());
      allStreams.add(elis);
    }
  }
  JournalSet.chainAndMakeRedundantStreams(streams, allStreams, fromTxnId);
}
项目：hadoop    文件：TestPBHelper.java    
@Test
public void testConvertRemoteEditLog() {
  RemoteEditLog l = new RemoteEditLog(1, 100);
  RemoteEditLogProto lProto = PBHelper.convert(l);
  RemoteEditLog l1 = PBHelper.convert(lProto);
  compare(l, l1);
}
项目：hadoop    文件：TestPBHelper.java    
@Test
public void testConvertRemoteEditLogManifest() {
  List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>();
  logs.add(new RemoteEditLog(1, 10));
  logs.add(new RemoteEditLog(11, 20));
  RemoteEditLogManifest m = new RemoteEditLogManifest(logs);
  RemoteEditLogManifestProto mProto = PBHelper.convert(m);
  RemoteEditLogManifest m1 = PBHelper.convert(mProto);

  List<RemoteEditLog> logs1 = m1.getLogs();
  assertEquals(logs.size(), logs1.size());
  for (int i = 0; i < logs.size(); i++) {
    compare(logs.get(i), logs1.get(i));
  }
}
项目：aliyun-oss-hadoop-fs    文件：ImageServlet.java    
static String getParamStringForLog(RemoteEditLog log,
    StorageInfo remoteStorageInfo) {
  return "getedit=1&" + START_TXID_PARAM + "=" + log.getStartTxId()
      + "&" + END_TXID_PARAM + "=" + log.getEndTxId()
      + "&" + STORAGEINFO_PARAM + "=" +
        remoteStorageInfo.toColonSeparatedString();
}
项目：aliyun-oss-hadoop-fs    文件：FileJournalManager.java    
/**
 * Find all editlog segments starting at or above the given txid.
 * @param firstTxId the txnid which to start looking
 * @param inProgressOk whether or not to include the in-progress edit log 
 *        segment       
 * @return a list of remote edit logs
 * @throws IOException if edit logs cannot be listed.
 */
public List<RemoteEditLog> getRemoteEditLogs(long firstTxId,
    boolean inProgressOk) throws IOException {
  File currentDir = sd.getCurrentDir();
  List<EditLogFile> allLogFiles = matchEditLogs(currentDir);
  List<RemoteEditLog> ret = Lists.newArrayListWithCapacity(
      allLogFiles.size());
  for (EditLogFile elf : allLogFiles) {
    if (elf.hasCorruptHeader() || (!inProgressOk && elf.isInProgress())) {
      continue;
    }
    if (elf.isInProgress()) {
      try {
        elf.scanLog(getLastReadableTxId(), true);
      } catch (IOException e) {
        LOG.error("got IOException while trying to validate header of " +
            elf + ".  Skipping.", e);
        continue;
      }
    }
    if (elf.getFirstTxId() >= firstTxId) {
      ret.add(new RemoteEditLog(elf.firstTxId, elf.lastTxId,
          elf.isInProgress()));
    } else if (elf.getFirstTxId() < firstTxId && firstTxId <= elf.getLastTxId()) {
      // If the firstTxId is in the middle of an edit log segment. Return this
      // anyway and let the caller figure out whether it wants to use it.
      ret.add(new RemoteEditLog(elf.firstTxId, elf.lastTxId,
          elf.isInProgress()));
    }
  }

  Collections.sort(ret);

  return ret;
}
项目：aliyun-oss-hadoop-fs    文件：PBHelper.java    
public static RemoteEditLogManifestProto convert(
    RemoteEditLogManifest manifest) {
  RemoteEditLogManifestProto.Builder builder = RemoteEditLogManifestProto
      .newBuilder();
  for (RemoteEditLog log : manifest.getLogs()) {
    builder.addLogs(convert(log));
  }
  return builder.build();
}
项目：aliyun-oss-hadoop-fs    文件：PBHelper.java    
public static RemoteEditLogManifest convert(
    RemoteEditLogManifestProto manifest) {
  List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>(manifest
      .getLogsList().size());
  for (RemoteEditLogProto l : manifest.getLogsList()) {
    logs.add(convert(l));
  }
  return new RemoteEditLogManifest(logs);
}
项目：aliyun-oss-hadoop-fs    文件：QuorumJournalManager.java    
@Override
public void selectInputStreams(Collection<EditLogInputStream> streams,
    long fromTxnId, boolean inProgressOk) throws IOException {

  QuorumCall<AsyncLogger, RemoteEditLogManifest> q =
      loggers.getEditLogManifest(fromTxnId, inProgressOk);
  Map<AsyncLogger, RemoteEditLogManifest> resps =
      loggers.waitForWriteQuorum(q, selectInputStreamsTimeoutMs,
          "selectInputStreams");

  LOG.debug("selectInputStream manifests:\n" +
      Joiner.on("\n").withKeyValueSeparator(": ").join(resps));

  final PriorityQueue<EditLogInputStream> allStreams = 
      new PriorityQueue<EditLogInputStream>(64,
          JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR);
  for (Map.Entry<AsyncLogger, RemoteEditLogManifest> e : resps.entrySet()) {
    AsyncLogger logger = e.getKey();
    RemoteEditLogManifest manifest = e.getValue();

    for (RemoteEditLog remoteLog : manifest.getLogs()) {
      URL url = logger.buildURLToFetchLogs(remoteLog.getStartTxId());

      EditLogInputStream elis = EditLogFileInputStream.fromUrl(
          connectionFactory, url, remoteLog.getStartTxId(),
          remoteLog.getEndTxId(), remoteLog.isInProgress());
      allStreams.add(elis);
    }
  }
  JournalSet.chainAndMakeRedundantStreams(streams, allStreams, fromTxnId);
}
项目：aliyun-oss-hadoop-fs    文件：TestPBHelper.java    
@Test
public void testConvertRemoteEditLog() {
  RemoteEditLog l = new RemoteEditLog(1, 100);
  RemoteEditLogProto lProto = PBHelper.convert(l);
  RemoteEditLog l1 = PBHelper.convert(lProto);
  compare(l, l1);
}
项目：aliyun-oss-hadoop-fs    文件：TestPBHelper.java    
@Test
public void testConvertRemoteEditLogManifest() {
  List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>();
  logs.add(new RemoteEditLog(1, 10));
  logs.add(new RemoteEditLog(11, 20));
  RemoteEditLogManifest m = new RemoteEditLogManifest(logs);
  RemoteEditLogManifestProto mProto = PBHelper.convert(m);
  RemoteEditLogManifest m1 = PBHelper.convert(mProto);

  List<RemoteEditLog> logs1 = m1.getLogs();
  assertEquals(logs.size(), logs1.size());
  for (int i = 0; i < logs.size(); i++) {
    compare(logs.get(i), logs1.get(i));
  }
}
项目：big-c    文件：ImageServlet.java    
static String getParamStringForLog(RemoteEditLog log,
    StorageInfo remoteStorageInfo) {
  return "getedit=1&" + START_TXID_PARAM + "=" + log.getStartTxId()
      + "&" + END_TXID_PARAM + "=" + log.getEndTxId()
      + "&" + STORAGEINFO_PARAM + "=" +
        remoteStorageInfo.toColonSeparatedString();
}
项目：big-c    文件：FileJournalManager.java    
/**
 * Find all editlog segments starting at or above the given txid.
 * @param firstTxId the txnid which to start looking
 * @param inProgressOk whether or not to include the in-progress edit log 
 *        segment       
 * @return a list of remote edit logs
 * @throws IOException if edit logs cannot be listed.
 */
public List<RemoteEditLog> getRemoteEditLogs(long firstTxId,
    boolean inProgressOk) throws IOException {
  File currentDir = sd.getCurrentDir();
  List<EditLogFile> allLogFiles = matchEditLogs(currentDir);
  List<RemoteEditLog> ret = Lists.newArrayListWithCapacity(
      allLogFiles.size());
  for (EditLogFile elf : allLogFiles) {
    if (elf.hasCorruptHeader() || (!inProgressOk && elf.isInProgress())) {
      continue;
    }
    if (elf.isInProgress()) {
      try {
        elf.validateLog();
      } catch (IOException e) {
        LOG.error("got IOException while trying to validate header of " +
            elf + ".  Skipping.", e);
        continue;
      }
    }
    if (elf.getFirstTxId() >= firstTxId) {
      ret.add(new RemoteEditLog(elf.firstTxId, elf.lastTxId,
          elf.isInProgress()));
    } else if (elf.getFirstTxId() < firstTxId && firstTxId <= elf.getLastTxId()) {
      // If the firstTxId is in the middle of an edit log segment. Return this
      // anyway and let the caller figure out whether it wants to use it.
      ret.add(new RemoteEditLog(elf.firstTxId, elf.lastTxId,
          elf.isInProgress()));
    }
  }

  Collections.sort(ret);

  return ret;
}
项目：big-c    文件：PBHelper.java    
public static RemoteEditLogManifestProto convert(
    RemoteEditLogManifest manifest) {
  RemoteEditLogManifestProto.Builder builder = RemoteEditLogManifestProto
      .newBuilder();
  for (RemoteEditLog log : manifest.getLogs()) {
    builder.addLogs(convert(log));
  }
  return builder.build();
}
项目：big-c    文件：PBHelper.java    
public static RemoteEditLogManifest convert(
    RemoteEditLogManifestProto manifest) {
  List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>(manifest
      .getLogsList().size());
  for (RemoteEditLogProto l : manifest.getLogsList()) {
    logs.add(convert(l));
  }
  return new RemoteEditLogManifest(logs);
}
项目：big-c    文件：QuorumJournalManager.java    
@Override
public void selectInputStreams(Collection<EditLogInputStream> streams,
    long fromTxnId, boolean inProgressOk) throws IOException {

  QuorumCall<AsyncLogger, RemoteEditLogManifest> q =
      loggers.getEditLogManifest(fromTxnId, inProgressOk);
  Map<AsyncLogger, RemoteEditLogManifest> resps =
      loggers.waitForWriteQuorum(q, selectInputStreamsTimeoutMs,
          "selectInputStreams");

  LOG.debug("selectInputStream manifests:\n" +
      Joiner.on("\n").withKeyValueSeparator(": ").join(resps));

  final PriorityQueue<EditLogInputStream> allStreams = 
      new PriorityQueue<EditLogInputStream>(64,
          JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR);
  for (Map.Entry<AsyncLogger, RemoteEditLogManifest> e : resps.entrySet()) {
    AsyncLogger logger = e.getKey();
    RemoteEditLogManifest manifest = e.getValue();

    for (RemoteEditLog remoteLog : manifest.getLogs()) {
      URL url = logger.buildURLToFetchLogs(remoteLog.getStartTxId());

      EditLogInputStream elis = EditLogFileInputStream.fromUrl(
          connectionFactory, url, remoteLog.getStartTxId(),
          remoteLog.getEndTxId(), remoteLog.isInProgress());
      allStreams.add(elis);
    }
  }
  JournalSet.chainAndMakeRedundantStreams(streams, allStreams, fromTxnId);
}
项目：big-c    文件：TestPBHelper.java    
@Test
public void testConvertRemoteEditLog() {
  RemoteEditLog l = new RemoteEditLog(1, 100);
  RemoteEditLogProto lProto = PBHelper.convert(l);
  RemoteEditLog l1 = PBHelper.convert(lProto);
  compare(l, l1);
}
项目：big-c    文件：TestPBHelper.java    
@Test
public void testConvertRemoteEditLogManifest() {
  List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>();
  logs.add(new RemoteEditLog(1, 10));
  logs.add(new RemoteEditLog(11, 20));
  RemoteEditLogManifest m = new RemoteEditLogManifest(logs);
  RemoteEditLogManifestProto mProto = PBHelper.convert(m);
  RemoteEditLogManifest m1 = PBHelper.convert(mProto);

  List<RemoteEditLog> logs1 = m1.getLogs();
  assertEquals(logs.size(), logs1.size());
  for (int i = 0; i < logs.size(); i++) {
    compare(logs.get(i), logs1.get(i));
  }
}
项目：hadoop-2.6.0-cdh5.4.3    文件：ImageServlet.java    
static String getParamStringForLog(RemoteEditLog log,
    StorageInfo remoteStorageInfo) {
  return "getedit=1&" + START_TXID_PARAM + "=" + log.getStartTxId()
      + "&" + END_TXID_PARAM + "=" + log.getEndTxId()
      + "&" + STORAGEINFO_PARAM + "=" +
        remoteStorageInfo.toColonSeparatedString();
}
项目：hadoop-2.6.0-cdh5.4.3    文件：FileJournalManager.java    
/**
 * Find all editlog segments starting at or above the given txid.
 * @param firstTxId the txnid which to start looking
 * @param inProgressOk whether or not to include the in-progress edit log 
 *        segment       
 * @return a list of remote edit logs
 * @throws IOException if edit logs cannot be listed.
 */
public List<RemoteEditLog> getRemoteEditLogs(long firstTxId,
    boolean inProgressOk) throws IOException {
  File currentDir = sd.getCurrentDir();
  List<EditLogFile> allLogFiles = matchEditLogs(currentDir);
  List<RemoteEditLog> ret = Lists.newArrayListWithCapacity(
      allLogFiles.size());
  for (EditLogFile elf : allLogFiles) {
    if (elf.hasCorruptHeader() || (!inProgressOk && elf.isInProgress())) {
      continue;
    }
    if (elf.isInProgress()) {
      try {
        elf.validateLog();
      } catch (IOException e) {
        LOG.error("got IOException while trying to validate header of " +
            elf + ".  Skipping.", e);
        continue;
      }
    }
    if (elf.getFirstTxId() >= firstTxId) {
      ret.add(new RemoteEditLog(elf.firstTxId, elf.lastTxId,
          elf.isInProgress()));
    } else if (elf.getFirstTxId() < firstTxId && firstTxId <= elf.getLastTxId()) {
      // If the firstTxId is in the middle of an edit log segment. Return this
      // anyway and let the caller figure out whether it wants to use it.
      ret.add(new RemoteEditLog(elf.firstTxId, elf.lastTxId,
          elf.isInProgress()));
    }
  }

  Collections.sort(ret);

  return ret;
}