static void rollForwardByApplyingLogs( RemoteEditLogManifest manifest, FSImage dstImage, FSNamesystem dstNamesystem) throws IOException { NNStorage dstStorage = dstImage.getStorage(); List<EditLogInputStream> editsStreams = Lists.newArrayList(); for (RemoteEditLog log : manifest.getLogs()) { if (log.getEndTxId() > dstImage.getLastAppliedTxId()) { File f = dstStorage.findFinalizedEditsFile( log.getStartTxId(), log.getEndTxId()); editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), log.getEndTxId(), true)); } } LOG.info("Checkpointer about to load edits from " + editsStreams.size() + " stream(s)."); dstImage.loadEdits(editsStreams, dstNamesystem); }
/** * @see QJournalProtocol#getEditLogManifest(String, long, boolean) */ public RemoteEditLogManifest getEditLogManifest(long sinceTxId, boolean inProgressOk) throws IOException { // No need to checkRequest() here - anyone may ask for the list // of segments. checkFormatted(); List<RemoteEditLog> logs = fjm.getRemoteEditLogs(sinceTxId, inProgressOk); if (inProgressOk) { RemoteEditLog log = null; for (Iterator<RemoteEditLog> iter = logs.iterator(); iter.hasNext();) { log = iter.next(); if (log.isInProgress()) { iter.remove(); break; } } if (log != null && log.isInProgress()) { logs.add(new RemoteEditLog(log.getStartTxId(), getHighestWrittenTxId(), true)); } } return new RemoteEditLogManifest(logs); }
static void rollForwardByApplyingLogs( RemoteEditLogManifest manifest, FSImage dstImage) throws IOException { NNStorage dstStorage = dstImage.storage; List<EditLogInputStream> editsStreams = new ArrayList<EditLogInputStream>(); for (RemoteEditLog log : manifest.getLogs()) { if (log.inProgress()) break; File f = dstStorage.findFinalizedEditsFile( log.getStartTxId(), log.getEndTxId()); if (log.getStartTxId() > dstImage.getLastAppliedTxId()) { editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), log.getEndTxId(), false)); } } dstImage.loadEdits(editsStreams); }
/** * Merge image and edits, and verify consistency with the signature. */ private void doMerge(CheckpointSignature sig, RemoteEditLogManifest manifest, boolean loadImage) throws IOException { NNStorage dstStorage = this.storage; dstStorage.setStorageInfo(sig); if (loadImage) { File file = dstStorage.findImageFile(sig.mostRecentCheckpointTxId); if (file == null) { throw new IOException("Couldn't find image file at txid " + sig.mostRecentCheckpointTxId + " even though it should have " + "just been downloaded"); } loadFSImage(new ImageInputStream(sig.mostRecentCheckpointTxId, new FileInputStream(file), null, "image", file.length())); } FSImage.rollForwardByApplyingLogs(manifest, this); this.saveFSImageInAllDirs(this.getLastAppliedTxId(), false); dstStorage.writeAll(); }
@Override public GetEditLogManifestResponseProto getEditLogManifest(byte[] jid, long sinceTxId) throws IOException { RemoteEditLogManifest manifest = jn.getOrCreateJournal(jid) .getEditLogManifest(sinceTxId); GetEditLogManifestResponseProto ret = new GetEditLogManifestResponseProto(); ret.setLogs(manifest.getLogs()); ret.setHttpPort(jn.getBoundHttpAddress().getPort()); if (JournalNode.LOG.isDebugEnabled()) { JournalNode.LOG.info("Returning manifest " + manifest.toString()); } return ret; }
static void rollForwardByApplyingLogs( RemoteEditLogManifest manifest, FSImage dstImage, FSNamesystem dstNamesystem) throws IOException { NNStorage dstStorage = dstImage.getStorage(); List<EditLogInputStream> editsStreams = Lists.newArrayList(); for (RemoteEditLog log : manifest.getLogs()) { if (log.getEndTxId() > dstImage.getLastAppliedTxId()) { File f = dstStorage.findFinalizedEditsFile( log.getStartTxId(), log.getEndTxId()); editsStreams.add(new EditLogFileInputStream(f, log.getStartTxId(), log.getEndTxId(), true)); } } LOG.info("Checkpointer about to load edits from " + editsStreams.size() + " stream(s)."); dstImage.loadEdits(editsStreams, dstNamesystem, null); }
static void doMerge( CheckpointSignature sig, RemoteEditLogManifest manifest, boolean loadImage, FSImage dstImage, FSNamesystem dstNamesystem) throws IOException { NNStorage dstStorage = dstImage.getStorage(); dstStorage.setStorageInfo(sig); if (loadImage) { File file = dstStorage.findImageFile(sig.mostRecentCheckpointTxId); if (file == null) { throw new IOException("Couldn't find image file at txid " + sig.mostRecentCheckpointTxId + " even though it should have " + "just been downloaded"); } dstImage.reloadFromImageFile(file, dstNamesystem); dstNamesystem.dir.imageLoadComplete(); } // error simulation code for junit test CheckpointFaultInjector.getInstance().duringMerge(); Checkpointer.rollForwardByApplyingLogs(manifest, dstImage, dstNamesystem); // The following has the side effect of purging old fsimages/edit logs. dstImage.saveFSImageInAllDirs(dstNamesystem, dstImage.getLastAppliedTxId()); dstStorage.writeAll(); }
@Override public ListenableFuture<RemoteEditLogManifest> getEditLogManifest( final long fromTxnId, final boolean forReading, final boolean inProgressOk) { return executor.submit(new Callable<RemoteEditLogManifest>() { @Override public RemoteEditLogManifest call() throws IOException { GetEditLogManifestResponseProto ret = getProxy().getEditLogManifest( journalId, fromTxnId, forReading, inProgressOk); // Update the http port, since we need this to build URLs to any of the // returned logs. httpPort = ret.getHttpPort(); return PBHelper.convert(ret.getManifest()); } }); }
/** * @see QJournalProtocol#getEditLogManifest(String, long, boolean) */ public RemoteEditLogManifest getEditLogManifest(long sinceTxId, boolean inProgressOk) throws IOException { // No need to checkRequest() here - anyone may ask for the list // of segments. checkFormatted(); List<RemoteEditLog> logs = fjm.getRemoteEditLogs(sinceTxId, inProgressOk); if (inProgressOk) { RemoteEditLog log = null; for (Iterator<RemoteEditLog> iter = logs.iterator(); iter.hasNext();) { log = iter.next(); if (log.isInProgress()) { iter.remove(); break; } } if (log != null && log.isInProgress()) { logs.add(new RemoteEditLog(log.getStartTxId(), getHighestWrittenTxId())); } } return new RemoteEditLogManifest(logs); }
@Override // NamenodeProtocol public RemoteEditLogManifest getEditLogManifest(long sinceTxId) throws IOException { checkNNStartup(); namesystem.checkOperation(OperationCategory.READ); namesystem.checkSuperuserPrivilege(); return namesystem.getEditLog().getEditLogManifest(sinceTxId); }
static void doMerge( CheckpointSignature sig, RemoteEditLogManifest manifest, boolean loadImage, FSImage dstImage, FSNamesystem dstNamesystem) throws IOException { NNStorage dstStorage = dstImage.getStorage(); dstStorage.setStorageInfo(sig); if (loadImage) { File file = dstStorage.findImageFile(NameNodeFile.IMAGE, sig.mostRecentCheckpointTxId); if (file == null) { throw new IOException("Couldn't find image file at txid " + sig.mostRecentCheckpointTxId + " even though it should have " + "just been downloaded"); } dstNamesystem.writeLock(); try { dstImage.reloadFromImageFile(file, dstNamesystem); } finally { dstNamesystem.writeUnlock(); } dstNamesystem.imageLoadComplete(); } // error simulation code for junit test CheckpointFaultInjector.getInstance().duringMerge(); Checkpointer.rollForwardByApplyingLogs(manifest, dstImage, dstNamesystem); // The following has the side effect of purging old fsimages/edit logs. dstImage.saveFSImageInAllDirs(dstNamesystem, dstImage.getLastAppliedTxId()); dstStorage.writeAll(); }
public static RemoteEditLogManifestProto convert( RemoteEditLogManifest manifest) { RemoteEditLogManifestProto.Builder builder = RemoteEditLogManifestProto .newBuilder(); for (RemoteEditLog log : manifest.getLogs()) { builder.addLogs(convert(log)); } return builder.build(); }
public static RemoteEditLogManifest convert( RemoteEditLogManifestProto manifest) { List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>(manifest .getLogsList().size()); for (RemoteEditLogProto l : manifest.getLogsList()) { logs.add(convert(l)); } return new RemoteEditLogManifest(logs); }
@Override public GetEditLogManifestResponseProto getEditLogManifest( RpcController unused, GetEditLogManifestRequestProto request) throws ServiceException { RemoteEditLogManifest manifest; try { manifest = impl.getEditLogManifest(request.getSinceTxId()); } catch (IOException e) { throw new ServiceException(e); } return GetEditLogManifestResponseProto.newBuilder() .setManifest(PBHelper.convert(manifest)).build(); }
@Override public RemoteEditLogManifest getEditLogManifest(long sinceTxId) throws IOException { GetEditLogManifestRequestProto req = GetEditLogManifestRequestProto .newBuilder().setSinceTxId(sinceTxId).build(); try { return PBHelper.convert(rpcProxy.getEditLogManifest(NULL_CONTROLLER, req) .getManifest()); } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } }
public QuorumCall<AsyncLogger, RemoteEditLogManifest> getEditLogManifest( long fromTxnId, boolean inProgressOk) { Map<AsyncLogger, ListenableFuture<RemoteEditLogManifest>> calls = Maps.newHashMap(); for (AsyncLogger logger : loggers) { ListenableFuture<RemoteEditLogManifest> future = logger.getEditLogManifest(fromTxnId, inProgressOk); calls.put(logger, future); } return QuorumCall.create(calls); }
@Override public ListenableFuture<RemoteEditLogManifest> getEditLogManifest( final long fromTxnId, final boolean inProgressOk) { return parallelExecutor.submit(new Callable<RemoteEditLogManifest>() { @Override public RemoteEditLogManifest call() throws IOException { GetEditLogManifestResponseProto ret = getProxy().getEditLogManifest( journalId, fromTxnId, inProgressOk); // Update the http port, since we need this to build URLs to any of the // returned logs. constructHttpServerURI(ret); return PBHelper.convert(ret.getManifest()); } }); }
@Override public void selectInputStreams(Collection<EditLogInputStream> streams, long fromTxnId, boolean inProgressOk) throws IOException { QuorumCall<AsyncLogger, RemoteEditLogManifest> q = loggers.getEditLogManifest(fromTxnId, inProgressOk); Map<AsyncLogger, RemoteEditLogManifest> resps = loggers.waitForWriteQuorum(q, selectInputStreamsTimeoutMs, "selectInputStreams"); LOG.debug("selectInputStream manifests:\n" + Joiner.on("\n").withKeyValueSeparator(": ").join(resps)); final PriorityQueue<EditLogInputStream> allStreams = new PriorityQueue<EditLogInputStream>(64, JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR); for (Map.Entry<AsyncLogger, RemoteEditLogManifest> e : resps.entrySet()) { AsyncLogger logger = e.getKey(); RemoteEditLogManifest manifest = e.getValue(); for (RemoteEditLog remoteLog : manifest.getLogs()) { URL url = logger.buildURLToFetchLogs(remoteLog.getStartTxId()); EditLogInputStream elis = EditLogFileInputStream.fromUrl( connectionFactory, url, remoteLog.getStartTxId(), remoteLog.getEndTxId(), remoteLog.isInProgress()); allStreams.add(elis); } } JournalSet.chainAndMakeRedundantStreams(streams, allStreams, fromTxnId); }
@SuppressWarnings("deprecation") @Override public GetEditLogManifestResponseProto getEditLogManifest(String jid, long sinceTxId, boolean inProgressOk) throws IOException { RemoteEditLogManifest manifest = jn.getOrCreateJournal(jid) .getEditLogManifest(sinceTxId, inProgressOk); return GetEditLogManifestResponseProto.newBuilder() .setManifest(PBHelper.convert(manifest)) .setHttpPort(jn.getBoundHttpAddress().getPort()) .setFromURL(jn.getHttpServerURI()) .build(); }
@Test public void testConvertRemoteEditLogManifest() { List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>(); logs.add(new RemoteEditLog(1, 10)); logs.add(new RemoteEditLog(11, 20)); RemoteEditLogManifest m = new RemoteEditLogManifest(logs); RemoteEditLogManifestProto mProto = PBHelper.convert(m); RemoteEditLogManifest m1 = PBHelper.convert(mProto); List<RemoteEditLog> logs1 = m1.getLogs(); assertEquals(logs.size(), logs1.size()); for (int i = 0; i < logs.size(); i++) { compare(logs.get(i), logs1.get(i)); } }
static void doMerge( CheckpointSignature sig, RemoteEditLogManifest manifest, boolean loadImage, FSImage dstImage, FSNamesystem dstNamesystem) throws IOException { NNStorage dstStorage = dstImage.getStorage(); dstStorage.setStorageInfo(sig); if (loadImage) { File file = dstStorage.findImageFile(NameNodeFile.IMAGE, sig.mostRecentCheckpointTxId); if (file == null) { throw new IOException("Couldn't find image file at txid " + sig.mostRecentCheckpointTxId + " even though it should have " + "just been downloaded"); } dstNamesystem.writeLock(); try { dstImage.reloadFromImageFile(file, dstNamesystem); } finally { dstNamesystem.writeUnlock(); } dstNamesystem.imageLoadComplete(); } // error simulation code for junit test CheckpointFaultInjector.getInstance().duringMerge(); Checkpointer.rollForwardByApplyingLogs(manifest, dstImage, dstNamesystem); // The following has the side effect of purging old fsimages/edit logs. dstImage.saveFSImageInAllDirs(dstNamesystem, dstImage.getLastAppliedTxId()); if (!dstNamesystem.isRollingUpgrade()) { dstStorage.writeAll(); } }
