/** * Return true if this storage dir can roll back to the previous storage * state, false otherwise. The NN will refuse to run the rollback operation * unless at least one JM or fsimage storage directory can roll back. * * @param storage the storage info for the current state * @param prevStorage the storage info for the previous (unupgraded) state * @param targetLayoutVersion the layout version we intend to roll back to * @return true if this JM can roll back, false otherwise. * @throws IOException in the event of error */ static boolean canRollBack(StorageDirectory sd, StorageInfo storage, StorageInfo prevStorage, int targetLayoutVersion) throws IOException { File prevDir = sd.getPreviousDir(); if (!prevDir.exists()) { // use current directory then LOG.info("Storage directory " + sd.getRoot() + " does not contain previous fs state."); // read and verify consistency with other directories storage.readProperties(sd); return false; } // read and verify consistency of the prev dir prevStorage.readPreviousVersionProperties(sd); if (prevStorage.getLayoutVersion() != targetLayoutVersion) { throw new IOException( "Cannot rollback to storage version " + prevStorage.getLayoutVersion() + " using this version of the NameNode, which uses storage version " + targetLayoutVersion + ". " + "Please use the previous version of HDFS to perform the rollback."); } return true; }
private boolean checkStorageInfoOrSendError(JNStorage storage, HttpServletRequest request, HttpServletResponse response) throws IOException { int myNsId = storage.getNamespaceID(); String myClusterId = storage.getClusterID(); String theirStorageInfoString = StringEscapeUtils.escapeHtml( request.getParameter(STORAGEINFO_PARAM)); if (theirStorageInfoString != null) { int theirNsId = StorageInfo.getNsIdFromColonSeparatedString( theirStorageInfoString); String theirClusterId = StorageInfo.getClusterIdFromColonSeparatedString( theirStorageInfoString); if (myNsId != theirNsId || !myClusterId.equals(theirClusterId)) { String msg = "This node has namespaceId '" + myNsId + " and clusterId '" + myClusterId + "' but the requesting node expected '" + theirNsId + "' and '" + theirClusterId + "'"; response.sendError(HttpServletResponse.SC_FORBIDDEN, msg); LOG.warn("Received an invalid request file transfer request from " + request.getRemoteAddr() + ": " + msg); return false; } } return true; }
@Override public Boolean canRollBack(String journalId, StorageInfo storage, StorageInfo prevStorage, int targetLayoutVersion) throws IOException { try { CanRollBackResponseProto response = rpcProxy.canRollBack( NULL_CONTROLLER, CanRollBackRequestProto.newBuilder() .setJid(convertJournalId(journalId)) .setStorage(PBHelper.convert(storage)) .setPrevStorage(PBHelper.convert(prevStorage)) .setTargetLayoutVersion(targetLayoutVersion) .build()); return response.getCanRollBack(); } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } }
@Test public void testConvertNamenodeRegistration() { StorageInfo info = getStorageInfo(NodeType.NAME_NODE); NamenodeRegistration reg = new NamenodeRegistration("address:999", "http:1000", info, NamenodeRole.NAMENODE); NamenodeRegistrationProto regProto = PBHelper.convert(reg); NamenodeRegistration reg2 = PBHelper.convert(regProto); assertEquals(reg.getAddress(), reg2.getAddress()); assertEquals(reg.getClusterID(), reg2.getClusterID()); assertEquals(reg.getCTime(), reg2.getCTime()); assertEquals(reg.getHttpAddress(), reg2.getHttpAddress()); assertEquals(reg.getLayoutVersion(), reg2.getLayoutVersion()); assertEquals(reg.getNamespaceID(), reg2.getNamespaceID()); assertEquals(reg.getRegistrationID(), reg2.getRegistrationID()); assertEquals(reg.getRole(), reg2.getRole()); assertEquals(reg.getVersion(), reg2.getVersion()); }
/** * Create a <code>version</code> file for datanode inside the specified parent * directory. If such a file already exists, it will be overwritten. * The given version string will be written to the file as the layout * version. None of the parameters may be null. * * @param parent directory where namenode VERSION file is stored * @param version StorageInfo to create VERSION file from * @param bpid Block pool Id * @param bpidToWrite Block pool Id to write into the version file */ public static void createDataNodeVersionFile(File[] parent, StorageInfo version, String bpid, String bpidToWrite) throws IOException { DataStorage storage = new DataStorage(version); storage.setDatanodeUuid("FixedDatanodeUuid"); File[] versionFiles = new File[parent.length]; for (int i = 0; i < parent.length; i++) { File versionFile = new File(parent[i], "VERSION"); StorageDirectory sd = new StorageDirectory(parent[i].getParentFile()); storage.createStorageID(sd, false); storage.writeProperties(versionFile, sd); versionFiles[i] = versionFile; File bpDir = BlockPoolSliceStorage.getBpRoot(bpid, parent[i]); createBlockPoolVersionFile(bpDir, version, bpidToWrite); } }
static String getParamStringForImage(NameNodeFile nnf, long txid, StorageInfo remoteStorageInfo) { final String imageType = nnf == null ? "" : "&" + IMAGE_FILE_TYPE + "=" + nnf.name(); return "getimage=1&" + TXID_PARAM + "=" + txid + imageType + "&" + STORAGEINFO_PARAM + "=" + remoteStorageInfo.toColonSeparatedString(); }
static String getParamStringForLog(RemoteEditLog log, StorageInfo remoteStorageInfo) { return "getedit=1&" + START_TXID_PARAM + "=" + log.getStartTxId() + "&" + END_TXID_PARAM + "=" + log.getEndTxId() + "&" + STORAGEINFO_PARAM + "=" + remoteStorageInfo.toColonSeparatedString(); }
public CheckpointSignature(StorageInfo info, String blockpoolID, long mostRecentCheckpointTxId, long curSegmentTxId) { super(info); this.blockpoolID = blockpoolID; this.mostRecentCheckpointTxId = mostRecentCheckpointTxId; this.curSegmentTxId = curSegmentTxId; }
public synchronized boolean canRollBackSharedLog(StorageInfo prevStorage, int targetLayoutVersion) throws IOException { for (JournalAndStream jas : journalSet.getAllJournalStreams()) { if (jas.isShared()) { return jas.getManager().canRollBack(storage, prevStorage, targetLayoutVersion); } } throw new IOException("No shared log found."); }
public NamenodeRegistration(String address, String httpAddress, StorageInfo storageInfo, NamenodeRole role) { super(storageInfo); this.rpcAddress = address; this.httpAddress = httpAddress; this.role = role; }
public DatanodeRegistration(DatanodeID dn, StorageInfo info, ExportedBlockKeys keys, String softwareVersion) { super(dn); this.storageInfo = info; this.exportedKeys = keys; this.softwareVersion = softwareVersion; }
/** * Initializes the {@link #data}. The initialization is done only once, when * handshake with the the first namenode is completed. */ private void initStorage(final NamespaceInfo nsInfo) throws IOException { final FsDatasetSpi.Factory<? extends FsDatasetSpi<?>> factory = FsDatasetSpi.Factory.getFactory(conf); if (!factory.isSimulated()) { final StartupOption startOpt = getStartupOption(conf); if (startOpt == null) { throw new IOException("Startup option not set."); } final String bpid = nsInfo.getBlockPoolID(); //read storage info, lock data dirs and transition fs state if necessary synchronized (this) { storage.recoverTransitionRead(this, nsInfo, dataDirs, startOpt); } final StorageInfo bpStorage = storage.getBPStorage(bpid); LOG.info("Setting up storage: nsid=" + bpStorage.getNamespaceID() + ";bpid=" + bpid + ";lv=" + storage.getLayoutVersion() + ";nsInfo=" + nsInfo + ";dnuuid=" + storage.getDatanodeUuid()); } // If this is a newly formatted DataNode then assign a new DatanodeUuid. checkDatanodeUuid(); synchronized(this) { if (data == null) { data = factory.newInstance(this, storage, conf); } } }
public static CheckpointSignatureProto convert(CheckpointSignature s) { return CheckpointSignatureProto.newBuilder() .setBlockPoolId(s.getBlockpoolID()) .setCurSegmentTxId(s.getCurSegmentTxId()) .setMostRecentCheckpointTxId(s.getMostRecentCheckpointTxId()) .setStorageInfo(PBHelper.convert((StorageInfo) s)).build(); }
public static NamespaceInfoProto convert(NamespaceInfo info) { return NamespaceInfoProto.newBuilder() .setBlockPoolID(info.getBlockPoolID()) .setBuildVersion(info.getBuildVersion()) .setUnused(0) .setStorageInfo(PBHelper.convert((StorageInfo)info)) .setSoftwareVersion(info.getSoftwareVersion()) .setCapabilities(info.getCapabilities()) .build(); }
public QuorumCall<AsyncLogger, Void> doUpgrade(StorageInfo sInfo) { Map<AsyncLogger, ListenableFuture<Void>> calls = Maps.newHashMap(); for (AsyncLogger logger : loggers) { ListenableFuture<Void> future = logger.doUpgrade(sInfo); calls.put(logger, future); } return QuorumCall.create(calls); }
public QuorumCall<AsyncLogger, Boolean> canRollBack(StorageInfo storage, StorageInfo prevStorage, int targetLayoutVersion) { Map<AsyncLogger, ListenableFuture<Boolean>> calls = Maps.newHashMap(); for (AsyncLogger logger : loggers) { ListenableFuture<Boolean> future = logger.canRollBack(storage, prevStorage, targetLayoutVersion); calls.put(logger, future); } return QuorumCall.create(calls); }
@Override public ListenableFuture<Void> doUpgrade(final StorageInfo sInfo) { return singleThreadExecutor.submit(new Callable<Void>() { @Override public Void call() throws IOException { getProxy().doUpgrade(journalId, sInfo); return null; } }); }
@Override public ListenableFuture<Boolean> canRollBack(final StorageInfo storage, final StorageInfo prevStorage, final int targetLayoutVersion) { return singleThreadExecutor.submit(new Callable<Boolean>() { @Override public Boolean call() throws IOException { return getProxy().canRollBack(journalId, storage, prevStorage, targetLayoutVersion); } }); }
@Override public DoUpgradeResponseProto doUpgrade(RpcController controller, DoUpgradeRequestProto request) throws ServiceException { StorageInfo si = PBHelper.convert(request.getSInfo(), NodeType.JOURNAL_NODE); try { impl.doUpgrade(convert(request.getJid()), si); return DoUpgradeResponseProto.getDefaultInstance(); } catch (IOException e) { throw new ServiceException(e); } }
@Override public CanRollBackResponseProto canRollBack(RpcController controller, CanRollBackRequestProto request) throws ServiceException { try { StorageInfo si = PBHelper.convert(request.getStorage(), NodeType.JOURNAL_NODE); Boolean result = impl.canRollBack(convert(request.getJid()), si, PBHelper.convert(request.getPrevStorage(), NodeType.JOURNAL_NODE), request.getTargetLayoutVersion()); return CanRollBackResponseProto.newBuilder() .setCanRollBack(result) .build(); } catch (IOException e) { throw new ServiceException(e); } }
@Override public void doUpgrade(String journalId, StorageInfo sInfo) throws IOException { try { rpcProxy.doUpgrade(NULL_CONTROLLER, DoUpgradeRequestProto.newBuilder() .setJid(convertJournalId(journalId)) .setSInfo(PBHelper.convert(sInfo)) .build()); } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } }
@Test public void testConvertStoragInfo() { StorageInfo info = getStorageInfo(NodeType.NAME_NODE); StorageInfoProto infoProto = PBHelper.convert(info); StorageInfo info2 = PBHelper.convert(infoProto, NodeType.NAME_NODE); assertEquals(info.getClusterID(), info2.getClusterID()); assertEquals(info.getCTime(), info2.getCTime()); assertEquals(info.getLayoutVersion(), info2.getLayoutVersion()); assertEquals(info.getNamespaceID(), info2.getNamespaceID()); }
@Test public void testConvertDatanodeRegistration() { DatanodeID dnId = DFSTestUtil.getLocalDatanodeID(); BlockKey[] keys = new BlockKey[] { getBlockKey(2), getBlockKey(3) }; ExportedBlockKeys expKeys = new ExportedBlockKeys(true, 9, 10, getBlockKey(1), keys); DatanodeRegistration reg = new DatanodeRegistration(dnId, new StorageInfo(NodeType.DATA_NODE), expKeys, "3.0.0"); DatanodeRegistrationProto proto = PBHelper.convert(reg); DatanodeRegistration reg2 = PBHelper.convert(proto); compare(reg.getStorageInfo(), reg2.getStorageInfo()); compare(reg.getExportedKeys(), reg2.getExportedKeys()); compare(reg, reg2); assertEquals(reg.getSoftwareVersion(), reg2.getSoftwareVersion()); }
/** * Create a <code>version</code> file for namenode inside the specified parent * directory. If such a file already exists, it will be overwritten. * The given version string will be written to the file as the layout * version. None of the parameters may be null. * * @param parent directory where namenode VERSION file is stored * @param version StorageInfo to create VERSION file from * @param bpid Block pool Id * * @return the created version file */ public static File[] createNameNodeVersionFile(Configuration conf, File[] parent, StorageInfo version, String bpid) throws IOException { Storage storage = new NNStorage(conf, Collections.<URI>emptyList(), Collections.<URI>emptyList()); storage.setStorageInfo(version); File[] versionFiles = new File[parent.length]; for (int i = 0; i < parent.length; i++) { versionFiles[i] = new File(parent[i], "VERSION"); StorageDirectory sd = new StorageDirectory(parent[i].getParentFile()); storage.writeProperties(versionFiles[i], sd); } return versionFiles; }
public static void createBlockPoolVersionFile(File bpDir, StorageInfo version, String bpid) throws IOException { // Create block pool version files if (DataNodeLayoutVersion.supports( LayoutVersion.Feature.FEDERATION, version.layoutVersion)) { File bpCurDir = new File(bpDir, Storage.STORAGE_DIR_CURRENT); BlockPoolSliceStorage bpStorage = new BlockPoolSliceStorage(version, bpid); File versionFile = new File(bpCurDir, "VERSION"); StorageDirectory sd = new StorageDirectory(bpDir); bpStorage.writeProperties(versionFile, sd); } }
