protected void getFields(Properties props, StorageDirectory sd ) throws IOException { setLayoutVersion(props, sd); setStorageType(props, sd); // Read NamespaceID in version before federation if (layoutVersion > FSConstants.FEDERATION_VERSION) { setNamespaceID(props, sd); setcTime(props, sd); } String ssid = props.getProperty(STORAGE_ID); if (ssid == null || !("".equals(storageID) || "".equals(ssid) || storageID.equals(ssid))) throw new InconsistentFSStateException(sd.getRoot(), "has incompatible storage Id."); if ("".equals(storageID)) // update id only if it was empty storageID = ssid; }
public boolean isConversionNeeded(StorageDirectory sd) throws IOException { File oldImageDir = new File(sd.getRoot(), "image"); if (!oldImageDir.exists()) { if(sd.getVersionFile().exists()) throw new InconsistentFSStateException(sd.getRoot(), oldImageDir + " does not exist."); return false; } // check the layout version inside the image file File oldF = new File(oldImageDir, "fsimage"); RandomAccessFile oldFile = new RandomAccessFile(oldF, "rws"); try { oldFile.seek(0); int odlVersion = oldFile.readInt(); if (odlVersion < LAST_PRE_UPGRADE_LAYOUT_VERSION) return false; } finally { oldFile.close(); } return true; }
/** Validate and set block pool ID */ private void setBlockPoolID(File storage, String bpid) throws InconsistentFSStateException { if (bpid == null || bpid.equals("")) { throw new InconsistentFSStateException(storage, "file " + Storage.STORAGE_FILE_VERSION + " has no block pool Id."); } if (!blockpoolID.equals("") && !blockpoolID.equals(bpid)) { throw new InconsistentFSStateException(storage, "Unexepcted blockpoolID " + bpid + " . Expected " + blockpoolID); } setBlockPoolID(bpid); }
/** Check if upgrade is in progress. */ public static void checkUpgrade(NNStorage storage) throws IOException { // Upgrade or rolling upgrade is allowed only if there are // no previous fs states in any of the directories for (Iterator<StorageDirectory> it = storage.dirIterator(false); it.hasNext();) { StorageDirectory sd = it.next(); if (sd.getPreviousDir().exists()) throw new InconsistentFSStateException(sd.getRoot(), "previous fs state should not exist during upgrade. " + "Finalize or rollback first."); } }
void loadFSImageFile(FSNamesystem target, MetaRecoveryContext recovery, FSImageFile imageFile, StartupOption startupOption) throws IOException { LOG.debug("Planning to load image :\n" + imageFile); StorageDirectory sdForProperties = imageFile.sd; storage.readProperties(sdForProperties, startupOption); if (NameNodeLayoutVersion.supports( LayoutVersion.Feature.TXID_BASED_LAYOUT, getLayoutVersion())) { // For txid-based layout, we should have a .md5 file // next to the image file boolean isRollingRollback = RollingUpgradeStartupOption.ROLLBACK .matches(startupOption); loadFSImage(imageFile.getFile(), target, recovery, isRollingRollback); } else if (NameNodeLayoutVersion.supports( LayoutVersion.Feature.FSIMAGE_CHECKSUM, getLayoutVersion())) { // In 0.22, we have the checksum stored in the VERSION file. String md5 = storage.getDeprecatedProperty( NNStorage.DEPRECATED_MESSAGE_DIGEST_PROPERTY); if (md5 == null) { throw new InconsistentFSStateException(sdForProperties.getRoot(), "Message digest property " + NNStorage.DEPRECATED_MESSAGE_DIGEST_PROPERTY + " not set for storage directory " + sdForProperties.getRoot()); } loadFSImage(imageFile.getFile(), new MD5Hash(md5), target, recovery, false); } else { // We don't have any record of the md5sum loadFSImage(imageFile.getFile(), null, target, recovery, false); } }
/** * Analyze backup storage directories for consistency.<br> * Recover from incomplete checkpoints if required.<br> * Read VERSION and fstime files if exist.<br> * Do not load image or edits. * * @throws IOException if the node should shutdown. */ void recoverCreateRead() throws IOException { for (Iterator<StorageDirectory> it = storage.dirIterator(); it.hasNext();) { StorageDirectory sd = it.next(); StorageState curState; try { curState = sd.analyzeStorage(HdfsServerConstants.StartupOption.REGULAR, storage); // sd is locked but not opened switch(curState) { case NON_EXISTENT: // fail if any of the configured storage dirs are inaccessible throw new InconsistentFSStateException(sd.getRoot(), "checkpoint directory does not exist or is not accessible."); case NOT_FORMATTED: // for backup node all directories may be unformatted initially LOG.info("Storage directory " + sd.getRoot() + " is not formatted."); LOG.info("Formatting ..."); sd.clearDirectory(); // create empty current break; case NORMAL: break; default: // recovery is possible sd.doRecover(curState); } if(curState != StorageState.NOT_FORMATTED) { // read and verify consistency with other directories storage.readProperties(sd); } } catch(IOException ioe) { sd.unlock(); throw ioe; } } }
/** Validate and set block pool ID */ private void setBlockPoolID(File storage, String bpid) throws InconsistentFSStateException { if (bpid == null || bpid.equals("")) { throw new InconsistentFSStateException(storage, "file " + STORAGE_FILE_VERSION + " is invalid."); } if (!blockpoolID.equals("") && !blockpoolID.equals(bpid)) { throw new InconsistentFSStateException(storage, "Unexpected blockpoolID " + bpid + ". Expected " + blockpoolID); } blockpoolID = bpid; }
@Override protected void setLayoutVersion(Properties props, StorageDirectory sd) throws IncorrectVersionException, InconsistentFSStateException { int lv = Integer.parseInt(getProperty(props, sd, "layoutVersion")); // For journal node, since it now does not decode but just scan through the // edits, it can handle edits with future version in most of the cases. // Thus currently we may skip the layoutVersion check here. layoutVersion = lv; }
/** Check if upgrade is in progress. */ public static void checkUpgrade(NNStorage storage) throws IOException { // Upgrade or rolling upgrade is allowed only if there are // no previous fs states in any of the local directories for (Iterator<StorageDirectory> it = storage.dirIterator(false); it.hasNext();) { StorageDirectory sd = it.next(); if (sd.getPreviousDir().exists()) throw new InconsistentFSStateException(sd.getRoot(), "previous fs state should not exist during upgrade. " + "Finalize or rollback first."); } }
public static boolean recoverDirectory(StorageDirectory sd, StartupOption startOpt, StorageState curState, boolean checkImport) throws IOException { boolean isFormatted = false; // sd is locked but not opened switch (curState) { case NON_EXISTENT: // name-node fails if any of the configured storage dirs are missing throw new InconsistentFSStateException(sd.getRoot(), "storage directory does not exist or is not accessible."); case NOT_FORMATTED: break; case NORMAL: break; default: // recovery is possible sd.doRecover(curState); } if (curState != StorageState.NOT_FORMATTED && startOpt != StartupOption.ROLLBACK) { // read and verify consistency with other directories sd.read(); isFormatted = true; } if (checkImport && startOpt == StartupOption.IMPORT && isFormatted) // import of a checkpoint is allowed only into empty image directories throw new IOException("Cannot import image from a checkpoint. " + " NameNode already contains an image in " + sd.getRoot()); return isFormatted; }
/** Validate and set namespace ID */ private void setNameSpaceID(File storage, String nsid) throws InconsistentFSStateException { if (nsid == null || nsid.equals("")) { throw new InconsistentFSStateException(storage, "file " + STORAGE_FILE_VERSION + " is invalid."); } int newNsId = Integer.parseInt(nsid); if (namespaceID > 0 && namespaceID != newNsId) { throw new InconsistentFSStateException(storage, "Unexepcted namespaceID " + nsid + " . Expected " + namespaceID); } namespaceID = newNsId; }
@Override protected void getFields(Properties props, StorageDirectory sd) throws IOException { setNamespaceID(props, sd); setcTime(props, sd); String snsid = props.getProperty(NAMESPACE_ID); setNameSpaceID(sd.getRoot(), snsid); String property = props.getProperty(LAYOUT_VERSION); int lv; if (property == null) { Integer topLayout = getTopLevelLayout(sd); if (topLayout == null) { throw new InconsistentFSStateException(sd.getRoot(), "Top level layout and NS level layout do not exist"); } lv = topLayout; } else { lv = Integer.parseInt(property); } if (lv < FSConstants.LAYOUT_VERSION) { // future version throw new InconsistentFSStateException(sd.getRoot(), "has future layout version : " + lv); } layoutVersion = lv; }
void loadFSImageFile(FSNamesystem target, MetaRecoveryContext recovery, FSImageFile imageFile) throws IOException { LOG.debug("Planning to load image :\n" + imageFile); StorageDirectory sdForProperties = imageFile.sd; storage.readProperties(sdForProperties); if (LayoutVersion.supports(Feature.TXID_BASED_LAYOUT, getLayoutVersion())) { // For txid-based layout, we should have a .md5 file // next to the image file loadFSImage(imageFile.getFile(), target, recovery); } else if (LayoutVersion.supports(Feature.FSIMAGE_CHECKSUM, getLayoutVersion())) { // In 0.22, we have the checksum stored in the VERSION file. String md5 = storage.getDeprecatedProperty( NNStorage.DEPRECATED_MESSAGE_DIGEST_PROPERTY); if (md5 == null) { throw new InconsistentFSStateException(sdForProperties.getRoot(), "Message digest property " + NNStorage.DEPRECATED_MESSAGE_DIGEST_PROPERTY + " not set for storage directory " + sdForProperties.getRoot()); } loadFSImage(imageFile.getFile(), new MD5Hash(md5), target, recovery); } else { // We don't have any record of the md5sum loadFSImage(imageFile.getFile(), null, target, recovery); } }
/** Validate and set block pool ID */ private void setBlockPoolID(File storage, String bpid) throws InconsistentFSStateException { if (bpid == null || bpid.equals("")) { throw new InconsistentFSStateException(storage, "file " + STORAGE_FILE_VERSION + " is invalid."); } if (!blockpoolID.equals("") && !blockpoolID.equals(bpid)) { throw new InconsistentFSStateException(storage, "Unexepcted blockpoolID " + bpid + " . Expected " + blockpoolID); } blockpoolID = bpid; }
@Override protected void setFieldsFromProperties(Properties props, StorageDirectory sd) throws IOException { setLayoutVersion(props, sd); setcTime(props, sd); setStorageType(props, sd); setClusterId(props, layoutVersion, sd); // Read NamespaceID in version before federation if (!LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) { setNamespaceID(props, sd); } // valid storage id, storage id may be empty String ssid = props.getProperty("storageID"); if (ssid == null) { throw new InconsistentFSStateException(sd.getRoot(), "file " + STORAGE_FILE_VERSION + " is invalid."); } String sid = getStorageID(); if (!(sid.equals("") || ssid.equals("") || sid.equals(ssid))) { throw new InconsistentFSStateException(sd.getRoot(), "has incompatible storage Id."); } if (sid.equals("")) { // update id only if it was empty setStorageID(ssid); } }
