/** * Private helper methods to save delegation keys and tokens in fsimage */ private synchronized void saveCurrentTokens(DataOutputStream out, String sdPath) throws IOException { StartupProgress prog = NameNode.getStartupProgress(); Step step = new Step(StepType.DELEGATION_TOKENS, sdPath); prog.beginStep(Phase.SAVING_CHECKPOINT, step); prog.setTotal(Phase.SAVING_CHECKPOINT, step, currentTokens.size()); Counter counter = prog.getCounter(Phase.SAVING_CHECKPOINT, step); out.writeInt(currentTokens.size()); Iterator<DelegationTokenIdentifier> iter = currentTokens.keySet() .iterator(); while (iter.hasNext()) { DelegationTokenIdentifier id = iter.next(); id.write(out); DelegationTokenInformation info = currentTokens.get(id); out.writeLong(info.getRenewDate()); counter.increment(); } prog.endStep(Phase.SAVING_CHECKPOINT, step); }
private synchronized void saveAllKeys(DataOutputStream out, String sdPath) throws IOException { StartupProgress prog = NameNode.getStartupProgress(); Step step = new Step(StepType.DELEGATION_KEYS, sdPath); prog.beginStep(Phase.SAVING_CHECKPOINT, step); prog.setTotal(Phase.SAVING_CHECKPOINT, step, currentTokens.size()); Counter counter = prog.getCounter(Phase.SAVING_CHECKPOINT, step); out.writeInt(allKeys.size()); Iterator<Integer> iter = allKeys.keySet().iterator(); while (iter.hasNext()) { Integer key = iter.next(); allKeys.get(key).write(out); counter.increment(); } prog.endStep(Phase.SAVING_CHECKPOINT, step); }
/** * Private helper methods to load Delegation tokens from fsimage */ private synchronized void loadCurrentTokens(DataInput in) throws IOException { StartupProgress prog = NameNode.getStartupProgress(); Step step = new Step(StepType.DELEGATION_TOKENS); prog.beginStep(Phase.LOADING_FSIMAGE, step); int numberOfTokens = in.readInt(); prog.setTotal(Phase.LOADING_FSIMAGE, step, numberOfTokens); Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, step); for (int i = 0; i < numberOfTokens; i++) { DelegationTokenIdentifier id = new DelegationTokenIdentifier(); id.readFields(in); long expiryTime = in.readLong(); addPersistedDelegationToken(id, expiryTime); counter.increment(); } prog.endStep(Phase.LOADING_FSIMAGE, step); }
/** * Private helper method to load delegation keys from fsimage. * @throws IOException on error */ private synchronized void loadAllKeys(DataInput in) throws IOException { StartupProgress prog = NameNode.getStartupProgress(); Step step = new Step(StepType.DELEGATION_KEYS); prog.beginStep(Phase.LOADING_FSIMAGE, step); int numberOfKeys = in.readInt(); prog.setTotal(Phase.LOADING_FSIMAGE, step, numberOfKeys); Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, step); for (int i = 0; i < numberOfKeys; i++) { DelegationKey value = new DelegationKey(); value.readFields(in); addKey(value); counter.increment(); } prog.endStep(Phase.LOADING_FSIMAGE, step); }
/** * Load cache directives from the fsimage */ private void loadDirectives(DataInput in) throws IOException { StartupProgress prog = NameNode.getStartupProgress(); Step step = new Step(StepType.CACHE_ENTRIES); prog.beginStep(Phase.LOADING_FSIMAGE, step); int numDirectives = in.readInt(); prog.setTotal(Phase.LOADING_FSIMAGE, step, numDirectives); Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, step); for (int i = 0; i < numDirectives; i++) { CacheDirectiveInfo info = FSImageSerialization.readCacheDirectiveInfo(in); // Get pool reference by looking it up in the map final String poolName = info.getPool(); CacheDirective directive = new CacheDirective(info.getId(), info.getPath().toUri().getPath(), info.getReplication(), info.getExpiration().getAbsoluteMillis()); addCacheDirective(poolName, directive); counter.increment(); } prog.endStep(Phase.LOADING_FSIMAGE, step); }
/** * load fsimage files assuming only local names are stored. Used when * snapshots are not supported by the layout version. * * @param numFiles number of files expected to be read * @param in image input stream * @param counter Counter to increment for namenode startup progress * @throws IOException */ private void loadLocalNameINodes(long numFiles, DataInput in, Counter counter) throws IOException { assert NameNodeLayoutVersion.supports( LayoutVersion.Feature.FSIMAGE_NAME_OPTIMIZATION, getLayoutVersion()); assert numFiles > 0; // load root loadRoot(in, counter); // have loaded the first file (the root) numFiles--; // load rest of the nodes directory by directory while (numFiles > 0) { numFiles -= loadDirectory(in, counter); } if (numFiles != 0) { throw new IOException("Read unexpect number of files: " + -numFiles); } }
/** * Save children INodes. * @param children The list of children INodes * @param out The DataOutputStream to write * @param inSnapshot Whether the parent directory or its ancestor is in * the deleted list of some snapshot (caused by rename or * deletion) * @param counter Counter to increment for namenode startup progress * @return Number of children that are directory */ private int saveChildren(ReadOnlyList<INode> children, DataOutputStream out, boolean inSnapshot, Counter counter) throws IOException { // Write normal children INode. out.writeInt(children.size()); int dirNum = 0; for(INode child : children) { // print all children first // TODO: for HDFS-5428, we cannot change the format/content of fsimage // here, thus even if the parent directory is in snapshot, we still // do not handle INodeUC as those stored in deleted list saveINode2Image(child, out, false, referenceMap, counter); if (child.isDirectory()) { dirNum++; } else if (inSnapshot && child.isFile() && child.asFile().isUnderConstruction()) { this.snapshotUCMap.put(child.getId(), child.asFile()); } if (checkCancelCounter++ % CHECK_CANCEL_INTERVAL == 0) { context.checkCancelled(); } } return dirNum; }
private void loadSecretManagerSection(InputStream in, StartupProgress prog, Step currentStep) throws IOException { SecretManagerSection s = SecretManagerSection.parseDelimitedFrom(in); int numKeys = s.getNumKeys(), numTokens = s.getNumTokens(); ArrayList<SecretManagerSection.DelegationKey> keys = Lists .newArrayListWithCapacity(numKeys); ArrayList<SecretManagerSection.PersistToken> tokens = Lists .newArrayListWithCapacity(numTokens); for (int i = 0; i < numKeys; ++i) keys.add(SecretManagerSection.DelegationKey.parseDelimitedFrom(in)); prog.setTotal(Phase.LOADING_FSIMAGE, currentStep, numTokens); Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, currentStep); for (int i = 0; i < numTokens; ++i) { tokens.add(SecretManagerSection.PersistToken.parseDelimitedFrom(in)); counter.increment(); } fsn.loadSecretManagerState(s, keys, tokens); }
private void loadCacheManagerSection(InputStream in, StartupProgress prog, Step currentStep) throws IOException { CacheManagerSection s = CacheManagerSection.parseDelimitedFrom(in); int numPools = s.getNumPools(); ArrayList<CachePoolInfoProto> pools = Lists .newArrayListWithCapacity(numPools); ArrayList<CacheDirectiveInfoProto> directives = Lists .newArrayListWithCapacity(s.getNumDirectives()); prog.setTotal(Phase.LOADING_FSIMAGE, currentStep, numPools); Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, currentStep); for (int i = 0; i < numPools; ++i) { pools.add(CachePoolInfoProto.parseDelimitedFrom(in)); counter.increment(); } for (int i = 0; i < s.getNumDirectives(); ++i) directives.add(CacheDirectiveInfoProto.parseDelimitedFrom(in)); fsn.getCacheManager().loadState( new CacheManager.PersistState(s, pools, directives)); }
void loadINodeSection(InputStream in, StartupProgress prog, Step currentStep) throws IOException { INodeSection s = INodeSection.parseDelimitedFrom(in); fsn.dir.resetLastInodeId(s.getLastInodeId()); long numInodes = s.getNumInodes(); LOG.info("Loading " + numInodes + " INodes."); prog.setTotal(Phase.LOADING_FSIMAGE, currentStep, numInodes); Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, currentStep); for (int i = 0; i < numInodes; ++i) { INodeSection.INode p = INodeSection.INode.parseDelimitedFrom(in); if (p.getId() == INodeId.ROOT_INODE_ID) { loadRootINode(p); } else { INode n = loadINode(p); dir.addToInodeMap(n); } counter.increment(); } }
/** * Private helper method to load delegation keys from fsimage. * @param in * @throws IOException */ private synchronized void loadAllKeys(DataInput in) throws IOException { StartupProgress prog = NameNode.getStartupProgress(); Step step = new Step(StepType.DELEGATION_KEYS); prog.beginStep(Phase.LOADING_FSIMAGE, step); int numberOfKeys = in.readInt(); prog.setTotal(Phase.LOADING_FSIMAGE, step, numberOfKeys); Counter counter = prog.getCounter(Phase.LOADING_FSIMAGE, step); for (int i = 0; i < numberOfKeys; i++) { DelegationKey value = new DelegationKey(); value.readFields(in); addKey(value); counter.increment(); } prog.endStep(Phase.LOADING_FSIMAGE, step); }
/** * load fsimage files assuming only local names are stored * * @param numFiles number of files expected to be read * @param in image input stream * @param counter Counter to increment for namenode startup progress * @throws IOException */ private void loadLocalNameINodes(long numFiles, DataInput in, Counter counter) throws IOException { assert LayoutVersion.supports(Feature.FSIMAGE_NAME_OPTIMIZATION, getLayoutVersion()); assert numFiles > 0; // load root loadRoot(in, counter); // have loaded the first file (the root) numFiles--; // load rest of the nodes directory by directory while (numFiles > 0) { numFiles -= loadDirectory(in, counter); } if (numFiles != 0) { throw new IOException("Read unexpect number of files: " + -numFiles); } }
/** * Save children INodes. * @param children The list of children INodes * @param out The DataOutputStream to write * @param counter Counter to increment for namenode startup progress * @return Number of children that are directory */ private int saveChildren(ReadOnlyList<INode> children, DataOutputStream out, Counter counter) throws IOException { // Write normal children INode. out.writeInt(children.size()); int dirNum = 0; int i = 0; for(INode child : children) { // print all children first saveINode2Image(child, out, false, referenceMap, counter); if (child.isDirectory()) { dirNum++; } if (i++ % 50 == 0) { context.checkCancelled(); } } return dirNum; }
