private void output(Configuration conf, FileSummary summary, FileInputStream fin, ArrayList<FileSummary.Section> sections) throws IOException { InputStream is; long startTime = Time.monotonicNow(); out.println(getHeader()); for (FileSummary.Section section : sections) { if (SectionName.fromString(section.getName()) == SectionName.INODE) { fin.getChannel().position(section.getOffset()); is = FSImageUtil.wrapInputStreamForCompression(conf, summary.getCodec(), new BufferedInputStream(new LimitInputStream( fin, section.getLength()))); outputINodes(is); } } long timeTaken = Time.monotonicNow() - startTime; LOG.debug("Time to output inodes: {}ms", timeTaken); }
/** Load the directories in the INode section. */ private void loadDirectories( FileInputStream fin, List<FileSummary.Section> sections, FileSummary summary, Configuration conf) throws IOException { LOG.info("Loading directories"); long startTime = Time.monotonicNow(); for (FileSummary.Section section : sections) { if (SectionName.fromString(section.getName()) == SectionName.INODE) { fin.getChannel().position(section.getOffset()); InputStream is = FSImageUtil.wrapInputStreamForCompression(conf, summary.getCodec(), new BufferedInputStream(new LimitInputStream( fin, section.getLength()))); loadDirectoriesInINodeSection(is); } } long timeTaken = Time.monotonicNow() - startTime; LOG.info("Finished loading directories in {}ms", timeTaken); }
private void loadINodeDirSection( FileInputStream fin, List<FileSummary.Section> sections, FileSummary summary, Configuration conf, List<Long> refIdList) throws IOException { LOG.info("Loading INode directory section."); long startTime = Time.monotonicNow(); for (FileSummary.Section section : sections) { if (SectionName.fromString(section.getName()) == SectionName.INODE_DIR) { fin.getChannel().position(section.getOffset()); InputStream is = FSImageUtil.wrapInputStreamForCompression(conf, summary.getCodec(), new BufferedInputStream( new LimitInputStream(fin, section.getLength()))); buildNamespace(is, refIdList); } } long timeTaken = Time.monotonicNow() - startTime; LOG.info("Finished loading INode directory section in {}ms", timeTaken); }
void visit(RandomAccessFile file) throws IOException { if (!FSImageUtil.checkFileFormat(file)) { throw new IOException("Unrecognized FSImage"); } FileSummary summary = FSImageUtil.loadSummary(file); try (FileInputStream in = new FileInputStream(file.getFD())) { for (FileSummary.Section s : summary.getSectionsList()) { if (SectionName.fromString(s.getName()) != SectionName.INODE) { continue; } in.getChannel().position(s.getOffset()); InputStream is = FSImageUtil.wrapInputStreamForCompression(conf, summary.getCodec(), new BufferedInputStream(new LimitInputStream( in, s.getLength()))); run(is); output(); } } }
private void output(Configuration conf, FileSummary summary, FileInputStream fin, ArrayList<FileSummary.Section> sections) throws IOException { InputStream is; long startTime = Time.monotonicNow(); for (FileSummary.Section section : sections) { if (SectionName.fromString(section.getName()) == SectionName.INODE) { fin.getChannel().position(section.getOffset()); is = FSImageUtil.wrapInputStreamForCompression(conf, summary.getCodec(), new BufferedInputStream(new LimitInputStream( fin, section.getLength()))); outputINodes(is); } } long timeTaken = Time.monotonicNow() - startTime; LOG.debug("Time to output inodes: {}ms", timeTaken); }
private void loadINodeDirSection( FileInputStream fin, List<FileSummary.Section> sections, FileSummary summary, Configuration conf) throws IOException { LOG.info("Loading INode directory section."); long startTime = Time.monotonicNow(); for (FileSummary.Section section : sections) { if (SectionName.fromString(section.getName()) == SectionName.INODE_DIR) { fin.getChannel().position(section.getOffset()); InputStream is = FSImageUtil.wrapInputStreamForCompression(conf, summary.getCodec(), new BufferedInputStream( new LimitInputStream(fin, section.getLength()))); buildNamespace(is); } } long timeTaken = Time.monotonicNow() - startTime; LOG.info("Finished loading INode directory section in {}ms", timeTaken); }
void visit(RandomAccessFile file) throws IOException { if (!FSImageUtil.checkFileFormat(file)) { throw new IOException("Unrecognized FSImage"); } FileSummary summary = FSImageUtil.loadSummary(file); FileInputStream in = null; try { in = new FileInputStream(file.getFD()); for (FileSummary.Section s : summary.getSectionsList()) { if (SectionName.fromString(s.getName()) != SectionName.INODE) { continue; } in.getChannel().position(s.getOffset()); InputStream is = FSImageUtil.wrapInputStreamForCompression(conf, summary.getCodec(), new BufferedInputStream(new LimitInputStream( in, s.getLength()))); run(is); output(); } } finally { IOUtils.cleanup(null, in); } }
public void visit(RandomAccessFile file) throws IOException { Configuration conf = new Configuration(); if (!FSImageUtil.checkFileFormat(file)) { throw new IOException("Unrecognized FSImage"); } FileSummary summary = FSImageUtil.loadSummary(file); try (FileInputStream fin = new FileInputStream(file.getFD())) { InputStream is; ArrayList<FileSummary.Section> sections = Lists.newArrayList(summary.getSectionsList()); Collections.sort(sections, new Comparator<FileSummary.Section>() { @Override public int compare(FsImageProto.FileSummary.Section s1, FsImageProto.FileSummary.Section s2) { FSImageFormatProtobuf.SectionName n1 = FSImageFormatProtobuf.SectionName.fromString(s1.getName()); FSImageFormatProtobuf.SectionName n2 = FSImageFormatProtobuf.SectionName.fromString(s2.getName()); if (n1 == null) { return n2 == null ? 0 : -1; } else if (n2 == null) { return -1; } else { return n1.ordinal() - n2.ordinal(); } } }); ImmutableList<Long> refIdList = null; for (FileSummary.Section section : sections) { fin.getChannel().position(section.getOffset()); is = FSImageUtil.wrapInputStreamForCompression(conf, summary.getCodec(), new BufferedInputStream(new LimitInputStream( fin, section.getLength()))); switch (SectionName.fromString(section.getName())) { case STRING_TABLE: LOG.info("Loading string table"); stringTable = FSImageLoader.loadStringTable(is); break; case INODE_REFERENCE: // Load INodeReference so that all INodes can be processed. // Snapshots are not handled and will just be ignored for now. LOG.info("Loading inode references"); refIdList = FSImageLoader.loadINodeReferenceSection(is); break; default: break; } } loadDirectories(fin, sections, summary, conf); loadINodeDirSection(fin, sections, summary, conf, refIdList); metadataMap.sync(); output(conf, summary, fin, sections); } }
public void visit(RandomAccessFile file) throws IOException { if (!FSImageUtil.checkFileFormat(file)) { throw new IOException("Unrecognized FSImage"); } FileSummary summary = FSImageUtil.loadSummary(file); try (FileInputStream fin = new FileInputStream(file.getFD())) { out.print("<?xml version=\"1.0\"?>\n<fsimage>"); ArrayList<FileSummary.Section> sections = Lists.newArrayList(summary .getSectionsList()); Collections.sort(sections, new Comparator<FileSummary.Section>() { @Override public int compare(FileSummary.Section s1, FileSummary.Section s2) { SectionName n1 = SectionName.fromString(s1.getName()); SectionName n2 = SectionName.fromString(s2.getName()); if (n1 == null) { return n2 == null ? 0 : -1; } else if (n2 == null) { return -1; } else { return n1.ordinal() - n2.ordinal(); } } }); for (FileSummary.Section s : sections) { fin.getChannel().position(s.getOffset()); InputStream is = FSImageUtil.wrapInputStreamForCompression(conf, summary.getCodec(), new BufferedInputStream(new LimitInputStream( fin, s.getLength()))); switch (SectionName.fromString(s.getName())) { case NS_INFO: dumpNameSection(is); break; case STRING_TABLE: loadStringTable(is); break; case INODE: dumpINodeSection(is); break; case INODE_REFERENCE: dumpINodeReferenceSection(is); break; case INODE_DIR: dumpINodeDirectorySection(is); break; case FILES_UNDERCONSTRUCTION: dumpFileUnderConstructionSection(is); break; case SNAPSHOT: dumpSnapshotSection(is); break; case SNAPSHOT_DIFF: dumpSnapshotDiffSection(is); break; case SECRET_MANAGER: dumpSecretManagerSection(is); break; case CACHE_MANAGER: dumpCacheManagerSection(is); break; default: break; } } out.print("</fsimage>\n"); } }
/** * Load fsimage into the memory. * @param inputFile the filepath of the fsimage to load. * @return FSImageLoader * @throws IOException if failed to load fsimage. */ static FSImageLoader load(String inputFile) throws IOException { Configuration conf = new Configuration(); RandomAccessFile file = new RandomAccessFile(inputFile, "r"); if (!FSImageUtil.checkFileFormat(file)) { throw new IOException("Unrecognized FSImage"); } FsImageProto.FileSummary summary = FSImageUtil.loadSummary(file); try (FileInputStream fin = new FileInputStream(file.getFD())) { // Map to record INodeReference to the referred id ImmutableList<Long> refIdList = null; String[] stringTable = null; byte[][] inodes = null; Map<Long, long[]> dirmap = null; ArrayList<FsImageProto.FileSummary.Section> sections = Lists.newArrayList(summary.getSectionsList()); Collections.sort(sections, new Comparator<FsImageProto.FileSummary.Section>() { @Override public int compare(FsImageProto.FileSummary.Section s1, FsImageProto.FileSummary.Section s2) { FSImageFormatProtobuf.SectionName n1 = FSImageFormatProtobuf.SectionName.fromString(s1.getName()); FSImageFormatProtobuf.SectionName n2 = FSImageFormatProtobuf.SectionName.fromString(s2.getName()); if (n1 == null) { return n2 == null ? 0 : -1; } else if (n2 == null) { return -1; } else { return n1.ordinal() - n2.ordinal(); } } }); for (FsImageProto.FileSummary.Section s : sections) { fin.getChannel().position(s.getOffset()); InputStream is = FSImageUtil.wrapInputStreamForCompression(conf, summary.getCodec(), new BufferedInputStream(new LimitInputStream( fin, s.getLength()))); if (LOG.isDebugEnabled()) { LOG.debug("Loading section " + s.getName() + " length: " + s.getLength ()); } switch (FSImageFormatProtobuf.SectionName.fromString(s.getName())) { case STRING_TABLE: stringTable = loadStringTable(is); break; case INODE: inodes = loadINodeSection(is); break; case INODE_REFERENCE: refIdList = loadINodeReferenceSection(is); break; case INODE_DIR: dirmap = loadINodeDirectorySection(is, refIdList); break; default: break; } } return new FSImageLoader(stringTable, inodes, dirmap); } }
public void visit(RandomAccessFile file) throws IOException { Configuration conf = new Configuration(); if (!FSImageUtil.checkFileFormat(file)) { throw new IOException("Unrecognized FSImage"); } FileSummary summary = FSImageUtil.loadSummary(file); try (FileInputStream fin = new FileInputStream(file.getFD())) { InputStream is; ArrayList<FileSummary.Section> sections = Lists.newArrayList(summary.getSectionsList()); Collections.sort(sections, new Comparator<FileSummary.Section>() { @Override public int compare(FsImageProto.FileSummary.Section s1, FsImageProto.FileSummary.Section s2) { FSImageFormatProtobuf.SectionName n1 = FSImageFormatProtobuf.SectionName.fromString(s1.getName()); FSImageFormatProtobuf.SectionName n2 = FSImageFormatProtobuf.SectionName.fromString(s2.getName()); if (n1 == null) { return n2 == null ? 0 : -1; } else if (n2 == null) { return -1; } else { return n1.ordinal() - n2.ordinal(); } } }); for (FileSummary.Section section : sections) { fin.getChannel().position(section.getOffset()); is = FSImageUtil.wrapInputStreamForCompression(conf, summary.getCodec(), new BufferedInputStream(new LimitInputStream( fin, section.getLength()))); switch (SectionName.fromString(section.getName())) { case STRING_TABLE: stringTable = FSImageLoader.loadStringTable(is); break; default: break; } } loadDirectories(fin, sections, summary, conf); loadINodeDirSection(fin, sections, summary, conf); metadataMap.sync(); output(conf, summary, fin, sections); } }
/** * Load fsimage into the memory. * @param inputFile the filepath of the fsimage to load. * @return FSImageLoader * @throws IOException if failed to load fsimage. */ static FSImageLoader load(String inputFile) throws IOException { Configuration conf = new Configuration(); RandomAccessFile file = new RandomAccessFile(inputFile, "r"); if (!FSImageUtil.checkFileFormat(file)) { throw new IOException("Unrecognized FSImage"); } FsImageProto.FileSummary summary = FSImageUtil.loadSummary(file); try (FileInputStream fin = new FileInputStream(file.getFD())) { // Map to record INodeReference to the referred id ImmutableList<Long> refIdList = null; String[] stringTable = null; byte[][] inodes = null; Map<Long, long[]> dirmap = null; ArrayList<FsImageProto.FileSummary.Section> sections = Lists.newArrayList(summary.getSectionsList()); Collections.sort(sections, new Comparator<FsImageProto.FileSummary.Section>() { @Override public int compare(FsImageProto.FileSummary.Section s1, FsImageProto.FileSummary.Section s2) { FSImageFormatProtobuf.SectionName n1 = FSImageFormatProtobuf.SectionName.fromString(s1.getName()); FSImageFormatProtobuf.SectionName n2 = FSImageFormatProtobuf.SectionName.fromString(s2.getName()); if (n1 == null) { return n2 == null ? 0 : -1; } else if (n2 == null) { return -1; } else { return n1.ordinal() - n2.ordinal(); } } }); for (FsImageProto.FileSummary.Section s : sections) { fin.getChannel().position(s.getOffset()); InputStream is = FSImageUtil.wrapInputStreamForCompression(conf, summary.getCodec(), new BufferedInputStream(new LimitInputStream( fin, s.getLength()))); LOG.debug("Loading section " + s.getName() + " length: " + s.getLength ()); switch (FSImageFormatProtobuf.SectionName.fromString(s.getName())) { case STRING_TABLE: stringTable = loadStringTable(is); break; case INODE: inodes = loadINodeSection(is); break; case INODE_REFERENCE: refIdList = loadINodeReferenceSection(is); break; case INODE_DIR: dirmap = loadINodeDirectorySection(is, refIdList); break; default: break; } } return new FSImageLoader(stringTable, inodes, dirmap); } }
public void visit(RandomAccessFile file) throws IOException { if (!FSImageUtil.checkFileFormat(file)) { throw new IOException("Unrecognized FSImage"); } FileSummary summary = FSImageUtil.loadSummary(file); FileInputStream fin = null; try { fin = new FileInputStream(file.getFD()); out.print("<?xml version=\"1.0\"?>\n<fsimage>"); ArrayList<FileSummary.Section> sections = Lists.newArrayList(summary .getSectionsList()); Collections.sort(sections, new Comparator<FileSummary.Section>() { @Override public int compare(FileSummary.Section s1, FileSummary.Section s2) { SectionName n1 = SectionName.fromString(s1.getName()); SectionName n2 = SectionName.fromString(s2.getName()); if (n1 == null) { return n2 == null ? 0 : -1; } else if (n2 == null) { return -1; } else { return n1.ordinal() - n2.ordinal(); } } }); for (FileSummary.Section s : sections) { fin.getChannel().position(s.getOffset()); InputStream is = FSImageUtil.wrapInputStreamForCompression(conf, summary.getCodec(), new BufferedInputStream(new LimitInputStream( fin, s.getLength()))); switch (SectionName.fromString(s.getName())) { case NS_INFO: dumpNameSection(is); break; case STRING_TABLE: loadStringTable(is); break; case INODE: dumpINodeSection(is); break; case INODE_REFERENCE: dumpINodeReferenceSection(is); break; case INODE_DIR: dumpINodeDirectorySection(is); break; case FILES_UNDERCONSTRUCTION: dumpFileUnderConstructionSection(is); break; case SNAPSHOT: dumpSnapshotSection(is); break; case SNAPSHOT_DIFF: dumpSnapshotDiffSection(is); break; case SECRET_MANAGER: dumpSecretManagerSection(is); break; case CACHE_MANAGER: dumpCacheManagerSection(is); break; default: break; } } out.print("</fsimage>\n"); } finally { IOUtils.cleanup(null, fin); } }
/** * Load fsimage into the memory. * @param inputFile the filepath of the fsimage to load. * @return FSImageLoader * @throws IOException if failed to load fsimage. */ static FSImageLoader load(String inputFile) throws IOException { Configuration conf = new Configuration(); RandomAccessFile file = new RandomAccessFile(inputFile, "r"); if (!FSImageUtil.checkFileFormat(file)) { throw new IOException("Unrecognized FSImage"); } FsImageProto.FileSummary summary = FSImageUtil.loadSummary(file); FileInputStream fin = null; try { // Map to record INodeReference to the referred id ImmutableList<Long> refIdList = null; String[] stringTable = null; byte[][] inodes = null; Map<Long, long[]> dirmap = null; fin = new FileInputStream(file.getFD()); ArrayList<FsImageProto.FileSummary.Section> sections = Lists.newArrayList(summary.getSectionsList()); Collections.sort(sections, new Comparator<FsImageProto.FileSummary.Section>() { @Override public int compare(FsImageProto.FileSummary.Section s1, FsImageProto.FileSummary.Section s2) { FSImageFormatProtobuf.SectionName n1 = FSImageFormatProtobuf.SectionName.fromString(s1.getName()); FSImageFormatProtobuf.SectionName n2 = FSImageFormatProtobuf.SectionName.fromString(s2.getName()); if (n1 == null) { return n2 == null ? 0 : -1; } else if (n2 == null) { return -1; } else { return n1.ordinal() - n2.ordinal(); } } }); for (FsImageProto.FileSummary.Section s : sections) { fin.getChannel().position(s.getOffset()); InputStream is = FSImageUtil.wrapInputStreamForCompression(conf, summary.getCodec(), new BufferedInputStream(new LimitInputStream( fin, s.getLength()))); LOG.debug("Loading section " + s.getName() + " length: " + s.getLength ()); switch (FSImageFormatProtobuf.SectionName.fromString(s.getName())) { case STRING_TABLE: stringTable = loadStringTable(is); break; case INODE: inodes = loadINodeSection(is); break; case INODE_REFERENCE: refIdList = loadINodeReferenceSection(is); break; case INODE_DIR: dirmap = loadINodeDirectorySection(is, refIdList); break; default: break; } } return new FSImageLoader(stringTable, inodes, dirmap); } finally { IOUtils.cleanup(null, fin); } }
public void visit(RandomAccessFile file) throws IOException { if (!FSImageUtil.checkFileFormat(file)) { throw new IOException("Unrecognized FSImage"); } FileSummary summary = FSImageUtil.loadSummary(file); FileInputStream fin = null; try { fin = new FileInputStream(file.getFD()); ArrayList<FileSummary.Section> sections = Lists.newArrayList(summary .getSectionsList()); Collections.sort(sections, new Comparator<FileSummary.Section>() { @Override public int compare(FileSummary.Section s1, FileSummary.Section s2) { SectionName n1 = SectionName.fromString(s1.getName()); SectionName n2 = SectionName.fromString(s2.getName()); if (n1 == null) { return n2 == null ? 0 : -1; } else if (n2 == null) { return -1; } else { return n1.ordinal() - n2.ordinal(); } } }); for (FileSummary.Section s : sections) { fin.getChannel().position(s.getOffset()); InputStream is = FSImageUtil.wrapInputStreamForCompression(conf, summary.getCodec(), new BufferedInputStream(new LimitInputStream( fin, s.getLength()))); switch (SectionName.fromString(s.getName())) { case STRING_TABLE: loadStringTable(is); break; case INODE: loadINodeSection(is); break; case INODE_REFERENCE: loadINodeReferenceSection(is); break; case INODE_DIR: loadINodeDirectorySection(is); break; default: break; } } list("", INodeId.ROOT_INODE_ID); } finally { IOUtils.cleanup(null, fin); } }