/** * Load the filenames of the directories from the INode section. */ private void loadDirectoriesInINodeSection(InputStream in) throws IOException { INodeSection s = INodeSection.parseDelimitedFrom(in); LOG.info("Loading directories in INode section."); int numDirs = 0; for (int i = 0; i < s.getNumInodes(); ++i) { INode p = INode.parseDelimitedFrom(in); if (LOG.isDebugEnabled() && i % 10000 == 0) { LOG.debug("Scanned {} inodes.", i); } if (p.hasDirectory()) { metadataMap.putDir(p); numDirs++; } } LOG.info("Found {} directories in INode section.", numDirs); }
@Override public int compare(byte[] o1, byte[] o2) { try { final FsImageProto.INodeSection.INode l = FsImageProto.INodeSection .INode.parseFrom(o1); final FsImageProto.INodeSection.INode r = FsImageProto.INodeSection .INode.parseFrom(o2); if (l.getId() < r.getId()) { return -1; } else if (l.getId() > r.getId()) { return 1; } else { return 0; } } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } }
private List<Map<String, Object>> getFileStatusList(String path) throws IOException { List<Map<String, Object>> list = new ArrayList<Map<String, Object>>(); long id = lookup(path); FsImageProto.INodeSection.INode inode = fromINodeId(id); if (inode.getType() == FsImageProto.INodeSection.INode.Type.DIRECTORY) { if (!dirmap.containsKey(id)) { // if the directory is empty, return empty list return list; } long[] children = dirmap.get(id); for (long cid : children) { list.add(getFileStatus(fromINodeId(cid), true)); } } else { list.add(getFileStatus(inode, false)); } return list; }
private List<AclEntry> getAclEntryList(String path) throws IOException { long id = lookup(path); FsImageProto.INodeSection.INode inode = fromINodeId(id); switch (inode.getType()) { case FILE: { FsImageProto.INodeSection.INodeFile f = inode.getFile(); return FSImageFormatPBINode.Loader.loadAclEntries( f.getAcl(), stringTable); } case DIRECTORY: { FsImageProto.INodeSection.INodeDirectory d = inode.getDirectory(); return FSImageFormatPBINode.Loader.loadAclEntries( d.getAcl(), stringTable); } default: { return new ArrayList<AclEntry>(); } } }
private PermissionStatus getPermissionStatus(String path) throws IOException { long id = lookup(path); FsImageProto.INodeSection.INode inode = fromINodeId(id); switch (inode.getType()) { case FILE: { FsImageProto.INodeSection.INodeFile f = inode.getFile(); return FSImageFormatPBINode.Loader.loadPermission( f.getPermission(), stringTable); } case DIRECTORY: { FsImageProto.INodeSection.INodeDirectory d = inode.getDirectory(); return FSImageFormatPBINode.Loader.loadPermission( d.getPermission(), stringTable); } case SYMLINK: { FsImageProto.INodeSection.INodeSymlink s = inode.getSymlink(); return FSImageFormatPBINode.Loader.loadPermission( s.getPermission(), stringTable); } default: { return null; } } }
private FsImageProto.INodeSection.INode fromINodeId(final long id) throws IOException { int l = 0, r = inodes.length; while (l < r) { int mid = l + (r - l) / 2; FsImageProto.INodeSection.INode n = FsImageProto.INodeSection.INode .parseFrom(inodes[mid]); long nid = n.getId(); if (id > nid) { l = mid + 1; } else if (id < nid) { r = mid; } else { return n; } } return null; }
private void list(String parent, long dirId) { INode inode = inodes.get(dirId); if (LOG.isTraceEnabled()) { LOG.trace("Listing directory id " + dirId + " parent '" + parent + "' (INode is " + inode + ")"); } listINode(parent.isEmpty() ? "/" : parent, inode); long[] children = dirmap.get(dirId); if (children == null) { return; } String newParent = parent + inode.getName().toStringUtf8() + "/"; for (long cid : children) { list(newParent, cid); } }
private void outputINodes(InputStream in) throws IOException { INodeSection s = INodeSection.parseDelimitedFrom(in); LOG.info("Found {} INodes in the INode section", s.getNumInodes()); long ignored = 0; long ignoredSnapshots = 0; for (int i = 0; i < s.getNumInodes(); ++i) { INode p = INode.parseDelimitedFrom(in); try { String parentPath = metadataMap.getParentPath(p.getId()); out.println(getEntry(parentPath, p)); } catch (IOException ioe) { ignored++; if (!(ioe instanceof IgnoreSnapshotException)) { LOG.warn("Exception caught, ignoring node:{}", p.getId(), ioe); } else { ignoredSnapshots++; if (LOG.isDebugEnabled()) { LOG.debug("Exception caught, ignoring node:{}.", p.getId(), ioe); } } } if (LOG.isDebugEnabled() && i % 100000 == 0) { LOG.debug("Outputted {} INodes.", i); } } if (ignored > 0) { LOG.warn("Ignored {} nodes, including {} in snapshots. Please turn on" + " debug log for details", ignored, ignoredSnapshots); } LOG.info("Outputted {} INodes.", s.getNumInodes()); }
/** * Return the JSON formatted FileStatus of the specified file. * @param path a path specifies a file * @return JSON formatted FileStatus * @throws IOException if failed to serialize fileStatus to JSON. */ String getFileStatus(String path) throws IOException { ObjectMapper mapper = new ObjectMapper(); FsImageProto.INodeSection.INode inode = fromINodeId(lookup(path)); return "{\"FileStatus\":\n" + mapper.writeValueAsString(getFileStatus(inode, false)) + "\n}\n"; }
private Map<String, Object> getContentSummaryMap(String path) throws IOException { long id = lookup(path); INode inode = fromINodeId(id); long spaceQuota = 0; long nsQuota = 0; long[] data = new long[4]; FsImageProto.INodeSection.INodeFile f = inode.getFile(); switch (inode.getType()) { case FILE: data[0] = 0; data[1] = 1; data[2] = getFileSize(f); nsQuota = -1; data[3] = data[2] * f.getReplication(); spaceQuota = -1; return fillSummaryMap(spaceQuota, nsQuota, data); case DIRECTORY: fillDirSummary(id, data); nsQuota = inode.getDirectory().getNsQuota(); spaceQuota = inode.getDirectory().getDsQuota(); return fillSummaryMap(spaceQuota, nsQuota, data); case SYMLINK: data[0] = 0; data[1] = 1; data[2] = 0; nsQuota = -1; data[3] = 0; spaceQuota = -1; return fillSummaryMap(spaceQuota, nsQuota, data); default: return null; } }
private void fillDirSummary(long id, long[] data) throws IOException { data[0]++; long[] children = dirmap.get(id); if (children == null) { return; } for (long cid : children) { INode node = fromINodeId(cid); switch (node.getType()) { case DIRECTORY: fillDirSummary(cid, data); break; case FILE: FsImageProto.INodeSection.INodeFile f = node.getFile(); long curLength = getFileSize(f); data[1]++; data[2] += curLength; data[3] += (curLength) * (f.getReplication()); break; case SYMLINK: data[1]++; break; default: break; } } }
private List<XAttr> getXAttrList(String path) throws IOException { long id = lookup(path); FsImageProto.INodeSection.INode inode = fromINodeId(id); switch (inode.getType()) { case FILE: return FSImageFormatPBINode.Loader.loadXAttrs( inode.getFile().getXAttrs(), stringTable); case DIRECTORY: return FSImageFormatPBINode.Loader.loadXAttrs(inode.getDirectory() .getXAttrs(), stringTable); default: return null; } }
/** * Return the INodeId of the specified path. */ private long lookup(String path) throws IOException { Preconditions.checkArgument(path.startsWith("/")); long id = INodeId.ROOT_INODE_ID; for (int offset = 0, next; offset < path.length(); offset = next) { next = path.indexOf('/', offset + 1); if (next == -1) { next = path.length(); } if (offset + 1 > next) { break; } final String component = path.substring(offset + 1, next); if (component.isEmpty()) { continue; } final long[] children = dirmap.get(id); if (children == null) { throw new FileNotFoundException(path); } boolean found = false; for (long cid : children) { FsImageProto.INodeSection.INode child = fromINodeId(cid); if (component.equals(child.getName().toStringUtf8())) { found = true; id = child.getId(); break; } } if (!found) { throw new FileNotFoundException(path); } } return id; }
private void outputINodes(InputStream in) throws IOException { INodeSection s = INodeSection.parseDelimitedFrom(in); LOG.info("Found {} INodes in the INode section", s.getNumInodes()); for (int i = 0; i < s.getNumInodes(); ++i) { INode p = INode.parseDelimitedFrom(in); String parentPath = metadataMap.getParentPath(p.getId()); out.println(getEntry(parentPath, p)); if (LOG.isDebugEnabled() && i % 100000 == 0) { LOG.debug("Outputted {} INodes.", i); } } LOG.info("Outputted {} INodes.", s.getNumInodes()); }
private void loadINodeSection(InputStream in) throws IOException { INodeSection s = INodeSection.parseDelimitedFrom(in); if (LOG.isDebugEnabled()) { LOG.debug("Found " + s.getNumInodes() + " inodes in inode section"); } for (int i = 0; i < s.getNumInodes(); ++i) { INodeSection.INode p = INodeSection.INode.parseDelimitedFrom(in); inodes.put(p.getId(), p); if (LOG.isTraceEnabled()) { LOG.trace("Loaded inode id " + p.getId() + " type " + p.getType() + " name '" + p.getName().toStringUtf8() + "'"); } } }
@Override public void putDir(INode p) { Preconditions.checkState(!dirMap.containsKey(p.getId())); Dir dir = new Dir(p.getId(), p.getName().toStringUtf8()); dirMap.put(p.getId(), dir); }
@Override public void putDir(INode dir) throws IOException { Preconditions.checkArgument(dir.hasDirectory(), "INode %s (%s) is not a directory.", dir.getId(), dir.getName()); dirMap.put(toBytes(dir.getId()), toBytes(dir.getName().toStringUtf8())); }
@Override public String getEntry(String parent, INode inode) { StringBuffer buffer = new StringBuffer(); String inodeName = inode.getName().toStringUtf8(); Path path = new Path(parent.isEmpty() ? "/" : parent, inodeName.isEmpty() ? "/" : inodeName); buffer.append(path.toString()); PermissionStatus p = null; boolean isDir = false; switch (inode.getType()) { case FILE: INodeFile file = inode.getFile(); p = getPermission(file.getPermission()); append(buffer, file.getReplication()); append(buffer, formatDate(file.getModificationTime())); append(buffer, formatDate(file.getAccessTime())); append(buffer, file.getPreferredBlockSize()); append(buffer, file.getBlocksCount()); append(buffer, FSImageLoader.getFileSize(file)); append(buffer, 0); // NS_QUOTA append(buffer, 0); // DS_QUOTA append(buffer, file.getStoragePolicyID()); break; case DIRECTORY: INodeDirectory dir = inode.getDirectory(); p = getPermission(dir.getPermission()); append(buffer, 0); // Replication append(buffer, formatDate(dir.getModificationTime())); append(buffer, formatDate(0)); // Access time. append(buffer, 0); // Block size. append(buffer, 0); // Num blocks. append(buffer, 0); // Num bytes. append(buffer, dir.getNsQuota()); append(buffer, dir.getDsQuota()); append(buffer, dir.getStoragePolicyID()); isDir = true; break; case SYMLINK: INodeSymlink s = inode.getSymlink(); p = getPermission(s.getPermission()); append(buffer, 0); // Replication append(buffer, formatDate(s.getModificationTime())); append(buffer, formatDate(s.getAccessTime())); append(buffer, 0); // Block size. append(buffer, 0); // Num blocks. append(buffer, 0); // Num bytes. append(buffer, 0); // NS_QUOTA append(buffer, 0); // DS_QUOTA append(buffer, 0); // StoragePolicy break; default: break; } assert p != null; String dirString = isDir ? "d" : "-"; append(buffer, dirString + p.getPermission().toString()); append(buffer, p.getUserName()); append(buffer, p.getGroupName()); return buffer.toString(); }
@Override public String getEntry(String parent, INode inode) { StringBuffer buffer = new StringBuffer(); String inodeName = inode.getName().toStringUtf8(); Path path = new Path(parent.isEmpty() ? "/" : parent, inodeName.isEmpty() ? "/" : inodeName); buffer.append(path.toString()); PermissionStatus p = null; switch (inode.getType()) { case FILE: INodeFile file = inode.getFile(); p = getPermission(file.getPermission()); append(buffer, file.getReplication()); append(buffer, formatDate(file.getModificationTime())); append(buffer, formatDate(file.getAccessTime())); append(buffer, file.getPreferredBlockSize()); append(buffer, file.getBlocksCount()); append(buffer, FSImageLoader.getFileSize(file)); append(buffer, 0); // NS_QUOTA append(buffer, 0); // DS_QUOTA break; case DIRECTORY: INodeDirectory dir = inode.getDirectory(); p = getPermission(dir.getPermission()); append(buffer, 0); // Replication append(buffer, formatDate(dir.getModificationTime())); append(buffer, formatDate(0)); // Access time. append(buffer, 0); // Block size. append(buffer, 0); // Num blocks. append(buffer, 0); // Num bytes. append(buffer, dir.getNsQuota()); append(buffer, dir.getDsQuota()); break; case SYMLINK: INodeSymlink s = inode.getSymlink(); p = getPermission(s.getPermission()); append(buffer, 0); // Replication append(buffer, formatDate(s.getModificationTime())); append(buffer, formatDate(s.getAccessTime())); append(buffer, 0); // Block size. append(buffer, 0); // Num blocks. append(buffer, 0); // Num bytes. append(buffer, 0); // NS_QUOTA append(buffer, 0); // DS_QUOTA break; default: break; } assert p != null; append(buffer, p.getPermission().toString()); append(buffer, p.getUserName()); append(buffer, p.getGroupName()); return buffer.toString(); }
