@Override @SuppressWarnings("unchecked") public boolean hasNext() throws IOException { while (curStat == null && hasNextNoFilter()) { T next; HdfsFileStatus fileStat = thisListing.getPartialListing()[i++]; if (needLocation) { next = (T)((HdfsLocatedFileStatus)fileStat) .makeQualifiedLocated(getUri(), p); } else { next = (T)fileStat.makeQualified(getUri(), p); } // apply filter if not null if (filter == null || filter.accept(next.getPath())) { curStat = next; } } return curStat != null; }
public static HdfsFileStatus convert(HdfsFileStatusProto fs) { if (fs == null) return null; return new HdfsLocatedFileStatus( fs.getLength(), fs.getFileType().equals(FileType.IS_DIR), fs.getBlockReplication(), fs.getBlocksize(), fs.getModificationTime(), fs.getAccessTime(), PBHelper.convert(fs.getPermission()), fs.getOwner(), fs.getGroup(), fs.getFileType().equals(FileType.IS_SYMLINK) ? fs.getSymlink().toByteArray() : null, fs.getPath().toByteArray(), fs.hasFileId()? fs.getFileId(): INodeId.GRANDFATHER_INODE_ID, fs.hasLocations() ? PBHelper.convert(fs.getLocations()) : null, fs.hasChildrenNum() ? fs.getChildrenNum() : -1, fs.hasFileEncryptionInfo() ? convert(fs.getFileEncryptionInfo()) : null, fs.hasStoragePolicy() ? (byte) fs.getStoragePolicy() : BlockStoragePolicySuite.ID_UNSPECIFIED); }
private void verifyFile(final Path parent, final HdfsFileStatus status, final Byte expectedPolicyId) throws Exception { HdfsLocatedFileStatus fileStatus = (HdfsLocatedFileStatus) status; byte policyId = fileStatus.getStoragePolicy(); BlockStoragePolicy policy = policies.getPolicy(policyId); if (expectedPolicyId != null) { Assert.assertEquals((byte)expectedPolicyId, policy.getId()); } final List<StorageType> types = policy.chooseStorageTypes( status.getReplication()); for(LocatedBlock lb : fileStatus.getBlockLocations().getLocatedBlocks()) { final Mover.StorageTypeDiff diff = new Mover.StorageTypeDiff(types, lb.getStorageTypes()); Assert.assertTrue(fileStatus.getFullName(parent.toString()) + " with policy " + policy + " has non-empty overlap: " + diff + ", the corresponding block is " + lb.getBlock().getLocalBlock(), diff.removeOverlap(true)); } }
public static HdfsFileStatus convert(HdfsFileStatusProto fs) { if (fs == null) return null; return new HdfsLocatedFileStatus( fs.getLength(), fs.getFileType().equals(FileType.IS_DIR), fs.getBlockReplication(), fs.getBlocksize(), fs.getModificationTime(), fs.getAccessTime(), convert(fs.getPermission()), fs.getOwner(), fs.getGroup(), fs.getFileType().equals(FileType.IS_SYMLINK) ? fs.getSymlink().toByteArray() : null, fs.getPath().toByteArray(), fs.hasFileId()? fs.getFileId(): HdfsConstants.GRANDFATHER_INODE_ID, fs.hasLocations() ? convert(fs.getLocations()) : null, fs.hasChildrenNum() ? fs.getChildrenNum() : -1, fs.hasFileEncryptionInfo() ? convert(fs.getFileEncryptionInfo()) : null, fs.hasStoragePolicy() ? (byte) fs.getStoragePolicy() : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, fs.hasEcPolicy() ? convertErasureCodingPolicy(fs.getEcPolicy()) : null); }
@Override public RemoteIterator<LocatedFileStatus> listLocatedStatus( final Path p) throws FileNotFoundException, IOException { return new DirListingIterator<LocatedFileStatus>(p, true) { @Override public LocatedFileStatus next() throws IOException { return ((HdfsLocatedFileStatus)getNext()).makeQualifiedLocated( getUri(), p); } }; }
public static HdfsFileStatusProto convert(HdfsFileStatus fs) { if (fs == null) return null; FileType fType = FileType.IS_FILE; if (fs.isDir()) { fType = FileType.IS_DIR; } else if (fs.isSymlink()) { fType = FileType.IS_SYMLINK; } HdfsFileStatusProto.Builder builder = HdfsFileStatusProto.newBuilder(). setLength(fs.getLen()). setFileType(fType). setBlockReplication(fs.getReplication()). setBlocksize(fs.getBlockSize()). setModificationTime(fs.getModificationTime()). setAccessTime(fs.getAccessTime()). setPermission(PBHelper.convert(fs.getPermission())). setOwner(fs.getOwner()). setGroup(fs.getGroup()). setFileId(fs.getFileId()). setChildrenNum(fs.getChildrenNum()). setPath(ByteString.copyFrom(fs.getLocalNameInBytes())). setStoragePolicy(fs.getStoragePolicy()); if (fs.isSymlink()) { builder.setSymlink(ByteString.copyFrom(fs.getSymlinkInBytes())); } if (fs.getFileEncryptionInfo() != null) { builder.setFileEncryptionInfo(convert(fs.getFileEncryptionInfo())); } if (fs instanceof HdfsLocatedFileStatus) { final HdfsLocatedFileStatus lfs = (HdfsLocatedFileStatus) fs; LocatedBlocks locations = lfs.getBlockLocations(); if (locations != null) { builder.setLocations(PBHelper.convert(locations)); } } return builder.build(); }
public static DirectoryListing convert(DirectoryListingProto dl) { if (dl == null) return null; List<HdfsFileStatusProto> partList = dl.getPartialListingList(); return new DirectoryListing( partList.isEmpty() ? new HdfsLocatedFileStatus[0] : PBHelper.convert( partList.toArray(new HdfsFileStatusProto[partList.size()])), dl.getRemainingEntries()); }
public static DirectoryListing convert(DirectoryListingProto dl) { if (dl == null) return null; List<HdfsFileStatusProto> partList = dl.getPartialListingList(); return new DirectoryListing(partList.isEmpty() ? new HdfsLocatedFileStatus[0] : convert(partList.toArray(new HdfsFileStatusProto[partList.size()])), dl.getRemainingEntries()); }
/** * Create FileStatus with location info by file INode */ private HdfsLocatedFileStatus createLocatedFileStatus(byte[] path, INode node, Snapshot snapshot) throws IOException { assert hasReadLock(); long size = 0; // length is zero for directories short replication = 0; long blocksize = 0; LocatedBlocks loc = null; if (node.isFile()) { final INodeFile fileNode = node.asFile(); size = fileNode.computeFileSize(snapshot); replication = fileNode.getFileReplication(snapshot); blocksize = fileNode.getPreferredBlockSize(); final boolean inSnapshot = snapshot != null; final boolean isUc = inSnapshot ? false : fileNode.isUnderConstruction(); final long fileSize = !inSnapshot && isUc ? fileNode.computeFileSizeNotIncludingLastUcBlock() : size; loc = getFSNamesystem().getBlockManager().createLocatedBlocks( fileNode.getBlocks(), fileSize, isUc, 0L, size, false, inSnapshot); if (loc == null) { loc = new LocatedBlocks(); } } int childrenNum = node.isDirectory() ? node.asDirectory().getChildrenNum(snapshot) : 0; return new HdfsLocatedFileStatus(size, node.isDirectory(), replication, blocksize, node.getModificationTime(snapshot), node.getAccessTime(snapshot), node.getFsPermission(snapshot), node.getUserName(snapshot), node.getGroupName(snapshot), node.isSymlink() ? node.asSymlink().getSymlink() : null, path, node.getId(), loc, childrenNum); }
public static HdfsFileStatus convert(HdfsFileStatusProto fs) { if (fs == null) return null; return new HdfsLocatedFileStatus( fs.getLength(), fs.getFileType().equals(FileType.IS_DIR), fs.getBlockReplication(), fs.getBlocksize(), fs.getModificationTime(), fs.getAccessTime(), PBHelper.convert(fs.getPermission()), fs.getOwner(), fs.getGroup(), fs.getFileType().equals(FileType.IS_SYMLINK) ? fs.getSymlink().toByteArray() : null, fs.getPath().toByteArray(), fs.hasFileId()? fs.getFileId(): INodeId.GRANDFATHER_INODE_ID, fs.hasLocations() ? PBHelper.convert(fs.getLocations()) : null, fs.hasChildrenNum() ? fs.getChildrenNum() : 0); }
public static HdfsFileStatusProto convert(HdfsFileStatus fs) { if (fs == null) return null; FileType fType = FileType.IS_FILE; if (fs.isDir()) { fType = FileType.IS_DIR; } else if (fs.isSymlink()) { fType = FileType.IS_SYMLINK; } HdfsFileStatusProto.Builder builder = HdfsFileStatusProto.newBuilder(). setLength(fs.getLen()). setFileType(fType). setBlockReplication(fs.getReplication()). setBlocksize(fs.getBlockSize()). setModificationTime(fs.getModificationTime()). setAccessTime(fs.getAccessTime()). setPermission(PBHelper.convert(fs.getPermission())). setOwner(fs.getOwner()). setGroup(fs.getGroup()). setFileId(fs.getFileId()). setChildrenNum(fs.getChildrenNum()). setPath(ByteString.copyFrom(fs.getLocalNameInBytes())); if (fs.isSymlink()) { builder.setSymlink(ByteString.copyFrom(fs.getSymlinkInBytes())); } if (fs instanceof HdfsLocatedFileStatus) { LocatedBlocks locations = ((HdfsLocatedFileStatus)fs).getBlockLocations(); if (locations != null) { builder.setLocations(PBHelper.convert(locations)); } } return builder.build(); }
private LocatedFileStatus makeQualifiedLocated(HdfsLocatedFileStatus f, Path parent) throws IOException { return new LocatedFileStatus(f.getLen(), f.isDir(), f.getReplication(), f.getBlockSize(), f.getModificationTime(), f.getAccessTime(), f.getPermission(), f.getOwner(), f.getGroup(), f.isSymlink() ? f.getSymlink() : null, (f.getFullPath(parent)).makeQualified(getUri(), null), // fully-qualify path DFSUtil.locatedBlocks2Locations(f.getBlockLocations())); }
@Override public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path p) throws FileNotFoundException, IOException { return new DirListingIterator<LocatedFileStatus>(p, true) { @Override public LocatedFileStatus next() throws IOException { return makeQualifiedLocated((HdfsLocatedFileStatus) getNext(), p); } }; }
private LocatedFileStatus makeQualifiedLocated(HdfsLocatedFileStatus f, Path parent) { return new LocatedFileStatus(f.getLen(), f.isDir(), f.getReplication(), f.getBlockSize(), f.getModificationTime(), f.getAccessTime(), f.getPermission(), f.getOwner(), f.getGroup(), null, (f.getFullPath(parent)).makeQualified(getUri(), getWorkingDirectory()), // fully-qualify path DFSUtil.locatedBlocks2Locations(f.getBlockLocations())); }
/** * Create FileStatus with location info by file INode */ private HdfsLocatedFileStatus createLocatedFileStatus(byte[] path, INode node) throws IOException, StorageException { long size = 0; // length is zero for directories short replication = 0; long blocksize = 0; LocatedBlocks loc = null; boolean isFileStoredInDB = false; if (node instanceof INodeFile) { INodeFile fileNode = (INodeFile) node; isFileStoredInDB = fileNode.isFileStoredInDB(); if(isFileStoredInDB){ size = fileNode.getSize(); }else{ size = fileNode.computeFileSize(true); } replication = fileNode.getBlockReplication(); blocksize = fileNode.getPreferredBlockSize(); if(isFileStoredInDB){ loc = getFSNamesystem().getBlockManager().createPhantomLocatedBlocks(fileNode,null,fileNode.isUnderConstruction(),false); }else { loc = getFSNamesystem().getBlockManager() .createLocatedBlocks(fileNode.getBlocks(), fileNode.computeFileSize(false), fileNode.isUnderConstruction(), 0L, size, false); } if (loc == null) { loc = new LocatedBlocks(); } } return new HdfsLocatedFileStatus(node.getId(), size, node.isDirectory(), replication, blocksize, node.getModificationTime(), node.getAccessTime(), node.getFsPermission(), node.getUserName(), node.getGroupName(), node.isSymlink() ? ((INodeSymlink) node).getSymlink() : null, path, loc, isFileStoredInDB); }
public static HdfsFileStatus convert(HdfsFileStatusProto fs) throws IOException { if (fs == null) { return null; } return new HdfsLocatedFileStatus(fs.getFileId(), fs.getLength(), fs.getFileType().equals(FileType.IS_DIR), fs.getBlockReplication(), fs.getBlocksize(), fs.getModificationTime(), fs.getAccessTime(), PBHelper.convert(fs.getPermission()), fs.getOwner(), fs.getGroup(), fs.getFileType().equals(FileType.IS_SYMLINK) ? fs.getSymlink().toByteArray() : null, fs.getPath().toByteArray(), fs.hasLocations() ? PBHelper.convert(fs.getLocations()) : null, fs.hasIsFileStoredInDB() ? fs.getIsFileStoredInDB() : false); }
public static HdfsFileStatusProto convert(HdfsFileStatus fs) { if (fs == null) { return null; } FileType fType = FileType.IS_FILE; if (fs.isDir()) { fType = FileType.IS_DIR; } else if (fs.isSymlink()) { fType = FileType.IS_SYMLINK; } HdfsFileStatusProto.Builder builder = HdfsFileStatusProto.newBuilder(). setFileId(fs.getFileId()). setLength(fs.getLen()). setFileType(fType). setBlockReplication(fs.getReplication()). setBlocksize(fs.getBlockSize()). setModificationTime(fs.getModificationTime()). setAccessTime(fs.getAccessTime()). setPermission(PBHelper.convert(fs.getPermission())). setOwner(fs.getOwner()). setGroup(fs.getGroup()). setPath(ByteString.copyFrom(fs.getLocalNameInBytes())). setIsFileStoredInDB(fs.isFileStoredInDB()). setIsFileStoredInDB(fs.isFileStoredInDB()); if (fs.isSymlink()) { builder.setSymlink(ByteString.copyFrom(fs.getSymlinkInBytes())); } if (fs instanceof HdfsLocatedFileStatus) { LocatedBlocks locations = ((HdfsLocatedFileStatus) fs).getBlockLocations(); if (locations != null) { builder.setLocations(PBHelper.convert(locations)); } } return builder.build(); }
public static DirectoryListing convert(DirectoryListingProto dl) throws IOException { if (dl == null) { return null; } List<HdfsFileStatusProto> partList = dl.getPartialListingList(); return new DirectoryListing( partList.isEmpty() ? new HdfsLocatedFileStatus[0] : PBHelper.convert( partList.toArray(new HdfsFileStatusProto[partList.size()])), dl.getRemainingEntries()); }