/** * Used by readdir and readdirplus to get dirents. It retries the listing if * the startAfter can't be found anymore. */ private DirectoryListing listPaths(DFSClient dfsClient, String dirFileIdPath, byte[] startAfter) throws IOException { DirectoryListing dlisting; try { dlisting = dfsClient.listPaths(dirFileIdPath, startAfter); } catch (RemoteException e) { IOException io = e.unwrapRemoteException(); if (!(io instanceof DirectoryListingStartAfterNotFoundException)) { throw io; } // This happens when startAfter was just deleted LOG.info("Cookie couldn't be found: " + new String(startAfter, Charset.forName("UTF-8")) + ", do listing from beginning"); dlisting = dfsClient .listPaths(dirFileIdPath, HdfsFileStatus.EMPTY_NAME); } return dlisting; }
/** * Get a partial listing of the indicated directory * * @param src the directory name * @param startAfter the name to start after * @param needLocation if blockLocations need to be returned * @return a partial listing starting after startAfter * * @throws AccessControlException if access is denied * @throws UnresolvedLinkException if symbolic link is encountered * @throws IOException if other I/O error occurred */ DirectoryListing getListing(String src, byte[] startAfter, boolean needLocation) throws IOException { checkOperation(OperationCategory.READ); DirectoryListing dl = null; readLock(); try { checkOperation(NameNode.OperationCategory.READ); dl = FSDirStatAndListingOp.getListingInt(dir, src, startAfter, needLocation); } catch (AccessControlException e) { logAuditEvent(false, "listStatus", src); throw e; } finally { readUnlock(); } logAuditEvent(true, "listStatus", src); return dl; }
@Override public GetListingResponseProto getListing(RpcController controller, GetListingRequestProto req) throws ServiceException { try { DirectoryListing result = server.getListing( req.getSrc(), req.getStartAfter().toByteArray(), req.getNeedLocation()); if (result !=null) { return GetListingResponseProto.newBuilder().setDirList( PBHelper.convert(result)).build(); } else { return VOID_GETLISTING_RESPONSE; } } catch (IOException e) { throw new ServiceException(e); } }
@Override public DirectoryListing getListing(String src, byte[] startAfter, boolean needLocation) throws AccessControlException, FileNotFoundException, UnresolvedLinkException, IOException { GetListingRequestProto req = GetListingRequestProto.newBuilder() .setSrc(src) .setStartAfter(ByteString.copyFrom(startAfter)) .setNeedLocation(needLocation).build(); try { GetListingResponseProto result = rpcProxy.getListing(null, req); if (result.hasDirList()) { return PBHelper.convert(result.getDirList()); } return null; } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } }
void verifyFile(final Path file, final Byte expectedPolicyId) throws Exception { final Path parent = file.getParent(); DirectoryListing children = dfs.getClient().listPaths( parent.toString(), HdfsFileStatus.EMPTY_NAME, true); for (HdfsFileStatus child : children.getPartialListing()) { if (child.getLocalName().equals(file.getName())) { verifyFile(parent, child, expectedPolicyId); return; } } Assert.fail("File " + file + " not found."); }
static void recoverAllLeases(DFSClient dfs, Path path) throws IOException { String pathStr = path.toString(); HdfsFileStatus status = dfs.getFileInfo(pathStr); if (!status.isDir()) { dfs.recoverLease(pathStr); return; } byte prev[] = HdfsFileStatus.EMPTY_NAME; DirectoryListing dirList; do { dirList = dfs.listPaths(pathStr, prev); HdfsFileStatus files[] = dirList.getPartialListing(); for (HdfsFileStatus f : files) { recoverAllLeases(dfs, f.getFullPath(path)); } prev = dirList.getLastName(); } while (dirList.hasMore()); }
@Override public DirectoryListing getListing(String src, byte[] startAfter, boolean needLocation) throws IOException { GetListingRequestProto req = GetListingRequestProto.newBuilder() .setSrc(src) .setStartAfter(ByteString.copyFrom(startAfter)) .setNeedLocation(needLocation).build(); try { GetListingResponseProto result = rpcProxy.getListing(null, req); if (result.hasDirList()) { return PBHelperClient.convert(result.getDirList()); } return null; } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } }
/** * Get a partial listing of the indicated directory * * @param src the directory name * @param startAfter the name to start after * @param needLocation if blockLocations need to be returned * @return a partial listing starting after startAfter * * @throws AccessControlException if access is denied * @throws UnresolvedLinkException if symbolic link is encountered * @throws IOException if other I/O error occurred */ DirectoryListing getListing(String src, byte[] startAfter, boolean needLocation) throws IOException { checkOperation(OperationCategory.READ); DirectoryListing dl = null; readLock(); try { checkOperation(NameNode.OperationCategory.READ); dl = getListingInt(dir, src, startAfter, needLocation); } catch (AccessControlException e) { logAuditEvent(false, "listStatus", src); throw e; } finally { readUnlock(); } logAuditEvent(true, "listStatus", src); return dl; }
@Override public GetListingResponseProto getListing(RpcController controller, GetListingRequestProto req) throws ServiceException { try { DirectoryListing result = server.getListing( req.getSrc(), req.getStartAfter().toByteArray(), req.getNeedLocation()); if (result !=null) { return GetListingResponseProto.newBuilder().setDirList( PBHelperClient.convert(result)).build(); } else { return VOID_GETLISTING_RESPONSE; } } catch (IOException e) { throw new ServiceException(e); } }
/** * Used by readdir and readdirplus to get dirents. It retries the listing if * the startAfter can't be found anymore. */ private DirectoryListing listPaths(DFSClient dfsClient, String dirFileIdPath, byte[] startAfter) throws IOException { DirectoryListing dlisting; try { dlisting = dfsClient.listPaths(dirFileIdPath, startAfter); } catch (RemoteException e) { IOException io = e.unwrapRemoteException(); if (!(io instanceof DirectoryListingStartAfterNotFoundException)) { throw io; } // This happens when startAfter was just deleted LOG.info("Cookie couldn't be found: " + new String(startAfter) + ", do listing from beginning"); dlisting = dfsClient .listPaths(dirFileIdPath, HdfsFileStatus.EMPTY_NAME); } return dlisting; }
/** * Get a partial listing of the indicated directory * * @param src the directory name * @param startAfter the name to start after * @param needLocation if blockLocations need to be returned * @return a partial listing starting after startAfter */ public DirectoryListing getPartialListing(String src, byte[] startAfter, boolean needLocation) throws IOException { DirectoryListing stats; src = dir.normalizePath(src); byte[][] names = INode.getPathComponents(src); INode[] inodes = new INode[names.length]; readLock(); try { dir.rootDir.getExistingPathINodes(names, inodes); getListingCheck(src, inodes); stats = dir.getPartialListing(src, inodes[inodes.length-1], startAfter, needLocation); } finally { readUnlock(); } if (auditLog.isInfoEnabled()) { logAuditEvent(getCurrentUGI(), Server.getRemoteIp(), "listStatus", src, null, null); } return stats; }
/** * Get a listing of all the snapshots of a snapshottable directory */ private DirectoryListing getSnapshotsListing(String src, byte[] startAfter) throws UnresolvedLinkException, IOException { Preconditions.checkState(hasReadLock()); Preconditions.checkArgument( src.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR), "%s does not end with %s", src, HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR); final String dirPath = normalizePath(src.substring(0, src.length() - HdfsConstants.DOT_SNAPSHOT_DIR.length())); final INode node = this.getINode(dirPath); final INodeDirectorySnapshottable dirNode = INodeDirectorySnapshottable .valueOf(node, dirPath); final ReadOnlyList<Snapshot> snapshots = dirNode.getSnapshotList(); int skipSize = ReadOnlyList.Util.binarySearch(snapshots, startAfter); skipSize = skipSize < 0 ? -skipSize - 1 : skipSize + 1; int numOfListing = Math.min(snapshots.size() - skipSize, this.lsLimit); final HdfsFileStatus listing[] = new HdfsFileStatus[numOfListing]; for (int i = 0; i < numOfListing; i++) { Root sRoot = snapshots.get(i + skipSize).getRoot(); listing[i] = createFileStatus(sRoot.getLocalNameBytes(), sRoot, null); } return new DirectoryListing( listing, snapshots.size() - skipSize - numOfListing); }
private DirectoryListing getListingInt(String src, byte[] startAfter, boolean needLocation) throws AccessControlException, UnresolvedLinkException, IOException { DirectoryListing dl; FSPermissionChecker pc = getPermissionChecker(); checkOperation(OperationCategory.READ); byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src); readLock(); try { checkOperation(OperationCategory.READ); src = FSDirectory.resolvePath(src, pathComponents, dir); if (isPermissionEnabled) { if (dir.isDir(src)) { checkPathAccess(pc, src, FsAction.READ_EXECUTE); } else { checkTraverse(pc, src); } } logAuditEvent(true, "listStatus", src); dl = dir.getListing(src, startAfter, needLocation); } finally { readUnlock(); } return dl; }
/** * Retrieve list of files under @hdfsDir for @hdfsClient. */ public static List<HdfsFileStatus> getHdfsFileList(DFSClient hdfsClient, String hdfsDir) throws IOException { List<HdfsFileStatus> fileList = Lists.newArrayList(); // Build a list of files. DirectoryListing listing = null; String continuation = ""; while (true) { listing = hdfsClient.listPaths(hdfsDir, continuation.getBytes()); for (HdfsFileStatus fileStatus : listing.getPartialListing()) { fileList.add(fileStatus); } // Go through the listing and paginate. if (!listing.hasMore()) { break; } else { continuation = new String(listing.getLastName()); } } return fileList; }
/** * Used by readdir and readdirplus to get dirents. It retries the listing if * the startAfter can't be found anymore. */ private DirectoryListing listPaths(DFSClient dfsClient, String dirFileIdPath, byte[] startAfter) throws IOException { DirectoryListing dlisting = null; try { dlisting = dfsClient.listPaths(dirFileIdPath, startAfter); } catch (RemoteException e) { IOException io = e.unwrapRemoteException(); if (!(io instanceof DirectoryListingStartAfterNotFoundException)) { throw io; } // This happens when startAfter was just deleted LOG.info("Cookie cound't be found: " + new String(startAfter) + ", do listing from beginning"); dlisting = dfsClient.listPaths(dirFileIdPath, HdfsFileStatus.EMPTY_NAME); } return dlisting; }
/** * Get a partial listing of the indicated directory * <p/> * Recommend to use HdfsFileStatus.EMPTY_NAME as startAfter * if the application wants to fetch a listing starting from * the first entry in the directory * * @see ClientProtocol#getListing(String, byte[], boolean) */ public DirectoryListing listPaths(final String src, final byte[] startAfter, final boolean needLocation) throws IOException { checkOpen(); try { ClientActionHandler handler = new ClientActionHandler() { @Override public Object doAction(ClientProtocol namenode) throws RemoteException, IOException { return namenode.getListing(src, startAfter, needLocation); } }; return (DirectoryListing) doClientActionWithRetry(handler, "listPaths"); } catch (RemoteException re) { throw re.unwrapRemoteException(AccessControlException.class, FileNotFoundException.class, UnresolvedPathException.class); } }
private DirectoryListing getListingInt(String src, byte[] startAfter, boolean needLocation) throws IOException { DirectoryListing dl; FSPermissionChecker pc = getPermissionChecker(); if (isPermissionEnabled) { if (dir.isDir(src)) { checkPathAccess(pc, src, FsAction.READ_EXECUTE); } else { checkTraverse(pc, src); } } logAuditEvent(true, "listStatus", src); dl = dir.getListing(src, startAfter, needLocation); return dl; }
@Override public GetListingResponseProto getListing(RpcController controller, GetListingRequestProto req) throws ServiceException { try { DirectoryListing result = server .getListing(req.getSrc(), req.getStartAfter().toByteArray(), req.getNeedLocation()); if (result != null) { return GetListingResponseProto.newBuilder() .setDirList(PBHelper.convert(result)).build(); } else { return VOID_GETLISTING_RESPONSE; } } catch (IOException e) { throw new ServiceException(e); } }
@Override public DirectoryListing getListing(String src, byte[] startAfter, boolean needLocation) throws AccessControlException, FileNotFoundException, UnresolvedLinkException, IOException { GetListingRequestProto req = GetListingRequestProto.newBuilder().setSrc(src) .setStartAfter(ByteString.copyFrom(startAfter)) .setNeedLocation(needLocation).build(); try { GetListingResponseProto result = rpcProxy.getListing(null, req); if (result.hasDirList()) { return PBHelper.convert(result.getDirList()); } return null; } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } }
