/** * Throws an exception if the CachePool does not have enough capacity to * cache the given path at the replication factor. * * @param pool CachePool where the path is being cached * @param path Path that is being cached * @param replication Replication factor of the path * @throws InvalidRequestException if the pool does not have enough capacity */ private void checkLimit(CachePool pool, String path, short replication) throws InvalidRequestException { CacheDirectiveStats stats = computeNeeded(path, replication); if (pool.getLimit() == CachePoolInfo.LIMIT_UNLIMITED) { return; } if (pool.getBytesNeeded() + (stats.getBytesNeeded() * replication) > pool .getLimit()) { throw new InvalidRequestException("Caching path " + path + " of size " + stats.getBytesNeeded() / replication + " bytes at replication " + replication + " would exceed pool " + pool.getPoolName() + "'s remaining capacity of " + (pool.getLimit() - pool.getBytesNeeded()) + " bytes."); } }
/** * Computes the needed number of bytes and files for a path. * @return CacheDirectiveStats describing the needed stats for this path */ private CacheDirectiveStats computeNeeded(String path, short replication) { FSDirectory fsDir = namesystem.getFSDirectory(); INode node; long requestedBytes = 0; long requestedFiles = 0; CacheDirectiveStats.Builder builder = new CacheDirectiveStats.Builder(); try { node = fsDir.getINode(path); } catch (UnresolvedLinkException e) { // We don't cache through symlinks return builder.build(); } if (node == null) { return builder.build(); } if (node.isFile()) { requestedFiles = 1; INodeFile file = node.asFile(); requestedBytes = file.computeFileSize(); } else if (node.isDirectory()) { INodeDirectory dir = node.asDirectory(); ReadOnlyList<INode> children = dir .getChildrenList(Snapshot.CURRENT_STATE_ID); requestedFiles = children.size(); for (INode child : children) { if (child.isFile()) { requestedBytes += child.asFile().computeFileSize(); } } } return new CacheDirectiveStats.Builder() .setBytesNeeded(requestedBytes) .setFilesCached(requestedFiles) .build(); }
public static CacheDirectiveStatsProto convert(CacheDirectiveStats stats) { CacheDirectiveStatsProto.Builder builder = CacheDirectiveStatsProto.newBuilder(); builder.setBytesNeeded(stats.getBytesNeeded()); builder.setBytesCached(stats.getBytesCached()); builder.setFilesNeeded(stats.getFilesNeeded()); builder.setFilesCached(stats.getFilesCached()); builder.setHasExpired(stats.hasExpired()); return builder.build(); }
public static CacheDirectiveStats convert(CacheDirectiveStatsProto proto) { CacheDirectiveStats.Builder builder = new CacheDirectiveStats.Builder(); builder.setBytesNeeded(proto.getBytesNeeded()); builder.setBytesCached(proto.getBytesCached()); builder.setFilesNeeded(proto.getFilesNeeded()); builder.setFilesCached(proto.getFilesCached()); builder.setHasExpired(proto.getHasExpired()); return builder.build(); }
public static CacheDirectiveEntry convert(CacheDirectiveEntryProto proto) { CacheDirectiveInfo info = PBHelper.convert(proto.getInfo()); CacheDirectiveStats stats = PBHelper.convert(proto.getStats()); return new CacheDirectiveEntry(info, stats); }
@Override public int run(Configuration conf, List<String> args) throws IOException { CacheDirectiveInfo.Builder builder = new CacheDirectiveInfo.Builder(); String pathFilter = StringUtils.popOptionWithArgument("-path", args); if (pathFilter != null) { builder.setPath(new Path(pathFilter)); } String poolFilter = StringUtils.popOptionWithArgument("-pool", args); if (poolFilter != null) { builder.setPool(poolFilter); } boolean printStats = StringUtils.popOption("-stats", args); String idFilter = StringUtils.popOptionWithArgument("-id", args); if (idFilter != null) { builder.setId(Long.parseLong(idFilter)); } if (!args.isEmpty()) { System.err.println("Can't understand argument: " + args.get(0)); return 1; } TableListing.Builder tableBuilder = new TableListing.Builder(). addField("ID", Justification.RIGHT). addField("POOL", Justification.LEFT). addField("REPL", Justification.RIGHT). addField("EXPIRY", Justification.LEFT). addField("PATH", Justification.LEFT); if (printStats) { tableBuilder.addField("BYTES_NEEDED", Justification.RIGHT). addField("BYTES_CACHED", Justification.RIGHT). addField("FILES_NEEDED", Justification.RIGHT). addField("FILES_CACHED", Justification.RIGHT); } TableListing tableListing = tableBuilder.build(); try { DistributedFileSystem dfs = AdminHelper.getDFS(conf); RemoteIterator<CacheDirectiveEntry> iter = dfs.listCacheDirectives(builder.build()); int numEntries = 0; while (iter.hasNext()) { CacheDirectiveEntry entry = iter.next(); CacheDirectiveInfo directive = entry.getInfo(); CacheDirectiveStats stats = entry.getStats(); List<String> row = new LinkedList<String>(); row.add("" + directive.getId()); row.add(directive.getPool()); row.add("" + directive.getReplication()); String expiry; // This is effectively never, round for nice printing if (directive.getExpiration().getMillis() > Expiration.MAX_RELATIVE_EXPIRY_MS / 2) { expiry = "never"; } else { expiry = directive.getExpiration().toString(); } row.add(expiry); row.add(directive.getPath().toUri().getPath()); if (printStats) { row.add("" + stats.getBytesNeeded()); row.add("" + stats.getBytesCached()); row.add("" + stats.getFilesNeeded()); row.add("" + stats.getFilesCached()); } tableListing.addRow(row.toArray(new String[row.size()])); numEntries++; } System.out.print(String.format("Found %d entr%s%n", numEntries, numEntries == 1 ? "y" : "ies")); if (numEntries > 0) { System.out.print(tableListing); } } catch (IOException e) { System.err.println(AdminHelper.prettifyException(e)); return 2; } return 0; }
private static void waitForCacheDirectiveStats(final DistributedFileSystem dfs, final long targetBytesNeeded, final long targetBytesCached, final long targetFilesNeeded, final long targetFilesCached, final CacheDirectiveInfo filter, final String infoString) throws Exception { LOG.info("Polling listCacheDirectives " + ((filter == null) ? "ALL" : filter.toString()) + " for " + targetBytesNeeded + " targetBytesNeeded, " + targetBytesCached + " targetBytesCached, " + targetFilesNeeded + " targetFilesNeeded, " + targetFilesCached + " targetFilesCached"); GenericTestUtils.waitFor(new Supplier<Boolean>() { @Override public Boolean get() { RemoteIterator<CacheDirectiveEntry> iter = null; CacheDirectiveEntry entry = null; try { iter = dfs.listCacheDirectives(filter); entry = iter.next(); } catch (IOException e) { fail("got IOException while calling " + "listCacheDirectives: " + e.getMessage()); } Assert.assertNotNull(entry); CacheDirectiveStats stats = entry.getStats(); if ((targetBytesNeeded == stats.getBytesNeeded()) && (targetBytesCached == stats.getBytesCached()) && (targetFilesNeeded == stats.getFilesNeeded()) && (targetFilesCached == stats.getFilesCached())) { return true; } else { LOG.info(infoString + ": " + "filesNeeded: " + stats.getFilesNeeded() + "/" + targetFilesNeeded + ", filesCached: " + stats.getFilesCached() + "/" + targetFilesCached + ", bytesNeeded: " + stats.getBytesNeeded() + "/" + targetBytesNeeded + ", bytesCached: " + stats.getBytesCached() + "/" + targetBytesCached); return false; } } }, 500, 60000); }
public static CacheDirectiveEntry convert(CacheDirectiveEntryProto proto) { CacheDirectiveInfo info = convert(proto.getInfo()); CacheDirectiveStats stats = convert(proto.getStats()); return new CacheDirectiveEntry(info, stats); }
@Override public int run(Configuration conf, List<String> args) throws IOException { CacheDirectiveInfo.Builder builder = new CacheDirectiveInfo.Builder(); String pathFilter = StringUtils.popOptionWithArgument("-path", args); if (pathFilter != null) { builder.setPath(new Path(pathFilter)); } String poolFilter = StringUtils.popOptionWithArgument("-pool", args); if (poolFilter != null) { builder.setPool(poolFilter); } boolean printStats = StringUtils.popOption("-stats", args); String idFilter = StringUtils.popOptionWithArgument("-id", args); if (idFilter != null) { builder.setId(Long.parseLong(idFilter)); } if (!args.isEmpty()) { System.err.println("Can't understand argument: " + args.get(0)); return 1; } TableListing.Builder tableBuilder = new TableListing.Builder(). addField("ID", Justification.RIGHT). addField("POOL", Justification.LEFT). addField("REPL", Justification.RIGHT). addField("EXPIRY", Justification.LEFT). addField("PATH", Justification.LEFT); if (printStats) { tableBuilder.addField("BYTES_NEEDED", Justification.RIGHT). addField("BYTES_CACHED", Justification.RIGHT). addField("FILES_NEEDED", Justification.RIGHT). addField("FILES_CACHED", Justification.RIGHT); } TableListing tableListing = tableBuilder.build(); try { DistributedFileSystem dfs = getDFS(conf); RemoteIterator<CacheDirectiveEntry> iter = dfs.listCacheDirectives(builder.build()); int numEntries = 0; while (iter.hasNext()) { CacheDirectiveEntry entry = iter.next(); CacheDirectiveInfo directive = entry.getInfo(); CacheDirectiveStats stats = entry.getStats(); List<String> row = new LinkedList<String>(); row.add("" + directive.getId()); row.add(directive.getPool()); row.add("" + directive.getReplication()); String expiry; // This is effectively never, round for nice printing if (directive.getExpiration().getMillis() > Expiration.MAX_RELATIVE_EXPIRY_MS / 2) { expiry = "never"; } else { expiry = directive.getExpiration().toString(); } row.add(expiry); row.add(directive.getPath().toUri().getPath()); if (printStats) { row.add("" + stats.getBytesNeeded()); row.add("" + stats.getBytesCached()); row.add("" + stats.getFilesNeeded()); row.add("" + stats.getFilesCached()); } tableListing.addRow(row.toArray(new String[0])); numEntries++; } System.out.print(String.format("Found %d entr%s%n", numEntries, numEntries == 1 ? "y" : "ies")); if (numEntries > 0) { System.out.print(tableListing); } } catch (IOException e) { System.err.println(prettifyException(e)); return 2; } return 0; }