/** * Return true if we may find this type of block in block cache. * <p/> * TODO: today {@code family.isBlockCacheEnabled()} only means {@code cacheDataOnRead}, so here we * consider lots of other configurations such as {@code cacheDataOnWrite}. We should fix this in * the future, {@code cacheDataOnWrite} should honor the CF level {@code isBlockCacheEnabled} * configuration. */ public boolean shouldReadBlockFromCache(BlockType blockType) { if (!isBlockCacheEnabled()) { return false; } if (cacheDataOnRead) { return true; } if (prefetchOnOpen) { return true; } if (cacheDataOnWrite) { return true; } if (blockType == null) { return true; } if (blockType.getCategory() == BlockCategory.BLOOM || blockType.getCategory() == BlockCategory.INDEX) { return true; } return false; }
/** * Return true if we may find this type of block in block cache. * <p> * TODO: today {@code family.isBlockCacheEnabled()} only means {@code cacheDataOnRead}, so here we * consider lots of other configurations such as {@code cacheDataOnWrite}. We should fix this in * the future, {@code cacheDataOnWrite} should honor the CF level {@code isBlockCacheEnabled} * configuration. */ public boolean shouldReadBlockFromCache(BlockType blockType) { if (!isBlockCacheEnabled()) { return false; } if (cacheDataOnRead) { return true; } if (prefetchOnOpen) { return true; } if (cacheDataOnWrite) { return true; } if (blockType == null) { return true; } if (blockType.getCategory() == BlockCategory.BLOOM || blockType.getCategory() == BlockCategory.INDEX) { return true; } return false; }
/** * Should we cache a block of a particular category? We always cache * important blocks such as index blocks, as long as the block cache is * available. */ public boolean shouldCacheBlockOnRead(BlockCategory category) { return isBlockCacheEnabled() && (cacheDataOnRead || category == BlockCategory.INDEX || category == BlockCategory.BLOOM || (prefetchOnOpen && (category != BlockCategory.META && category != BlockCategory.UNKNOWN))); }
/** * @return true if this {@link BlockCategory} should be compressed in blockcache, false otherwise */ public boolean shouldCacheCompressed(BlockCategory category) { if (!isBlockCacheEnabled()) return false; switch (category) { case DATA: return this.cacheDataCompressed; default: return false; } }
@Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory, final boolean cacheDataInL1) { boolean isMetaBlock = buf.getBlockType().getCategory() != BlockCategory.DATA; if (isMetaBlock || cacheDataInL1) { lruCache.cacheBlock(cacheKey, buf, inMemory, cacheDataInL1); } else { l2Cache.cacheBlock(cacheKey, buf, inMemory, false); } }
private static final int getBlockMetricIndex(BlockCategory blockCategory, boolean isCompaction, BlockMetricType metricType) { int i = 0; i = i * NUM_BLOCK_CATEGORIES + blockCategory.ordinal(); i = i * BOOL_VALUES.length + (isCompaction ? 1 : 0); i = i * NUM_METRIC_TYPES + metricType.ordinal(); return i; }
public String getBlockMetricName(BlockCategory blockCategory, boolean isCompaction, BlockMetricType metricType) { if (isCompaction && !metricType.compactionAware) { throw new IllegalArgumentException("isCompaction cannot be true for " + metricType); } return blockMetricNames[getBlockMetricIndex(blockCategory, isCompaction, metricType)]; }
/** * Increments the given metric, both per-CF and aggregate, for both the given * category and all categories in aggregate (four counters total). */ private void incrNumericMetric(BlockCategory blockCategory, boolean isCompaction, BlockMetricType metricType, long amount) { if (blockCategory == null) { blockCategory = BlockCategory.UNKNOWN; // So that we see this in stats. } RegionMetricsStorage.incrNumericMetric(getBlockMetricName(blockCategory, isCompaction, metricType), amount); if (blockCategory != BlockCategory.ALL_CATEGORIES) { incrNumericMetric(BlockCategory.ALL_CATEGORIES, isCompaction, metricType, amount); } }
private void addToReadTime(BlockCategory blockCategory, boolean isCompaction, long timeMs) { RegionMetricsStorage.incrTimeVaryingMetric(getBlockMetricName(blockCategory, isCompaction, BlockMetricType.READ_TIME), timeMs); // Also update the read time aggregated across all block categories if (blockCategory != BlockCategory.ALL_CATEGORIES) { addToReadTime(BlockCategory.ALL_CATEGORIES, isCompaction, timeMs); } }
/** * Updates the number of hits and the total number of block reads on a block * cache hit. */ public void updateOnCacheHit(BlockCategory blockCategory, boolean isCompaction, long count) { blockCategory.expectSpecific(); int idx = getCacheHitMetricIndex(blockCategory, isCompaction); if (this.onHitCacheMetrics.addAndGet(idx, count) > THRESHOLD_METRICS_FLUSH) { flushCertainOnCacheHitMetrics(blockCategory, isCompaction); } if (this != ALL_SCHEMA_METRICS) { ALL_SCHEMA_METRICS.updateOnCacheHit(blockCategory, isCompaction, count); } }
private void flushCertainOnCacheHitMetrics(BlockCategory blockCategory, boolean isCompaction) { int idx = getCacheHitMetricIndex(blockCategory, isCompaction); long tempCount = this.onHitCacheMetrics.getAndSet(idx, 0); if (tempCount > 0) { incrNumericMetric(blockCategory, isCompaction, BlockMetricType.CACHE_HIT, tempCount); incrNumericMetric(blockCategory, isCompaction, BlockMetricType.READ_COUNT, tempCount); } }
/** * Flush the on cache hit metrics; */ private void flushOnCacheHitMetrics() { for (BlockCategory blockCategory : BlockCategory.values()) { for (boolean isCompaction : BOOL_VALUES) { flushCertainOnCacheHitMetrics (blockCategory, isCompaction); } } if (this != ALL_SCHEMA_METRICS) { ALL_SCHEMA_METRICS.flushOnCacheHitMetrics(); } }
/** * Updates read time, the number of misses, and the total number of block * reads on a block cache miss. */ public void updateOnCacheMiss(BlockCategory blockCategory, boolean isCompaction, long timeMs) { blockCategory.expectSpecific(); addToReadTime(blockCategory, isCompaction, timeMs); incrNumericMetric(blockCategory, isCompaction, BlockMetricType.CACHE_MISS); incrNumericMetric(blockCategory, isCompaction, BlockMetricType.READ_COUNT); if (this != ALL_SCHEMA_METRICS) { ALL_SCHEMA_METRICS.updateOnCacheMiss(blockCategory, isCompaction, timeMs); } }
/** * Adds the given delta to the cache size for the given block category and * the aggregate metric for all block categories. Updates both the per-CF * counter and the counter for all CFs (four metrics total). The cache size * metric is "persistent", i.e. it does not get reset when metrics are * collected. */ public void addToCacheSize(BlockCategory category, long cacheSizeDelta) { if (category == null) { category = BlockCategory.ALL_CATEGORIES; } RegionMetricsStorage.incrNumericPersistentMetric(getBlockMetricName(category, false, BlockMetricType.CACHE_SIZE), cacheSizeDelta); if (category != BlockCategory.ALL_CATEGORIES) { addToCacheSize(BlockCategory.ALL_CATEGORIES, cacheSizeDelta); } }
public void updateOnCachePutOrEvict(BlockCategory blockCategory, long cacheSizeDelta, boolean isEviction) { addToCacheSize(blockCategory, cacheSizeDelta); incrNumericMetric(blockCategory, false, isEviction ? BlockMetricType.EVICTED : BlockMetricType.CACHED); if (this != ALL_SCHEMA_METRICS) { ALL_SCHEMA_METRICS.updateOnCachePutOrEvict(blockCategory, cacheSizeDelta, isEviction); } }
void printMetricNames() { for (BlockCategory blockCategory : BlockCategory.values()) { for (boolean isCompaction : BOOL_VALUES) { for (BlockMetricType metricType : BlockMetricType.values()) { int i = getBlockMetricIndex(blockCategory, isCompaction, metricType); LOG.debug("blockCategory=" + blockCategory + ", " + "metricType=" + metricType + ", isCompaction=" + isCompaction + ", metricName=" + blockMetricNames[i]); } } } }
/** * Should we cache a block of a particular category? We always cache important blocks such as * index blocks, as long as the block cache is available. */ public boolean shouldCacheBlockOnRead(BlockCategory category) { boolean shouldCache = isBlockCacheEnabled() && (cacheDataOnRead || category == BlockCategory.INDEX || category == BlockCategory.BLOOM); return shouldCache; }
@Test public void testIncrements() { Random rand = new Random(23982737L); for (int i = 1; i <= 3; ++i) { final String tableName = "table" + i; for (int j = 1; j <= 3; ++j) { final String cfName = "cf" + j; SchemaMetrics sm = SchemaMetrics.getInstance(tableName, cfName); for (boolean isInBloom : BOOL_VALUES) { sm.updateBloomMetrics(isInBloom); checkMetrics(); } for (BlockCategory blockCat : BlockType.BlockCategory.values()) { if (blockCat == BlockCategory.ALL_CATEGORIES) { continue; } for (boolean isCompaction : BOOL_VALUES) { sm.updateOnCacheHit(blockCat, isCompaction); checkMetrics(); sm.updateOnCacheMiss(blockCat, isCompaction, rand.nextInt()); checkMetrics(); } for (boolean isEviction : BOOL_VALUES) { sm.updateOnCachePutOrEvict(blockCat, (isEviction ? -1 : 1) * rand.nextInt(1024 * 1024), isEviction); } } } } }
private void verifyDataAndIndexBlockRead(Map<String, Long> previousMetricSnapshot, SchemaMetrics schemaMetrics, long expectDataBlockRead, long expectedIndexBlockRead){ Map<String, Long> currentMetricsSnapshot = SchemaMetrics.getMetricsSnapshot(); Map<String, Long> diffs = SchemaMetrics.diffMetrics(previousMetricSnapshot, currentMetricsSnapshot); long dataBlockRead = SchemaMetrics.getLong(diffs, schemaMetrics.getBlockMetricName(BlockCategory.DATA, false, BlockMetricType.READ_COUNT)); long indexBlockRead = SchemaMetrics.getLong(diffs, schemaMetrics.getBlockMetricName(BlockCategory.INDEX, false, BlockMetricType.READ_COUNT)); Assert.assertEquals(expectDataBlockRead, dataBlockRead); Assert.assertEquals(expectedIndexBlockRead, indexBlockRead); }
@Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory, final boolean cacheDataInL1) { boolean isMetaBlock = buf.getBlockType().getCategory() != BlockCategory.DATA; if (isMetaBlock || cacheDataInL1) { lruCache.cacheBlock(cacheKey, buf, inMemory, cacheDataInL1); } else { bucketCache.cacheBlock(cacheKey, buf, inMemory, cacheDataInL1); } }
/** * Should we cache a block of a particular category? We always cache * important blocks such as index blocks, as long as the block cache is * available. */ public boolean shouldCacheBlockOnRead(BlockCategory category) { boolean shouldCache = isBlockCacheEnabled() && (cacheDataOnRead || category == BlockCategory.INDEX || category == BlockCategory.BLOOM || (prefetchOnOpen && (category != BlockCategory.META && category != BlockCategory.UNKNOWN))); return shouldCache; }
@Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { boolean isMetaBlock = buf.getBlockType().getCategory() != BlockCategory.DATA; if (isMetaBlock) { lruCache.cacheBlock(cacheKey, buf, inMemory); } else { bucketCache.cacheBlock(cacheKey, buf, inMemory); } }
@Test public void testCacheBlocks() throws IOException { // Set index block size to be the same as normal block size. TEST_UTIL.getConfiguration().setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, BLOCK_SIZE); HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes(CF)) .setMaxVersions(MAX_VERSIONS) .setCompressionType(COMPRESSION_ALGORITHM) .setBloomFilterType(BLOOM_TYPE); hcd.setBlocksize(BLOCK_SIZE); hcd.setBlockCacheEnabled(cfCacheEnabled); HRegion region = TEST_UTIL.createTestRegion(TABLE, hcd); writeTestData(region); for (int i = 0; i < NUM_ROWS; ++i) { Get get = new Get(Bytes.toBytes("row" + i)); region.get(get); } List<BlockCategory> importantBlockCategories = new ArrayList<BlockCategory>(); importantBlockCategories.add(BlockCategory.BLOOM); if (hfileVersion == 2) { // We only have index blocks for HFile v2. importantBlockCategories.add(BlockCategory.INDEX); } }
/** * Should we cache a block of a particular category? We always cache * important blocks such as index blocks, as long as the block cache is * available. */ public boolean shouldCacheBlockOnRead(BlockCategory category) { boolean shouldCache = isBlockCacheEnabled() && (cacheDataOnRead || category == BlockCategory.INDEX || category == BlockCategory.BLOOM); return shouldCache; }