RegionLoad.Builder setCompleteSequenceId(RegionLoad.Builder regionLoadBldr) { long lastFlushOpSeqIdLocal = this.lastFlushOpSeqId; byte[] encodedRegionName = this.getRegionInfo().getEncodedNameAsBytes(); regionLoadBldr.clearStoreCompleteSequenceId(); for (byte[] familyName : this.stores.keySet()) { long earliest = this.wal.getEarliestMemstoreSeqNum(encodedRegionName, familyName); // Subtract - 1 to go earlier than the current oldest, unflushed edit in // memstore; this will // give us a sequence id that is for sure flushed. We want edit replay to // start after this // sequence id in this region. If NO_SEQNUM, use the regions maximum flush // id. long csid = (earliest == HConstants.NO_SEQNUM) ? lastFlushOpSeqIdLocal : earliest - 1; regionLoadBldr.addStoreCompleteSequenceId( StoreSequenceId.newBuilder().setFamilyName(ByteString.copyFrom(familyName)) .setSequenceId(csid).build()); } return regionLoadBldr.setCompleteSequenceId(getMaxFlushedSeqId()); }
public static long getTotalRequestsCount(RegionLoad rl) { if (rl == null) { return 0; } return rl.getReadRequestsCount() + rl.getWriteRequestsCount(); }
ClusterStatusProtos.ServerLoad buildServerLoad(long reportStartTime, long reportEndTime) { // We're getting the MetricsRegionServerWrapper here because the wrapper computes requests // per second, and other metrics As long as metrics are part of ServerLoad it's best to use // the wrapper to compute those numbers in one place. // In the long term most of these should be moved off of ServerLoad and the heart beat. // Instead they should be stored in an HBase table so that external visibility into HBase is // improved; Additionally the load balancer will be able to take advantage of a more complete // history. MetricsRegionServerWrapper regionServerWrapper = this.metricsRegionServer.getRegionServerWrapper(); Collection<HRegion> regions = getOnlineRegionsLocalContext(); MemoryUsage memory = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); ClusterStatusProtos.ServerLoad.Builder serverLoad = ClusterStatusProtos.ServerLoad.newBuilder(); serverLoad.setNumberOfRequests((int) regionServerWrapper.getRequestsPerSecond()); serverLoad.setTotalNumberOfRequests((int) regionServerWrapper.getTotalRequestCount()); serverLoad.setUsedHeapMB((int)(memory.getUsed() / 1024 / 1024)); serverLoad.setMaxHeapMB((int) (memory.getMax() / 1024 / 1024)); Set<String> coprocessors = this.hlog.getCoprocessorHost().getCoprocessors(); for (String coprocessor : coprocessors) { serverLoad.addCoprocessors( Coprocessor.newBuilder().setName(coprocessor).build()); } RegionLoad.Builder regionLoadBldr = RegionLoad.newBuilder(); RegionSpecifier.Builder regionSpecifier = RegionSpecifier.newBuilder(); for (HRegion region : regions) { serverLoad.addRegionLoads(createRegionLoad(region, regionLoadBldr, regionSpecifier)); } serverLoad.setReportStartTime(reportStartTime); serverLoad.setReportEndTime(reportEndTime); if (this.infoServer != null) { serverLoad.setInfoServerPort(this.infoServer.getPort()); } else { serverLoad.setInfoServerPort(-1); } return serverLoad.build(); }
private RegionLoad createRegionLoad(final Region r, RegionLoad.Builder regionLoadBldr, RegionSpecifier.Builder regionSpecifier) throws IOException { byte[] name = r.getRegionInfo().getRegionName(); int stores = 0; int storefiles = 0; int storeUncompressedSizeMB = 0; int storefileSizeMB = 0; int memstoreSizeMB = (int) (r.getMemstoreSize() / 1024 / 1024); int storefileIndexSizeMB = 0; int rootIndexSizeKB = 0; int totalStaticIndexSizeKB = 0; int totalStaticBloomSizeKB = 0; long totalCompactingKVs = 0; long currentCompactedKVs = 0; List<Store> storeList = r.getStores(); stores += storeList.size(); for (Store store : storeList) { storefiles += store.getStorefilesCount(); storeUncompressedSizeMB += (int) (store.getStoreSizeUncompressed() / 1024 / 1024); storefileSizeMB += (int) (store.getStorefilesSize() / 1024 / 1024); storefileIndexSizeMB += (int) (store.getStorefilesIndexSize() / 1024 / 1024); CompactionProgress progress = store.getCompactionProgress(); if (progress != null) { totalCompactingKVs += progress.totalCompactingKVs; currentCompactedKVs += progress.currentCompactedKVs; } rootIndexSizeKB += (int) (store.getStorefilesIndexSize() / 1024); totalStaticIndexSizeKB += (int) (store.getTotalStaticIndexSize() / 1024); totalStaticBloomSizeKB += (int) (store.getTotalStaticBloomSize() / 1024); } float dataLocality = r.getHDFSBlocksDistribution().getBlockLocalityIndex(serverName.getHostname()); if (regionLoadBldr == null) { regionLoadBldr = RegionLoad.newBuilder(); } if (regionSpecifier == null) { regionSpecifier = RegionSpecifier.newBuilder(); } regionSpecifier.setType(RegionSpecifierType.REGION_NAME); regionSpecifier.setValue(ByteStringer.wrap(name)); regionLoadBldr.setRegionSpecifier(regionSpecifier.build()).setStores(stores) .setStorefiles(storefiles).setStoreUncompressedSizeMB(storeUncompressedSizeMB) .setStorefileSizeMB(storefileSizeMB).setMemstoreSizeMB(memstoreSizeMB) .setStorefileIndexSizeMB(storefileIndexSizeMB).setRootIndexSizeKB(rootIndexSizeKB) .setTotalStaticIndexSizeKB(totalStaticIndexSizeKB) .setTotalStaticBloomSizeKB(totalStaticBloomSizeKB) .setReadRequestsCount(r.getReadRequestsCount()) .setWriteRequestsCount(r.getWriteRequestsCount()).setTotalCompactingKVs(totalCompactingKVs) .setCurrentCompactedKVs(currentCompactedKVs).setDataLocality(dataLocality) .setLastMajorCompactionTs(r.getOldestHfileTs(true)); ((HRegion) r).setCompleteSequenceId(regionLoadBldr); return regionLoadBldr.build(); }
/** * @param encodedRegionName * @return An instance of RegionLoad. */ public RegionLoad createRegionLoad(final String encodedRegionName) throws IOException { Region r = onlineRegions.get(encodedRegionName); return r != null ? createRegionLoad(r, null, null) : null; }
private RegionLoad createRegionLoad(final HRegion r, RegionLoad.Builder regionLoadBldr, RegionSpecifier.Builder regionSpecifier) { byte[] name = r.getRegionName(); int stores = 0; int storefiles = 0; int storeUncompressedSizeMB = 0; int storefileSizeMB = 0; int memstoreSizeMB = (int) (r.memstoreSize.get() / 1024 / 1024); int storefileIndexSizeMB = 0; int rootIndexSizeKB = 0; int totalStaticIndexSizeKB = 0; int totalStaticBloomSizeKB = 0; long totalCompactingKVs = 0; long currentCompactedKVs = 0; synchronized (r.stores) { stores += r.stores.size(); for (Store store : r.stores.values()) { storefiles += store.getStorefilesCount(); storeUncompressedSizeMB += (int) (store.getStoreSizeUncompressed() / 1024 / 1024); storefileSizeMB += (int) (store.getStorefilesSize() / 1024 / 1024); storefileIndexSizeMB += (int) (store.getStorefilesIndexSize() / 1024 / 1024); CompactionProgress progress = store.getCompactionProgress(); if (progress != null) { totalCompactingKVs += progress.totalCompactingKVs; currentCompactedKVs += progress.currentCompactedKVs; } rootIndexSizeKB += (int) (store.getStorefilesIndexSize() / 1024); totalStaticIndexSizeKB += (int) (store.getTotalStaticIndexSize() / 1024); totalStaticBloomSizeKB += (int) (store.getTotalStaticBloomSize() / 1024); } } float dataLocality = r.getHDFSBlocksDistribution().getBlockLocalityIndex(serverName.getHostname()); if (regionLoadBldr == null) { regionLoadBldr = RegionLoad.newBuilder(); } if (regionSpecifier == null) { regionSpecifier = RegionSpecifier.newBuilder(); } regionSpecifier.setType(RegionSpecifierType.REGION_NAME); regionSpecifier.setValue(ByteStringer.wrap(name)); regionLoadBldr.setRegionSpecifier(regionSpecifier.build()) .setStores(stores) .setStorefiles(storefiles) .setStoreUncompressedSizeMB(storeUncompressedSizeMB) .setStorefileSizeMB(storefileSizeMB) .setMemstoreSizeMB(memstoreSizeMB) .setStorefileIndexSizeMB(storefileIndexSizeMB) .setRootIndexSizeKB(rootIndexSizeKB) .setTotalStaticIndexSizeKB(totalStaticIndexSizeKB) .setTotalStaticBloomSizeKB(totalStaticBloomSizeKB) .setReadRequestsCount(r.readRequestsCount.get()) .setWriteRequestsCount(r.writeRequestsCount.get()) .setTotalCompactingKVs(totalCompactingKVs) .setCurrentCompactedKVs(currentCompactedKVs) .setCompleteSequenceId(r.lastFlushSeqId) .setDataLocality(dataLocality); return regionLoadBldr.build(); }
/** * @param encodedRegionName * @return An instance of RegionLoad. */ public RegionLoad createRegionLoad(final String encodedRegionName) { HRegion r = null; r = this.onlineRegions.get(encodedRegionName); return r != null ? createRegionLoad(r, null, null) : null; }
private RegionLoad createRegionLoad(final HRegion r, RegionLoad.Builder regionLoadBldr, RegionSpecifier.Builder regionSpecifier) { byte[] name = r.getRegionName(); int stores = 0; int storefiles = 0; int storeUncompressedSizeMB = 0; int storefileSizeMB = 0; int memstoreSizeMB = (int) (r.memstoreSize.get() / 1024 / 1024); int storefileIndexSizeMB = 0; int rootIndexSizeKB = 0; int totalStaticIndexSizeKB = 0; int totalStaticBloomSizeKB = 0; long totalCompactingKVs = 0; long currentCompactedKVs = 0; synchronized (r.stores) { stores += r.stores.size(); for (Store store : r.stores.values()) { storefiles += store.getStorefilesCount(); storeUncompressedSizeMB += (int) (store.getStoreSizeUncompressed() / 1024 / 1024); storefileSizeMB += (int) (store.getStorefilesSize() / 1024 / 1024); storefileIndexSizeMB += (int) (store.getStorefilesIndexSize() / 1024 / 1024); CompactionProgress progress = store.getCompactionProgress(); if (progress != null) { totalCompactingKVs += progress.totalCompactingKVs; currentCompactedKVs += progress.currentCompactedKVs; } rootIndexSizeKB += (int) (store.getStorefilesIndexSize() / 1024); totalStaticIndexSizeKB += (int) (store.getTotalStaticIndexSize() / 1024); totalStaticBloomSizeKB += (int) (store.getTotalStaticBloomSize() / 1024); } } if (regionLoadBldr == null) { regionLoadBldr = RegionLoad.newBuilder(); } if (regionSpecifier == null) { regionSpecifier = RegionSpecifier.newBuilder(); } regionSpecifier.setType(RegionSpecifierType.REGION_NAME); regionSpecifier.setValue(HBaseZeroCopyByteString.wrap(name)); regionLoadBldr.setRegionSpecifier(regionSpecifier.build()) .setStores(stores) .setStorefiles(storefiles) .setStoreUncompressedSizeMB(storeUncompressedSizeMB) .setStorefileSizeMB(storefileSizeMB) .setMemstoreSizeMB(memstoreSizeMB) .setStorefileIndexSizeMB(storefileIndexSizeMB) .setRootIndexSizeKB(rootIndexSizeKB) .setTotalStaticIndexSizeKB(totalStaticIndexSizeKB) .setTotalStaticBloomSizeKB(totalStaticBloomSizeKB) .setReadRequestsCount(r.readRequestsCount.get()) .setWriteRequestsCount(r.writeRequestsCount.get()) .setTotalCompactingKVs(totalCompactingKVs) .setCurrentCompactedKVs(currentCompactedKVs) .setCompleteSequenceId(r.completeSequenceId); return regionLoadBldr.build(); }
private RegionLoad createRegionLoad(final HRegion r, RegionLoad.Builder regionLoadBldr, RegionSpecifier.Builder regionSpecifier) { byte[] name = r.getRegionName(); int stores = 0; int storefiles = 0; int storeUncompressedSizeMB = 0; int storefileSizeMB = 0; int memstoreSizeMB = (int) (r.memstoreSize.get() / 1024 / 1024); int storefileIndexSizeMB = 0; int rootIndexSizeKB = 0; int totalStaticIndexSizeKB = 0; int totalStaticBloomSizeKB = 0; long totalCompactingKVs = 0; long currentCompactedKVs = 0; synchronized (r.stores) { stores += r.stores.size(); for (Store store : r.stores.values()) { storefiles += store.getStorefilesCount(); storeUncompressedSizeMB += (int) (store.getStoreSizeUncompressed() / 1024 / 1024); storefileSizeMB += (int) (store.getStorefilesSize() / 1024 / 1024); storefileIndexSizeMB += (int) (store.getStorefilesIndexSize() / 1024 / 1024); CompactionProgress progress = store.getCompactionProgress(); if (progress != null) { totalCompactingKVs += progress.totalCompactingKVs; currentCompactedKVs += progress.currentCompactedKVs; } rootIndexSizeKB += (int) (store.getStorefilesIndexSize() / 1024); totalStaticIndexSizeKB += (int) (store.getTotalStaticIndexSize() / 1024); totalStaticBloomSizeKB += (int) (store.getTotalStaticBloomSize() / 1024); } } if (regionLoadBldr == null) { regionLoadBldr = RegionLoad.newBuilder(); } if (regionSpecifier == null) { regionSpecifier = RegionSpecifier.newBuilder(); } regionSpecifier.setType(RegionSpecifierType.REGION_NAME); regionSpecifier.setValue(HBaseZeroCopyByteString.wrap(name)); regionLoadBldr.setRegionSpecifier(regionSpecifier.build()) .setStores(stores) .setStorefiles(storefiles) .setStoreUncompressedSizeMB(storeUncompressedSizeMB) .setStorefileSizeMB(storefileSizeMB) .setMemstoreSizeMB(memstoreSizeMB) .setStorefileIndexSizeMB(storefileIndexSizeMB) .setRootIndexSizeKB(rootIndexSizeKB) .setTotalStaticIndexSizeKB(totalStaticIndexSizeKB) .setTotalStaticBloomSizeKB(totalStaticBloomSizeKB) .setReadRequestsCount((int) r.readRequestsCount.get()) .setWriteRequestsCount((int) r.writeRequestsCount.get()) .setTotalCompactingKVs(totalCompactingKVs) .setCurrentCompactedKVs(currentCompactedKVs) .setCompleteSequenceId(r.lastFlushSeqId); return regionLoadBldr.build(); }
private RegionLoad createRegionLoad(final HRegion r) { byte[] name = r.getRegionName(); int stores = 0; int storefiles = 0; int storeUncompressedSizeMB = 0; int storefileSizeMB = 0; int memstoreSizeMB = (int) (r.memstoreSize.get() / 1024 / 1024); int storefileIndexSizeMB = 0; int rootIndexSizeKB = 0; int totalStaticIndexSizeKB = 0; int totalStaticBloomSizeKB = 0; long totalCompactingKVs = 0; long currentCompactedKVs = 0; synchronized (r.stores) { stores += r.stores.size(); for (Store store : r.stores.values()) { storefiles += store.getStorefilesCount(); storeUncompressedSizeMB += (int) (store.getStoreSizeUncompressed() / 1024 / 1024); storefileSizeMB += (int) (store.getStorefilesSize() / 1024 / 1024); storefileIndexSizeMB += (int) (store.getStorefilesIndexSize() / 1024 / 1024); CompactionProgress progress = store.getCompactionProgress(); if (progress != null) { totalCompactingKVs += progress.totalCompactingKVs; currentCompactedKVs += progress.currentCompactedKVs; } rootIndexSizeKB += (int) (store.getStorefilesIndexSize() / 1024); totalStaticIndexSizeKB += (int) (store.getTotalStaticIndexSize() / 1024); totalStaticBloomSizeKB += (int) (store.getTotalStaticBloomSize() / 1024); } } RegionLoad.Builder regionLoad = RegionLoad.newBuilder(); RegionSpecifier.Builder regionSpecifier = RegionSpecifier.newBuilder(); regionSpecifier.setType(RegionSpecifierType.REGION_NAME); regionSpecifier.setValue(ZeroCopyLiteralByteString.wrap(name)); regionLoad.setRegionSpecifier(regionSpecifier.build()) .setStores(stores) .setStorefiles(storefiles) .setStoreUncompressedSizeMB(storeUncompressedSizeMB) .setStorefileSizeMB(storefileSizeMB) .setMemstoreSizeMB(memstoreSizeMB) .setStorefileIndexSizeMB(storefileIndexSizeMB) .setRootIndexSizeKB(rootIndexSizeKB) .setTotalStaticIndexSizeKB(totalStaticIndexSizeKB) .setTotalStaticBloomSizeKB(totalStaticBloomSizeKB) .setReadRequestsCount((int) r.readRequestsCount.get()) .setWriteRequestsCount((int) r.writeRequestsCount.get()) .setTotalCompactingKVs(totalCompactingKVs) .setCurrentCompactedKVs(currentCompactedKVs) .setCompleteSequenceId(r.completeSequenceId); return regionLoad.build(); }
/** * @param encodedRegionName * @return An instance of RegionLoad. */ public RegionLoad createRegionLoad(final String encodedRegionName) { HRegion r = null; r = this.onlineRegions.get(encodedRegionName); return r != null ? createRegionLoad(r) : null; }