public static String printRange(Range r) { StringBuilder sb = new StringBuilder(); sb.append("[" + Bytes.toString(r.getFamily()) + ":" + Bytes.toString(r.getQualifier()) + "], values ("); if (r.getStartValue() != null) { sb.append(LCIndexConstant.getStringOfValueAndType(r.getDataType(), r.getStartValue())); if (r.getStartType() == CompareOp.EQUAL || r.getStartType() == CompareOp.NOT_EQUAL) { sb.append(" <== ").append(r.getStartType()).append(" )"); return sb.toString(); } } else { sb.append("null"); } sb.append(", "); if (r.getStopValue() != null) { sb.append(LCIndexConstant.getStringOfValueAndType(r.getDataType(), r.getStopValue())); } else { sb.append("MAX"); } sb.append(")"); return sb.toString(); }
public float getIndexScanScale(Range r) { long rangeSize = 0, totalSize = 0; lock.readLock().lock(); try { for (StoreFile sf : this.getStorefiles()) { // LCTODO: next, init index file reader after compaction! long[] tmp = sf.getIndexReader().getIndexFileReader().getRangeScale(r); rangeSize += tmp[0]; totalSize += tmp[1]; } } finally { lock.readLock().unlock(); } if (totalSize == 0) { return 0.0f; } else { return (float) (((double) rangeSize) / ((double) totalSize)); } }
private String mWinterToPrintRange(Range r, byte[] cf, boolean isMain) { StringBuilder sb = new StringBuilder(); byte[] targetQualifier = isMain ? null : r.getQualifier(); sb.append("[" + Bytes.toString(cf) + ":" + Bytes.toString(targetQualifier) + "], values ("); if (r.getStartValue() != null) { sb.append(LCCIndexConstant.getStringOfValue(lccIndexQualifierType, targetQualifier, r.getStartValue())); if (r.getStartType() == CompareOp.EQUAL || r.getStartType() == CompareOp.NOT_EQUAL) { sb.append(", ").append(r.getStartType()).append(")"); return sb.toString(); } } else { sb.append("null"); } sb.append(", "); if (r.getStopValue() != null) { sb.append(LCCIndexConstant.getStringOfValue(lccIndexQualifierType, targetQualifier, r.getStopValue())); } else { sb.append("MAX"); } sb.append(")"); return sb.toString(); }
private boolean mWinterShouldDiscardRow(Range r, byte[] row) { if (r.getStartValue() != null && r.getStartType() == CompareOp.GREATER) { if (mWinterStringStartsWith(row, r.getStartValue())) { // System.out.println("winter report on greater than start"); return true; } } if (r.getStopValue() != null) { if (r.getStopType() == CompareOp.LESS) { if (mWinterStringStartsWith(row, r.getStopValue())) { // System.out.println("winter report on stop and less"); return true; } } } return false; }
public float getIndexScanScale(Range r) { long rangeSize = 0, totalSize = 0; lock.readLock().lock(); try { for (StoreFile sf : this.getStorefiles()) { long[] tmp = sf.getIndexReader().getIndexFileReader().getRangeScale(r); rangeSize += tmp[0]; totalSize += tmp[1]; } } finally { lock.readLock().unlock(); } if (totalSize == 0) { return 0.0f; } else { return (float) (((double) rangeSize) / ((double) totalSize)); } }
public static String printRange(Range r) { StringBuilder sb = new StringBuilder(); sb.append("[" + Bytes.toString(r.getFamily()) + ":" + Bytes.toString(r.getQualifier()) + "], values ("); if (r.getStartValue() != null) { sb.append(LCCIndexConstant.getStringOfValueAndType(r.getDataType(), r.getStartValue())); if (r.getStartType() == CompareOp.EQUAL || r.getStartType() == CompareOp.NOT_EQUAL) { sb.append(" <== ").append(r.getStartType()).append(" )"); return sb.toString(); } } else { sb.append("null"); } sb.append(", "); if (r.getStopValue() != null) { sb.append(LCCIndexConstant.getStringOfValueAndType(r.getDataType(), r.getStopValue())); } else { sb.append("MAX"); } sb.append(")"); return sb.toString(); }
@Override public ResultScanner getScanner() throws IOException { Scan scan = new Scan(); FilterList filters = new FilterList(); for (Range range : ranges) { if (range.getStartValue() != null) { filters.addFilter(new SingleColumnValueFilter(range.getFamily(), range.getQualifier(), range.getStartType(), range.getStartValue())); } if (range.getStopValue() != null) { filters.addFilter(new SingleColumnValueFilter(range.getFamily(), range.getQualifier(), range.getStopType(), range.getStopValue())); } System.out.println("coffey hbase main index range: " + Bytes.toString(range.getColumn()) + " [" + LCCIndexConstant.getStringOfValueAndType(range.getDataType(), range.getStartValue()) + "," + LCCIndexConstant.getStringOfValueAndType(range.getDataType(), range.getStopValue()) + "]"); scan.setCacheBlocks(false); } scan.setCacheBlocks(false); scan.setFilter(filters); return table.getScanner(scan); }
@Override public ResultScanner getScanner() throws IOException { Scan scan = new Scan(); RangeList list = new RangeList(); FilterList filters = new FilterList(); for (Range r : ranges) { list.addRange(r); if (r.getStartValue() != null) { filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes(TPCHConstants.FAMILY_NAME), r .getQualifier(), r.getStartType(), r.getStartValue())); } if (r.getStopValue() != null) { filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes(TPCHConstants.FAMILY_NAME), r .getQualifier(), r.getStopType(), r.getStopValue())); } } scan.setFilter(filters); scan.setAttribute(IndexConstants.SCAN_WITH_INDEX, Writables.getBytes(list)); scan.setAttribute(IndexConstants.MAX_SCAN_SCALE, Bytes.toBytes(0.3)); scan.setCacheBlocks(false); return table.getScanner(scan); }
public void work(String confPath, String assignedFile, String testClass, String rangeFilterFile, int cacheSize) throws IOException, InterruptedException { double totalTime = 0, temp; ArrayList<Double> timeList = new ArrayList<Double>(); List<Range> rangeList = getRangesFromFile(rangeFilterFile); for (Range r : rangeList) { System.out.println("coffey get range: " + TPCHConstants.printRange(r)); } for (int i = 0; i < ROUND; ++i) { temp = runOneTime(confPath, assignedFile, testClass, rangeList, cacheSize); totalTime += temp; timeList.add(temp); if (ROUND > 1) { Thread.sleep(TPCHConstants.ROUND_SLEEP_TIME); } } System.out.println("coffey report scan, run " + testClass + ", have run " + ROUND + " times, avg: " + totalTime / ROUND); System.out.println("coffey reporting scan each time: "); for (int i = 0; i < timeList.size(); ++i) { System.out.println("coffey report scan round " + i + ": " + timeList.get(i)); } }
public ConditionTreeLeafNode(HRegion region, Range range, float maxScale) { this.range = range; this.scale = region.getStore(range.getFamily()).getIndexScanScale(range); // prune large scale node if (this.scale > maxScale) { Log.info("LCINFO: A prune=true because (this.scale) " + this.scale + " > " + maxScale + "(max scale)"); this.prune = true; } }
@Override public StoreIndexScanner getStoreIndexScanner(Range r, Scan s, Set<ByteArray> joinSet, boolean isAND) throws IOException { lock.readLock().lock(); try { StoreIndexScanner sis = new StoreIndexScanner(this, this.memstore.getScanners(region.getMVCC().getReadPoint()), this.comparator, this.irIndexComparator, r, s, joinSet, isAND); this.addChangedReaderObserver(sis); return sis; } finally { lock.readLock().unlock(); } }
public ConditionTreeLeafNode(HRegion region, Range range, float maxScale) { this.range = range; this.scale = region.getStore(range.getFamily()).getIndexScanScale(range); // prune large scale node if (this.scale > maxScale) { System.out.println("winter scale too big"); this.prune = true; } }
private Range mWinterUpdatePriLCCIndex(ArrayList<Range> rangeList, Set<Entry<byte[], NavigableSet<byte[]>>> sets) throws IOException { assert rangeList != null && rangeList.size() > 0; // Range target = mWinterCalProperOneAsMainIndex(rangeList); Range target = selectTheBestRange(rangeList, sets); if (target == null) { throw new IOException("winter no main target range is found!"); } else { // System.out.println("winter select the best target range: " // + Bytes.toString(target.getQualifier())); } rangeList.remove(target); lccMainQualifier = target.getQualifier(); byte[] startValue = target.getStartValue(); byte[] stopValue = target.getStopValue(); if (startValue != null) { startValue = Bytes.add(mWinterCalRangeKey(target.getQualifier(), startValue), LCCIndexConstant.DELIMITER_BYTES); } if (stopValue != null) { stopValue = Bytes.add(mWinterCalRangeKey(target.getQualifier(), stopValue), LCCIndexConstant.DELIMITER_PLUS_ONE_BYTES); } else if (target.getStartType() == CompareOp.EQUAL) { stopValue = Bytes.add(mWinterCalRangeKey(target.getQualifier(), target.getStartValue()), LCCIndexConstant.DELIMITER_PLUS_ONE_BYTES); } return new Range( LCCIndexGenerator.mWinterGenerateLCCIndexFamily_Bytes(target.getQualifier()), startValue, target.getStartType(), stopValue, target.getStopType()); }
private void mWinterPrintRanges(Range mainRange, ArrayList<Range> lccRangeFilters) { String temp; temp = mWinterToPrintRange(mainRange, mainRange.getFamily(), true); System.out.println("winter filter main range: " + temp); for (Range r : lccRangeFilters) { temp = mWinterToPrintRange(r, mainRange.getFamily(), false); assert temp != null && temp.length() > 0; System.out.println("winter filter range: " + temp); } }
private Range mWinterCalProperOneAsMainIndex(ArrayList<Range> rangeList) { for (Range r : rangeList) { DataType type = LCCIndexConstant.getType(lccIndexQualifierType, r.getQualifier()); if (type == DataType.DOUBLE) { return r; } } return rangeList.get(0); }
protected Range selectTheBestRange(List<Range> ranges, Set<Entry<byte[], NavigableSet<byte[]>>> sets) throws IOException { if (ranges.size() == 0) return null; List<Range> filteredRanges = new ArrayList<Range>(); for (Range r : ranges) { if (r.getStartType() == CompareOp.NOT_EQUAL) continue; if (r.getStartType() == CompareOp.NO_OP && r.getStopType() == CompareOp.NO_OP) { continue; } filteredRanges.add(r); } if (filteredRanges.size() == 0) { System.out.println("winter no good after filter"); return ranges.get(0); } else if (filteredRanges.size() == 1) { System.out.println("winter only one is good after filter"); return filteredRanges.get(0); } long start = System.currentTimeMillis(); for (Map.Entry<byte[], NavigableSet<byte[]>> entry : sets) { Store store = stores.get(entry.getKey()); Range best = store.mWinterSelectTheBestRange(filteredRanges); if (best != null) { System.out.println("winter finding the best range cost " + 1.0 * (System.currentTimeMillis() - start) / 1000.0 + " seconds"); return best; } } System.out.println("winter finding the best range cost " + 1.0 * (System.currentTimeMillis() - start) / 1000.0 + " seconds"); return ranges.get(0); }
private boolean mWinterShouldDiscard(Range r, KeyValue kv) { if (Bytes.equals(r.getQualifier(), kv.getQualifier())) { // winter in the same qualifier byte[] qualifier = r.getQualifier(); DataType type = LCCIndexConstant.getType(lccIndexQualifierType, qualifier); if (r.getStartValue() != null) { if (!mWinterValueCoincident(r.getStartType(), kv.getValue(), r.getStartValue(), type)) { // System.out.println("winter kv: " + LCCIndexConstant.mWinterToPrint(kv) // + " not match start: " // + LCCIndexConstant.getStringOfValueAndType(type, r.getStartValue()) + ", " // + r.getStartType() + ", type: " + type); return true; } } if (r.getStopValue() != null) { if (!mWinterValueCoincident(r.getStopType(), kv.getValue(), r.getStopValue(), type)) { // System.out.println("winter kv: " + LCCIndexConstant.mWinterToPrint(kv) // + " not match stop: " // + LCCIndexConstant.getStringOfValueAndType(type, r.getStopValue()) + ", " // + r.getStopType() + ", type: " + type); return true; } } } return false; }
public StoreIndexScanner getStoreIndexScanner(Range r, Scan s, Set<ByteArray> joinSet, boolean isAND) throws IOException { lock.readLock().lock(); try { StoreIndexScanner sis = new StoreIndexScanner(this, this.memstore.getScanners(), this.comparator, this.indexComparator, r, s, joinSet, isAND); this.addChangedReaderObserver(sis); return sis; } finally { lock.readLock().unlock(); } }
private void init() throws IOException { rawTable = new HTable(conf, tableName); indexTable = new IndexTable(conf, tableName); Scan scan = new Scan(); Range bestRange = selectTheBestRange(); for (Range r : ranges) { if (r == bestRange) continue; if (r.getStartValue() != null) { filters.addFilter(new SingleColumnValueFilter(r.getFamily(), r.getQualifier(), r .getStartType(), r.getStartValue())); } if (r.getStopValue() != null) { filters.addFilter(new SingleColumnValueFilter(r.getFamily(), r.getQualifier(), r .getStopType(), r.getStopValue())); } System.out.println("coffey cmindex (one main then filter) aid filter for range: " + Bytes.toString(bestRange.getColumn()) + " [" + LCCIndexConstant.getStringOfValueAndType(bestRange.getDataType(), bestRange.getStartValue()) + "," + LCCIndexConstant.getStringOfValueAndType(bestRange.getDataType(), bestRange.getStopValue()) + "]"); } scannedTable = indexTable.indexTableMaps.get(bestRange.getColumn()); scan.setStartRow(bestRange.getStartValue()); scan.setStopRow(bestRange.getStopValue()); System.out.println("coffey cmindex (one main then filter) main index range: " + Bytes.toString(bestRange.getColumn()) + " [" + LCCIndexConstant.getStringOfValueAndType(bestRange.getDataType(), bestRange.getStartValue()) + "," + LCCIndexConstant.getStringOfValueAndType(bestRange.getDataType(), bestRange.getStopValue()) + "]"); scan.setCacheBlocks(false); indexScanner = scannedTable.getScanner(scan); }
public TPCHScanLCC(String confPath, String newAddedFile, String tableName, List<Range> ranges) throws IOException { super(confPath, newAddedFile, tableName, ranges); // System.out.println("coffey disable table at first: " + tableName); // admin.disableTable(tableName); // System.out.println("coffey enable table then: " + tableName); // admin.enableTable(tableName); table = new HTable(conf, tableName); }
@Override public ResultScanner getScanner() throws IOException { Scan scan = new Scan(); RangeList list = new RangeList(); for (Range r : ranges) { list.addRange(r); } scan.setCacheBlocks(false); scan.setAttribute(LCCIndexConstant.SCAN_WITH_LCCINDEX, Writables.getBytes(list)); return table.getScanner(scan); }
public TPCHScanBaseClass(String confPath, String newAddedFile, String tableName, List<Range> ranges) throws IOException { this.tableName = tableName; conf = HBaseConfiguration.create(); conf.addResource(confPath); TPCHConstants.parseMannuallyAssignedFile(conf, newAddedFile); System.out.println("coffey manually set zk " + "hbase.zookeeper.quorum " + " hec-14,hec-02,hec-03"); // conf.set("hbase.zookeeper.quorum", "hec-14,hec-02,hec-03"); admin = new HBaseAdmin(conf); table = new HTable(conf, tableName); this.ranges = ranges; }
protected Range selectTheBestRange() { int equal_id = -1, not_equal_id = -1, rangeID = -1; for (int i = 0; i < ranges.size(); ++i) { Range r = ranges.get(i); if (r.getStopValue() != null) { if (r.getStartValue() != null) return r; if (r.getStopType() == CompareOp.LESS || r.getStopType() == CompareOp.LESS_OR_EQUAL) { rangeID = i; } else if (r.getStopType() == CompareOp.EQUAL) { equal_id = i; } else if (r.getStopType() == CompareOp.NOT_EQUAL) { not_equal_id = i; } } else { if (r.getStartType() == CompareOp.GREATER || r.getStartType() == CompareOp.GREATER_OR_EQUAL) { rangeID = i; } else if (r.getStartType() == CompareOp.EQUAL) { equal_id = i; } else if (r.getStartType() == CompareOp.NOT_EQUAL) { not_equal_id = i; } } } if (rangeID != -1) return ranges.get(rangeID); if (equal_id != -1) return ranges.get(equal_id); if (not_equal_id != -1) return ranges.get(not_equal_id); return ranges.size() == 0 ? null : ranges.get(0); }
@Override public ResultScanner getScanner() throws IOException { Scan scan = new Scan(); RangeList list = new RangeList(); List<CF_INFO> cfs = PutTestConstants.getCFInfo(); FilterList filters = new FilterList(); for (CF_INFO ci : cfs) { if (ci.isIndex) { list.addRange(new Range(Bytes.toBytes(FAMILY_NAME + ":" + ci.qualifier), LCCIndexConstant .parsingStringToBytesWithType(ci.type, String.valueOf(startValue)), CompareOp.GREATER_OR_EQUAL, LCCIndexConstant.parsingStringToBytesWithType(ci.type, String.valueOf(stopValue)), CompareOp.LESS_OR_EQUAL)); filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes(FAMILY_NAME), Bytes .toBytes(ci.qualifier), CompareOp.GREATER_OR_EQUAL, LCCIndexConstant .parsingStringToBytesWithType(ci.type, String.valueOf(startValue)))); filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes(FAMILY_NAME), Bytes .toBytes(ci.qualifier), CompareOp.LESS_OR_EQUAL, LCCIndexConstant .parsingStringToBytesWithType(ci.type, String.valueOf(stopValue)))); System.out.println("coffey irindex add filter for type: " + ci.type + " [" + startValue + "," + stopValue + "]"); } } scan.setFilter(filters); scan.setAttribute(IndexConstants.SCAN_WITH_INDEX, Writables.getBytes(list)); scan.setAttribute(IndexConstants.MAX_SCAN_SCALE, Bytes.toBytes(0.3)); scan.setCacheBlocks(false); return table.getScanner(scan); }
@Override public ResultScanner getScanner() throws IOException { Scan scan = new Scan(); RangeList list = new RangeList(); List<CF_INFO> cfs = PutTestConstants.getCFInfo(); FilterList filters = new FilterList(); for (CF_INFO ci : cfs) { if (ci.isIndex) { list.addRange(new Range(Bytes.toBytes(FAMILY_NAME + ":" + ci.qualifier), LCCIndexConstant .parsingStringToBytesWithType(ci.type, String.valueOf(startValue)), CompareOp.GREATER_OR_EQUAL, LCCIndexConstant.parsingStringToBytesWithType(ci.type, String.valueOf(stopValue)), CompareOp.LESS_OR_EQUAL)); filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("if_A"), Bytes .toBytes(ci.qualifier), CompareOp.GREATER_OR_EQUAL, LCCIndexConstant .parsingStringToBytesWithType(ci.type, String.valueOf(startValue)))); filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("if_A"), Bytes .toBytes(ci.qualifier), CompareOp.LESS_OR_EQUAL, LCCIndexConstant .parsingStringToBytesWithType(ci.type, String.valueOf(stopValue)))); System.out.println("coffey lccindex add filter for type: " + ci.type + " [" + startValue + "," + stopValue + "]"); } } scan.setCacheBlocks(false); scan.setFilter(filters); scan.setAttribute(LCCIndexConstant.SCAN_WITH_LCCINDEX, Writables.getBytes(list)); return table.getScanner(scan); }
public ConditionTreeLeafNode(HRegion region, Range range, float maxScale) { this.range = range; this.scale = region.getStore(range.getFamily()).getIndexScanScale(range); // prune large scale node if (this.scale > maxScale) { this.prune = true; } }
public StoreIndexScanner(Store store, List<KeyValueScanner> scanners, KVComparator comparator, IndexKVComparator indexComparator, Range range, Scan scan, Set<ByteArray> joinSet, boolean isAND) throws IOException { // winter scanner is always 1? in my test it is 1 indeed this.store = store; this.joinSet = joinSet; this.isAND = isAND; this.memstoreScanner = scanners; this.comparator = comparator; this.indexComparator = indexComparator; this.range = range; this.isGet = scan.isGetScan(); this.cacheBlocks = scan.getCacheBlocks(); if (isAND) { this.isEmptySet = this.joinSet.isEmpty(); this.indexSet = new HashSet<ByteArray>(10000); } this.startRow = scan.getStartRow(); this.startKV = KeyValue.createFirstOnRow(startRow); this.stopRow = Bytes.compareTo(scan.getStopRow(), HConstants.EMPTY_BYTE_ARRAY) == 0 ? null : scan .getStopRow(); this.stopKV = Bytes.compareTo(scan.getStopRow(), HConstants.EMPTY_BYTE_ARRAY) == 0 ? null : KeyValue .createLastOnRow(scan.getStopRow()); this.stopRowCmpValue = scan.isGetScan() ? -1 : 0; if (range.getStartValue() != null) { switch (range.getStartType()) { case EQUAL: startIKV = IndexKeyValue.createFirstOnQualifier(range.getQualifier(), range.getStartValue()); stopIKV = startIKV; stopIKVCmpValue = -1; break; case GREATER_OR_EQUAL: startIKV = IndexKeyValue.createFirstOnQualifier(range.getQualifier(), range.getStartValue()); stopIKV = null; stopIKVCmpValue = 0; break; case GREATER: startIKV = IndexKeyValue.createLastOnQualifier(range.getQualifier(), range.getStartValue()); stopIKV = null; stopIKVCmpValue = 0; break; default: throw new IOException("Invalid Range:" + range); } } else { startIKV = IndexKeyValue.createFirstOnQualifier(range.getQualifier()); stopIKV = null; } if (range.getStopValue() != null) { switch (range.getStopType()) { case LESS: stopIKV = IndexKeyValue.createFirstOnQualifier(range.getQualifier(), range.getStopValue()); stopIKVCmpValue = 0; break; case LESS_OR_EQUAL: stopIKV = IndexKeyValue.createFirstOnQualifier(range.getQualifier(), range.getStopValue()); stopIKVCmpValue = -1; break; default: throw new IOException("Invalid Range:" + range); } } this.needToRefresh = false; getScanners(); }
public Range getRange() { return this.range; }
public StoreIndexScanner getStoreIndexScanner(Range r, Scan s, Set<ByteArray> joinSet, boolean isAND) throws IOException;
public static void main(String[] args) throws IOException { HTable table = new HTable(HBaseConfiguration.create(), "orders"); double c3_end = 15000.0; String c5_equal = "1-URGENT"; boolean useIndex = true; boolean print = false; int caching = 1000; int runTimes=5; long totalTime=0; for(int i=0;i<runTimes;i++){ Scan scan = new Scan(); FilterList filters = new FilterList(); RangeList list = new RangeList(); if (c5_equal != null) { filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("f"), Bytes.toBytes("c5"), CompareOp.EQUAL, Bytes.toBytes(c5_equal))); list.addRange(new Range(Bytes.toBytes("f:c5"), Bytes.toBytes(c5_equal), CompareOp.EQUAL, null, CompareOp.NO_OP)); } if (c3_end > 0) { filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("f"), Bytes.toBytes("c3"), CompareOp.LESS, Bytes.toBytes(c3_end))); list.addRange(new Range(Bytes.toBytes("f:c3"), null, CompareOp.NO_OP, Bytes.toBytes(c3_end), CompareOp.LESS)); } scan.setFilter(filters); if (useIndex) { // scan.setAttribute(IndexConstants.SCAN_INDEX_RANGE, Writables.getBytes(list)); } scan.setCacheBlocks(false); scan.setCaching(caching); long startTime = System.currentTimeMillis(); ResultScanner scanner = table.getScanner(scan); Result[] result = scanner.next(caching); if (print) { for (Result r : result) { println(r); } } long stopTime = System.currentTimeMillis(); System.out.println("Time elapsed: " + (stopTime - startTime) + " ms, result count: " + result.length); totalTime+=stopTime-startTime; scanner.close(); } System.out.println("Final Time elapsed: " + (totalTime/runTimes) + " ms"); table.close(); }
public static void main(String[] args) throws IOException { HTable table = new HTable(HBaseConfiguration.create(), "orders"); double[] c3_value = // new double[] {1500.0, 3000.0, 7000.0, 15000.0, 25000.0, 50000.0, 60000.0, 70000.0, // 90000.0,120000.0,150000.0,170000.0, 200000.0, 230000.0, 250000.0,500000.0}; // new double[] {1000.0, 1500.0, 2000.0, 3000.0, 5000.0, 7000.0, 10000.0, 13000.0, 15000.0, // 20000.0, 25000.0, 30000.0, 35000.0, 40000.0, 45000.0, 50000.0, 60000.0, 70000.0, // 80000.0,90000.0,120000.0,150000.0,170000.0}; new double[] {90000.0}; // String[] c5_value =new String[]{"1-URGENT","2-HIGH","3-MEDIUM","4-NOT SPECIFIED","5-LOW"}; for (double v : c3_value) { System.out.println(v); double c3_end = v; String c5_equal = "1-URGENT"; // "2-HIGH"; boolean useIndex = true; boolean printProgress = false; boolean print = false; Scan scan = new Scan(); FilterList filters = new FilterList(); RangeList list = new RangeList(); if (c5_equal != null) { filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("f"), Bytes.toBytes("c5"), CompareOp.EQUAL, Bytes.toBytes(c5_equal))); list.addRange(new Range(Bytes.toBytes("f:c5"), Bytes.toBytes(c5_equal), CompareOp.EQUAL, null, CompareOp.NO_OP)); } if (c3_end > 0) { filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("f"), Bytes.toBytes("c3"), CompareOp.LESS, Bytes.toBytes(c3_end))); list.addRange(new Range(Bytes.toBytes("f:c3"), null, CompareOp.NO_OP, Bytes.toBytes(c3_end), CompareOp.LESS)); } scan.setFilter(filters); if (useIndex) { // scan.setAttribute(IndexConstants.SCAN_INDEX_RANGE, Writables.getBytes(list)); } scan.setCacheBlocks(false); scan.setCaching(100000); long startTime = System.currentTimeMillis(); int count = 0; ResultScanner scanner = table.getScanner(scan); Result result = null; while ((result = scanner.next()) != null) { count++; if (print) { println(result); } if (printProgress && (count % 100000 == 0)) { System.out.println("Time elapsed: " + (System.currentTimeMillis() - startTime) + " ms, result count: " + count); } } long stopTime = System.currentTimeMillis(); System.out.println("Time elapsed: " + (stopTime - startTime) + " ms, result count: " + count); } table.close(); }
public void run() { HTable table = null; try { table = new HTable(HBaseConfiguration.create(), "orders"); Scan scan = new Scan(); scan.setStartRow(start); scan.setStopRow(stop); FilterList filters = new FilterList(); RangeList list = new RangeList(); if (c5_equal != null) { filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("f"), Bytes.toBytes("c5"), CompareOp.EQUAL, Bytes.toBytes(c5_equal))); list.addRange(new Range(Bytes.toBytes("f:c5"), Bytes.toBytes(c5_equal), CompareOp.EQUAL, null, CompareOp.NO_OP)); } if (c3_end > 0) { filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes("f"), Bytes.toBytes("c3"), CompareOp.LESS, Bytes.toBytes(c3_end))); list.addRange(new Range(Bytes.toBytes("f:c3"), null, CompareOp.NO_OP, Bytes .toBytes(c3_end), CompareOp.LESS)); } scan.setFilter(filters); if (useIndex) { // scan.setAttribute(IndexConstants.SCAN_INDEX_RANGE, Writables.getBytes(list)); } scan.setCacheBlocks(false); scan.setCaching(caching); long startTime = System.currentTimeMillis(); ResultScanner scanner = table.getScanner(scan); int count = 0; Result[] result = null; while ((result = scanner.next(caching > 10000 ? caching : 10000)) != null && result.length != 0) { addResults(result.length, false); count += result.length; if (print) { for (Result r : result) { println(r); } } } addResults(0, true); long stopTime = System.currentTimeMillis(); System.out.println(this.getName() + " throughput:" + count * 1000.0 / (stopTime - startTime)); scanner.close(); table.close(); } catch (IOException e) { e.printStackTrace(); } }
public static void main(String[] args) throws IOException { Configuration conf = HBaseConfiguration.create(); HBaseAdmin admin = new HBaseAdmin(conf); HTable table = null; // if (admin.tableExists(tableName)) { // indexadmin.disableTable(tableName); // indexadmin.deleteTable(tableName); // } if (!admin.tableExists(tableName)) { HTableDescriptor tableDesc = new HTableDescriptor(tableName); IndexDescriptor index1 = new IndexDescriptor(Bytes.toBytes("c1"), DataType.DOUBLE); IndexColumnDescriptor family = new IndexColumnDescriptor("f"); family.addIndex(index1); tableDesc.addFamily(family); admin.createTable(tableDesc); table = new HTable(conf, tableName); Put put = new Put(Bytes.toBytes("001")); put.add(Bytes.toBytes("f"), Bytes.toBytes("c1"), 10, Bytes.toBytes(120.0)); put.add(Bytes.toBytes("f"), Bytes.toBytes("c2"), 10, Bytes.toBytes("a")); table.put(put); put = new Put(Bytes.toBytes("001")); put.add(Bytes.toBytes("f"), Bytes.toBytes("c1"), 20, Bytes.toBytes(90.0)); put.add(Bytes.toBytes("f"), Bytes.toBytes("c2"), 20, Bytes.toBytes("a")); table.put(put); put = new Put(Bytes.toBytes("002")); put.add(Bytes.toBytes("f"), Bytes.toBytes("c1"), 30, Bytes.toBytes(150.0)); put.add(Bytes.toBytes("f"), Bytes.toBytes("c2"), 30, Bytes.toBytes("a")); table.put(put); } admin.close(); if (table == null) { table = new HTable(conf, tableName); } Scan scan = new Scan(); RangeList list = new RangeList(); list.addRange(new Range(Bytes.toBytes("f:c1"), Bytes.toBytes(100.0), CompareOp.GREATER, null, CompareOp.NO_OP)); SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("f"), Bytes.toBytes("c1"), CompareOp.GREATER, Bytes.toBytes(100.0)); filter.setLatestVersionOnly(false); scan.setFilter(filter); // scan.setAttribute(IndexConstants.SCAN_INDEX_RANGE, Writables.getBytes(list)); scan.setMaxVersions(); Result result = null; ResultScanner rs = table.getScanner(scan); while ((result = rs.next()) != null) { System.out.println(result); } rs.close(); table.close(); }
private boolean mWinterFilterResults(List<KeyValue> results) throws IOException { // default the results are in the same kv! byte[] rowkey = null; String strRowkey = null; for (KeyValue kv : results) { if (rowkey == null) { rowkey = kv.getRow(); strRowkey = Bytes.toString(rowkey); } else if (!Bytes.equals(rowkey, kv.getRow())) { System.out.println("winter error because of different rowkey in same results: " + LCCIndexConstant.mWinterToPrint(kv)); WinterOptimizer .ThrowWhenCalled("winter error because of different rowkey in same results: " + LCCIndexConstant.mWinterToPrint(kv)); } if (strRowkey.split(LCCIndexConstant.DELIMITER_STR).length < 3) { System.out.println("winter error in range of rowkey: " + strRowkey); WinterOptimizer.ThrowWhenCalled("winter error in range of rowkey: " + LCCIndexConstant.mWinterToPrint(kv)); } else if (!Bytes.toString(lccMainQualifier).equalsIgnoreCase( strRowkey.split(LCCIndexConstant.DELIMITER_STR)[1])) { System.out.println("winter discard A, say qualifier difference"); WinterOptimizer.ThrowWhenCalled("winter discard A, say qualifier difference: " + LCCIndexConstant.mWinterToPrint(kv)); return true; } // Bytes.toBytes("") // winter here use main filter to filter the range as well! if (mWinterShouldDiscardRow(lccMainRowkeyRange, kv.getRow())) { System.out.println("winter discard B, say rowkey out of range: " + LCCIndexConstant.mWinterToPrint(kv)); WinterOptimizer.ThrowWhenCalled("winter discard B, say rowkey out of range: " + LCCIndexConstant.mWinterToPrint(kv)); return true; } for (Range r : lccFilterRanges) { if (mWinterShouldDiscard(r, kv)) { return true; } } } return false; }
public TPCHScanHBase(String confPath, String newAddedFile, String tableName, List<Range> ranges) throws IOException { super(confPath, newAddedFile, tableName, ranges); }