public void restartScan() { if (finished == false && scanning == false) { LOG.debug("Try to restart " + this.getName() + "-" + getId() + " for the " + indexResultScanner.restartTimes + "th times"); scan.setStartRow(Bytes.add(currentStartKey, IndexConstants.MIN_ROW_KEY)); try { resultScanner = indexTable.getScanner(scan); scanning = true; } catch (IOException e) { LOG.error("Try to restart " + this.getName() + "-" + getId() + " failed!" + e.toString()); } } else { LOG.warn( this.getName() + "-" + getId() + " restart is ignored, because finished=" + finished + ",scanning=" + scanning); } }
private void doIndexPut(final List<IndexPut> puts) throws IOException { Table temptable = null; Table tempCCT = null; for (IndexPut put : puts) { for (Map.Entry<byte[], Put> entry : put.getPuts().entrySet()) { temptable = indexTableMaps.get(entry.getKey()); temptable.put(entry.getValue()); Put cctPut = IndexUtils.parseCCTPut(indexDesc, entry.getValue()); if (cctPut != null) { System.out.println( "winter index table name: " + Bytes.toString(entry.getKey()) + ", values: " + entry .getValue() + ", cct value: " + cctPut); tempCCT = cctTableMaps.get(Bytes.add(entry.getKey(), IndexConstants.CCT_FIX)); tempCCT.put(cctPut); } // something to do here } } }
/** * Creates a new table with indexes defined by IndexDescriptor. * * @param indexDesc table descriptor for table * @throws IOException * @throws IndexExistedException */ public void createTable(IndexTableDescriptor indexDesc) throws IOException, IndexExistedException { HTableDescriptor descriptor = new HTableDescriptor(indexDesc.getTableDescriptor()); descriptor.remove(IndexConstants.INDEX_KEY); admin.createTable(descriptor, indexDesc.getSplitKeys()); admin.disableTable(descriptor.getTableName()); if (indexDesc.hasIndex()) { // corresponding cct if (indexDesc.getIndexSpecifications()[0].getIndexType() == IndexType.CCIndex) { System.out.println("winter new cct of main table: " + Bytes.toString(Bytes .add(indexDesc.getTableDescriptor().getTableName().getName(), IndexConstants.CCT_FIX))); HTableDescriptor cctDesc = new HTableDescriptor(TableName.valueOf(Bytes .add(indexDesc.getTableDescriptor().getTableName().getName(), IndexConstants.CCT_FIX))); for (HColumnDescriptor f : descriptor.getFamilies()) { cctDesc.addFamily(f); } admin.createTable(cctDesc, indexDesc.getSplitKeys()); } this.addIndexes(indexDesc.getTableDescriptor().getTableName(), indexDesc.getIndexSpecifications()); } enableTable(descriptor.getTableName()); }
/** * list all tables, including tables with or without indexes. * * @return an array of {@link HTableDescriptor} * @throws IOException */ public HTableDescriptor[] listTables() throws IOException { ArrayList<HTableDescriptor> descList = new ArrayList<HTableDescriptor>(); HTableDescriptor[] tableDescriptor = admin.listTables(); if (tableDescriptor != null && tableDescriptor.length != 0) { for (HTableDescriptor desc : tableDescriptor) { byte[] indexType = desc.getValue(IndexConstants.INDEX_TYPE); // table without any index or main data table if (indexType == null) { descList.add(desc); } } } return descList.toArray(new HTableDescriptor[0]); }
/** * Write IndexDescription to base table Description. * * @throws IOException */ private void writeToTable() throws IOException { if (!this.indexSpecifications.isEmpty()) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(baos); IndexSpecificationArray indexArray = new IndexSpecificationArray( indexSpecifications.values().toArray(new IndexSpecification[0])); indexArray.write(dos); dos.flush(); descriptor.setValue(IndexConstants.INDEX_KEY, baos.toByteArray()); descriptor.setValue(IndexConstants.BASE_KEY, IndexConstants.BASE_KEY); descriptor.setValue(IndexConstants.KEYGEN, Bytes.toBytes(keygen.getName())); } else { if (descriptor.getValue(IndexConstants.INDEX_KEY) != null) { descriptor.remove(IndexConstants.INDEX_KEY); } if (descriptor.getValue(IndexConstants.BASE_KEY) != null) { descriptor.remove(IndexConstants.BASE_KEY); } if (descriptor.getValue(IndexConstants.KEYGEN) != null) { descriptor.remove(IndexConstants.KEYGEN); } } }
public HTableDescriptor createIndexTableDescriptor(byte[] indexColumn) throws IndexNotExistedException { IndexSpecification indexSpec = this.getIndexSpecification(indexColumn); HTableDescriptor indexTableDescriptor = new HTableDescriptor(indexSpec.getIndexTableName()); if (indexSpec.getIndexType() == IndexType.CCIndex) { for (HColumnDescriptor desc : this.descriptor.getFamilies()) { indexTableDescriptor.addFamily(desc); } } else if (indexSpec.getIndexType() == IndexType.UDGIndex) { Set<byte[]> family = indexSpec.getAdditionMap().keySet(); if (family.size() != 0) { for (byte[] name : family) { indexTableDescriptor.addFamily(this.descriptor.getFamily(name)); } } else { indexTableDescriptor.addFamily(this.descriptor.getFamily(indexSpec.getFamily())); } } else if (indexSpec.getIndexType() == IndexType.GSIndex) { indexTableDescriptor.addFamily(this.descriptor.getFamily(indexSpec.getFamily())); } indexTableDescriptor.setValue(IndexConstants.INDEX_TYPE, Bytes.toBytes(indexSpec.getIndexType().toString())); // record the index type return indexTableDescriptor; }
private ResultScanner getNextScanner() throws IOException { if (INIT_REGION_SIZE != getRegionNumber()) { throw new IOException( "region number changed from " + INIT_REGION_SIZE + " to " + getRegionNumber()); } if (regionLocationQueue.isEmpty()) return null; HRegionLocation regionLocation = regionLocationQueue.poll(); Scan newScan = new Scan(rawScan); byte[] key = regionLocation.getRegionInfo().getStartKey(); if (key != null && key.length > 0) newScan.setStartRow(key); key = regionLocation.getRegionInfo().getEndKey(); if (key != null && key.length > 0) newScan.setStopRow(key); newScan.setAttribute(IndexConstants.SCAN_WITH_INDEX, Bytes.toBytes("Hi")); newScan.setId(rawScan.getId()); newScan.setCacheBlocks(rawScan.getCacheBlocks()); newScan.setCaching(rawScan.getCaching()); return table.getScanner(newScan); }
private void innerAddScanner(HRegionLocation regionLocation) throws IOException { if (INIT_REGION_SIZE != getRegionNumber()) { throw new IOException( "region number changed from " + INIT_REGION_SIZE + " to " + getRegionNumber()); } Scan newScan = new Scan(rawScan); if (regionLocation.getRegionInfo().getStartKey() != null) newScan.setStartRow(regionLocation.getRegionInfo().getStartKey()); if (regionLocation.getRegionInfo().getEndKey() != null) newScan.setStopRow(regionLocation.getRegionInfo().getEndKey()); newScan.setAttribute(IndexConstants.SCAN_WITH_INDEX, Bytes.toBytes("Hi")); newScan.setFilter(rangeList.toFilterList()); newScan.setAttribute(IndexConstants.MAX_SCAN_SCALE, Bytes.toBytes(1.0f)); ResultScanner scanner = table.getScanner(newScan); synchronized (scannerList) { scannerList.add(scanner); } runningGet.decrementAndGet(); }
private ResultScanner getNextScanner() throws IOException { if (INIT_REGION_SIZE != getRegionNumber()) { throw new IOException( "region number changed from " + INIT_REGION_SIZE + " to " + getRegionNumber()); } if (regionLocationQueue.isEmpty()) return null; HRegionLocation regionLocation = regionLocationQueue.poll(); Scan newScan = new Scan(rawScan); if (regionLocation.getRegionInfo().getStartKey() != null) newScan.setStartRow(regionLocation.getRegionInfo().getStartKey()); if (regionLocation.getRegionInfo().getEndKey() != null) newScan.setStopRow(regionLocation.getRegionInfo().getEndKey()); newScan.setAttribute(IndexConstants.SCAN_WITH_INDEX, Bytes.toBytes("Hi")); newScan.setFilter(rangeList.toFilterList()); newScan.setAttribute(IndexConstants.MAX_SCAN_SCALE, Bytes.toBytes(1.0f)); newScan.setId(rawScan.getId()); newScan.setCacheBlocks(rawScan.getCacheBlocks()); newScan.setCaching(rawScan.getCaching()); return table.getScanner(newScan); }
public FileableIndexPath(FileSystem fs, Collection<StoreFile> compactedFiles) throws IOException { super(fs); if (compactedFiles != null && !compactedFiles.isEmpty()) { this.indexPath = new Path(compactedFiles.iterator().next().getPath().getParent(), IndexConstants.REGION_INDEX_DIR_NAME); for (StoreFile sf : compactedFiles) { Path indexfile = new Path(indexPath, sf.getPath().getName()); if (fs.exists(indexfile)) { this.compactedIndexFiles.add(indexfile); } } } else { indexPath = null; } }
@Override public ResultScanner getScanner() throws IOException { Scan scan = new Scan(); RangeList list = new RangeList(); FilterList filters = new FilterList(); for (Range r : ranges) { list.addRange(r); if (r.getStartValue() != null) { filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes(TPCHConstants.FAMILY_NAME), r .getQualifier(), r.getStartType(), r.getStartValue())); } if (r.getStopValue() != null) { filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes(TPCHConstants.FAMILY_NAME), r .getQualifier(), r.getStopType(), r.getStopValue())); } } scan.setFilter(filters); scan.setAttribute(IndexConstants.SCAN_WITH_INDEX, Writables.getBytes(list)); scan.setAttribute(IndexConstants.MAX_SCAN_SCALE, Bytes.toBytes(0.3)); scan.setCacheBlocks(false); return table.getScanner(scan); }
/** * Returns the store files available for the family. This methods performs the filtering based on * the valid store files. * * @param familyName Column Family Name * @return a set of {@link StoreFileInfo} for the specified family. */ public Collection<StoreFileInfo> getStoreFiles(final String familyName, final boolean validate) throws IOException { Path familyDir = getStoreDir(familyName); // FileStatus[] files = FSUtils.listStatus(this.fs, familyDir); FileStatus[] files = FSUtils.listStatus(this.fs, familyDir, new PathFilter() { @Override public boolean accept(Path path) { String name = path.getName(); if (name.endsWith(IndexConstants.REGION_INDEX_DIR_NAME) || name .endsWith(LMDIndexConstants.BUCKET_FILE_SUFFIX) || name .endsWith(LMDIndexConstants.DATA_FILE_SUFFIX)) return false; return true; } }); if (files == null) { LOG.debug("No StoreFiles for: " + familyDir); return null; } ArrayList<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>(files.length); for (FileStatus status : files) { if (validate && !StoreFileInfo.isValid(status)) { LOG.warn("Invalid StoreFile: " + status.getPath()); continue; } StoreFileInfo info = ServerRegionReplicaUtil .getStoreFileInfo(conf, fs, regionInfo, regionInfoForFs, familyName, status.getPath()); storeFiles.add(info); } return storeFiles; }
/** * @param familyName * @param buildPath * @param seqNum * @param generateNewName * @param store * @param dueToFlush used for LCIndex, tell it don't use commit job for new compacted files * @return * @throws IOException */ private Path commitStoreFile(final String familyName, final Path buildPath, final long seqNum, final boolean generateNewName, final HStore store, boolean dueToFlush) throws IOException { Path storeDir = getStoreDir(familyName); if (!fs.exists(storeDir) && !createDir(storeDir)) throw new IOException("Failed creating " + storeDir); String name = buildPath.getName(); if (generateNewName) { name = generateUniqueName((seqNum < 0) ? null : "_SeqId_" + seqNum + "_"); } Path dstPath = new Path(storeDir, name); if (store.getIndexType() == IndexType.IRIndex) { String irName = name + IndexConstants.REGION_INDEX_DIR_NAME; Path irSrc = new Path(buildPath.getParent(), irName); Path irDst = new Path(storeDir, irName); LOG.debug("IRINFO: committing IRIndex store file from " + irSrc + " to " + irDst); fs.rename(irSrc, irDst); } else if (store.getIndexType() == IndexType.LCIndex && dueToFlush) { CommitJobQueue.CommitJob commitJob = new CommitJobQueue.CommitJob(store, buildPath, dstPath); CommitJobQueue.getInstance().addJob(commitJob); } else if (store.getIndexType() == IndexType.LMDIndex_D || store.getIndexType() == IndexType.LMDIndex_S) { Path lmdDataSrc = store.getLMDIndexParameters().getTmpSecondaryFilePath(buildPath); Path lmdDataDst = store.getLMDIndexParameters().getTmpSecondaryFilePath(dstPath); Path lmdBucketSrc = store.getLMDIndexParameters().getTmpBucketFilePath(buildPath); Path lmdBucketDst = store.getLMDIndexParameters().getTmpBucketFilePath(dstPath); LOG.info(String .format("LMDINFO: committing LMDIndex, data files [%s==>%s], bucket file[%s==>%s]", lmdDataSrc, lmdDataDst, lmdBucketSrc, lmdBucketDst)); fs.rename(lmdDataSrc, lmdDataDst); fs.rename(lmdBucketSrc, lmdBucketDst); } if (!fs.exists(buildPath)) { throw new FileNotFoundException(buildPath.toString()); } LOG.debug("Committing store file " + buildPath + " as " + dstPath); // buildPath exists, therefore not doing an exists() check. /** * /hbase/data/default/t1/AAA/.tmp/bbb -->> /hbase/data/default/t1/AAA/c1/bbb */ if (!rename(buildPath, dstPath)) { throw new IOException("Failed rename of " + buildPath + " to " + dstPath); } return dstPath; }
private Queue<byte[]> generateLMDIndexCandidateRows(Scan scan) throws IOException { List<byte[]> list = new ArrayList<>(); for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) { // memstore HStore store = (HStore) stores.get(entry.getKey()); InternalScan internalScan = new InternalScan(scan); internalScan.setCacheBlocks(false); internalScan.setAttribute(IndexConstants.SCAN_WITH_INDEX, null); internalScan.checkOnlyMemStore(); RegionScanner memScanner = store.getHRegion().new RegionScannerImpl(internalScan, null, store.getHRegion()); // store files Collection<StoreFile> storeFiles = store.getStoreFilesToScan(scan); List<LMDIndexDirectStoreFileScanner> scanners = new ArrayList<>(storeFiles.size()); for (StoreFile sf : storeFiles) { System.out.println("viewing StoreFile :" + sf); scanners.add( new LMDIndexDirectStoreFileScanner(sf, false, false, false, false, null, readPt, true, indexTableRelation.getIndexFamilyMap(), ScanRange.ScanRangeList.getScanRangeList(scan), store.getFileSystem(), store.cacheConf, conf, true)); } LMDIndexSecondaryStoreScanner secondary = new LMDIndexSecondaryStoreScanner(memScanner, scanners); list.addAll(secondary.getRowkeys()); memScanner.close(); } Collections.sort(list, Bytes.BYTES_COMPARATOR); Queue<byte[]> ret = new LinkedList<>(list); return ret; }
private LCIndexMemStoreScanner2 getMemstoreScanner(NavigableSet<byte[]> columns, long readPt) throws IOException { InternalScan internalScan = new InternalScan(createScanOnMemstore(rawScan, rangeList)); internalScan.setCacheBlocks(false); internalScan.setAttribute(IndexConstants.SCAN_WITH_INDEX, null); internalScan.checkOnlyMemStore(); RegionScanner scanner = store.getHRegion().new RegionScannerImpl(internalScan, null, store.getHRegion()); return new LCIndexMemStoreScanner2(scanner, indexTableRelation, primaryRange); }
public Path getIndexFilePathInLocalTmp() throws IOException { Path indexCompactionPath = new Path(conf.get(IndexConstants.INDEX_COMPACTION_LOCAL_DIR)); if (!this.irLocalFS.exists(indexCompactionPath)) { this.irLocalFS.mkdirs(indexCompactionPath); } return new Path(StoreFile.getUniqueFile(irLocalFS, indexCompactionPath) + IndexConstants.REGION_INDEX_DIR_NAME); }
public byte[] parseIndexRowKey(byte[] indexKey) { int length = Integer.valueOf(Bytes.toString(Bytes.tail(indexKey, IndexConstants.LASTPART_LENGTH))) .intValue(); // get row key of main data table byte[] result = new byte[indexKey.length - IndexConstants.LASTPART_LENGTH - length]; for (int i = 0; i < result.length; i++) { result[i] = indexKey[i + length]; } return result; }
/** * Check if it is an index table. * * @param desc * @throws IllegalArgumentException-desc is null */ private boolean isIndexTable(HTableDescriptor desc) throws IOException { if (desc == null) { throw new IllegalArgumentException("Table Descriptor is empty"); } byte[] value = desc.getValue(IndexConstants.INDEX_TYPE); return (value != null) ? true : false; }
private void parseQuery() throws Exception { //TODO uncomment this in new version // if(columnInfo==null ||columnInfo.size()==0){ // throw new IllegalArgumentException("Column Info is not set! Please specify column info before query!"); // } //optimize the query condition if (queryCondition != null) { // ranges = scanOptimize.optimizeQuery(queryCondition, columnInfoMap); } else { ranges = new Range[1][1]; ranges[0][0] = new Range(IndexConstants.KEY, this.getTableName().getName()); ranges[0][0].setStartType(CompareOp.GREATER_OR_EQUAL); if (this.startKey != null) { ranges[0][0].setStartValue(this.startKey); } else { ranges[0][0].setStartValue(HConstants.EMPTY_BYTE_ARRAY); } if (this.endKey != null) { ranges[0][0].setEndType(CompareOp.LESS_OR_EQUAL); ranges[0][0].setEndValue(endKey); } } // ranges = new Range[range.length - 1][]; // // // set base columns of every query range // for (int i = 0; i < range.length - 1; i++) { // ranges[i] = new Range[range[i].length]; // for (int j = 0; j < range[i].length; j++) { // ranges[i][j] = range[i][j]; // } // } }
@Override public byte[] createIndexRowKey(byte[] rowKey, byte[] value) { if (value == null || value.length == 0) { return null; } value = Bytes.add(value, IndexConstants.MIN_ROW_KEY); byte[] a = Bytes.toBytes(IndexConstants.LASTPART_ZERO .substring(0, IndexConstants.LASTPART_LENGTH - ("" + value.length).length()) + value.length); return Bytes.add(value, rowKey, a); }
@Override public byte[][] parseIndexRowKey(byte[] indexKey) { int length = Integer.valueOf(Bytes.toString(Bytes.tail(indexKey, IndexConstants.LASTPART_LENGTH))) .intValue(); byte[][] result = new byte[2][]; // get row key of main data table result[0] = new byte[indexKey.length - IndexConstants.LASTPART_LENGTH - length]; for (int i = 0; i < result[0].length; i++) { result[0][i] = indexKey[i + length]; } // get index column value result[1] = Bytes.head(indexKey, length - 1); return result; }
/** * Set key generator class name. * * @param className * @throws ClassNotFoundException */ public void setKeygenClass(String className) throws ClassNotFoundException { Class<?> tempkeygen = null; if (className.contains(".")) { tempkeygen = Class.forName(className); } else { tempkeygen = Class.forName(this.getClass().getPackage().getName() + "." + className); } if (!IndexKeyGenerator.class.isAssignableFrom(tempkeygen)) { throw new IllegalArgumentException( tempkeygen.getName() + "doesn't implement interface IndexKeyGenerator!"); } keygen = tempkeygen.asSubclass(IndexKeyGenerator.class); descriptor.setValue(IndexConstants.KEYGEN, Bytes.toBytes(keygen.getName())); }
/** * Read IndexDescription from existed base table Description. * * @throws IOException */ private void readFromTable() throws IOException { byte[] bytes = descriptor.getValue(IndexConstants.INDEX_KEY); if (bytes == null) { return; } ByteArrayInputStream bais = new ByteArrayInputStream(bytes); DataInputStream dis = new DataInputStream(bais); IndexSpecificationArray indexArray = new IndexSpecificationArray(); indexArray.readFields(dis); for (IndexSpecification indexSpec : indexArray.getIndexSpecifications()) { checkNewIndex(indexSpec); indexSpecifications.put(indexSpec.getIndexColumn(), indexSpec); } this.update(); byte[] kg = descriptor.getValue(IndexConstants.KEYGEN); if (kg != null && kg.length != 0) { try { Class<?> tempkeygen = Class.forName(Bytes.toString(kg)); if (!IndexKeyGenerator.class.isAssignableFrom(tempkeygen)) { throw new IllegalArgumentException( tempkeygen.getName() + "doesn't implement interface IndexKeyGenerator!"); } keygen = tempkeygen.asSubclass(IndexKeyGenerator.class); } catch (ClassNotFoundException e) { throw new IOException(e.getMessage()); } } }
protected HTableDescriptor createCCTTableDescriptor(byte[] indexColumn) throws IndexNotExistedException { IndexSpecification indexSpec = this.getIndexSpecification(indexColumn); HTableDescriptor indexTableDescriptor = new HTableDescriptor(IndexUtils.getCCTName(indexSpec.getTableName())); System.out.println("winter new cct table name: " + indexTableDescriptor.getTableName()); if (indexSpec.getIndexType() == IndexType.CCIndex) { for (HColumnDescriptor desc : this.descriptor.getFamilies()) { // column is f, the only family indexTableDescriptor.addFamily(desc); } } else if (indexSpec.getIndexType() == IndexType.UDGIndex) { Set<byte[]> family = indexSpec.getAdditionMap().keySet(); if (family.size() != 0) { for (byte[] name : family) { indexTableDescriptor.addFamily(this.descriptor.getFamily(name)); } } else { indexTableDescriptor.addFamily(this.descriptor.getFamily(indexSpec.getFamily())); } } else if (indexSpec.getIndexType() == IndexType.GSIndex) { indexTableDescriptor.addFamily(this.descriptor.getFamily(indexSpec.getFamily())); } indexTableDescriptor.setValue(IndexConstants.INDEX_TYPE, Bytes.toBytes(indexSpec.getIndexType().toString())); // record the index type return indexTableDescriptor; }
private void innerAddScanner(HRegionLocation regionLocation) throws IOException { Scan newScan = new Scan(rawScan); if (regionLocation.getRegionInfo().getStartKey() != null) newScan.setStartRow(regionLocation.getRegionInfo().getStartKey()); if (regionLocation.getRegionInfo().getEndKey() != null) newScan.setStopRow(regionLocation.getRegionInfo().getEndKey()); newScan.setAttribute(IndexConstants.SCAN_WITH_INDEX, Bytes.toBytes("Hi")); ResultScanner scanner = table.getScanner(newScan); synchronized (scannerList) { scannerList.add(scanner); } }
/** * Walk the given directory for all HFiles, and return a Queue * containing all such files. */ private void discoverLoadQueue(Deque<LoadQueueItem> ret, Path hfofDir) throws IOException { FileSystem fs = hfofDir.getFileSystem(getConf()); if (!fs.exists(hfofDir)) { throw new FileNotFoundException("HFileOutputFormat dir " + hfofDir + " not found"); } FileStatus[] familyDirStatuses = fs.listStatus(hfofDir); if (familyDirStatuses == null) { throw new FileNotFoundException("No families found in " + hfofDir); } for (FileStatus stat : familyDirStatuses) { if (!stat.isDir()) { LOG.warn("Skipping non-directory " + stat.getPath()); continue; } Path familyDir = stat.getPath(); // Skip _logs, etc if (familyDir.getName().startsWith("_")) continue; byte[] family = familyDir.getName().getBytes(); Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir)); for (Path hfile : hfiles) { if (hfile.getName().startsWith("_") || hfile.getName().endsWith(IndexConstants.REGION_INDEX_DIR_NAME) ) continue; ret.add(new LoadQueueItem(family, hfile)); } } }
/** * @param name file name to check. * @return True if the path has format of a HStoreFile reference. */ public static boolean isReference(final String name) { if (name.equals(IndexConstants.REGION_INDEX_DIR_NAME)) return false; if (name.equals(LCCIndexConstant.INDEX_DIR_NAME)) return false; Matcher m = REF_NAME_PATTERN.matcher(name); return m.matches() && m.groupCount() > 1; }
Path getIndexFilePathInLocalTmp() throws IOException { Path indexCompactionPath = new Path(conf.get(IndexConstants.INDEX_COMPACTION_LOCAL_DIR)); if (!this.localfs.exists(indexCompactionPath)) { this.localfs.mkdirs(indexCompactionPath); } return new Path(StoreFile.getUniqueFile(localfs, indexCompactionPath) + IndexConstants.REGION_INDEX_DIR_NAME); }
@Override public ResultScanner getScanner() throws IOException { Scan scan = new Scan(); RangeList list = new RangeList(); List<CF_INFO> cfs = PutTestConstants.getCFInfo(); FilterList filters = new FilterList(); for (CF_INFO ci : cfs) { if (ci.isIndex) { list.addRange(new Range(Bytes.toBytes(FAMILY_NAME + ":" + ci.qualifier), LCCIndexConstant .parsingStringToBytesWithType(ci.type, String.valueOf(startValue)), CompareOp.GREATER_OR_EQUAL, LCCIndexConstant.parsingStringToBytesWithType(ci.type, String.valueOf(stopValue)), CompareOp.LESS_OR_EQUAL)); filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes(FAMILY_NAME), Bytes .toBytes(ci.qualifier), CompareOp.GREATER_OR_EQUAL, LCCIndexConstant .parsingStringToBytesWithType(ci.type, String.valueOf(startValue)))); filters.addFilter(new SingleColumnValueFilter(Bytes.toBytes(FAMILY_NAME), Bytes .toBytes(ci.qualifier), CompareOp.LESS_OR_EQUAL, LCCIndexConstant .parsingStringToBytesWithType(ci.type, String.valueOf(stopValue)))); System.out.println("coffey irindex add filter for type: " + ci.type + " [" + startValue + "," + stopValue + "]"); } } scan.setFilter(filters); scan.setAttribute(IndexConstants.SCAN_WITH_INDEX, Writables.getBytes(list)); scan.setAttribute(IndexConstants.MAX_SCAN_SCALE, Bytes.toBytes(0.3)); scan.setCacheBlocks(false); return table.getScanner(scan); }
Path getIndexFilePathInLocalTmp() throws IOException { Path indexCompactionPath = new Path(conf.get(IndexConstants.INDEX_COMPACTION_LOCAL_DIR)); if(!this.localfs.exists(indexCompactionPath)){ this.localfs.mkdirs(indexCompactionPath); } return new Path(StoreFile.getUniqueFile(localfs, indexCompactionPath) + IndexConstants.REGION_INDEX_DIR_NAME); }
public static Path getIRIndexPathFromPath(Path hfilePath) { return new Path(hfilePath + IndexConstants.REGION_INDEX_DIR_NAME); }
public Path getIndexFilePathFromHFilePathInTmp(Path hfilePath) throws IOException { return new Path((hfilePath != null ? hfilePath.toString() : StoreFile.getUniqueFile(fs.getFileSystem(), fs.getTempDir()).toString()) + IndexConstants.REGION_INDEX_DIR_NAME); }
public IndexSingleScanner(Scan scan, Range[] range, int flag, byte[][] resultColumns, Connection conn, Table table, IndexKeyGenerator keyGen, IndexSpecification indexSpec, boolean containAll, TableName mainTableName, FilterList list, int maxGets) throws IOException { super(scan, resultColumns, table); this.range = range; this.flag = flag; this.kegGen = keyGen; this.indexColumn = KeyValue.parseColumn(range[flag].getColumn()); for (int i = 0; i < range.length; i++) { if (Bytes.compareTo(range[i].getColumn(), IndexConstants.KEY) == 0) { existKey = true; keyflag = i; break; } } this.containAll = containAll; this.ftlist = list; this.maxGetsPerScanner = maxGets; if (containAll == false) { threadPool = new GetThread[maxGetsPerScanner]; poolCounter = 0; resultPool = new ArrayList<Result>(); tables = new ArrayList<Table>(maxGetsPerScanner); for (int i = 0; i < maxGetsPerScanner; i++) { tables.add(conn.getTable(mainTableName)); } } else { if (indexSpec.getIndexType() == IndexType.GSIndex || ( indexSpec.getIndexType() == IndexType.UDGIndex && indexSpec.getAdditionMap().size() == 0) || (resultColumns != null && resultColumns.length == 1 && flag >= 0 && Bytes.compareTo(range[flag].getColumn(), resultColumns[0]) == 0)) { omitkv = true; } else { omitkv = false; } } }
/** * Construct with given configuration. * * @param conf * @param tableName * @throws IOException */ public IndexTable(final Connection conn, final TableName tableName) throws IOException { this.conn = conn; this.conf = conn.getConfiguration(); this.tableName = tableName; this.writeBufferSize = conf.getLong("hbase.client.write.buffer", 2097152); this.autoFlush = true; this.scannerCaching = conf.getInt("hbase.client.scanner.caching", 1000); this.mainTable = conn.getTable(tableName); this.mainCCTTable = conn.getTable(TableName.valueOf(Bytes.add(tableName.getName(), IndexConstants.CCT_FIX))); this.indexDesc = new IndexTableDescriptor(mainTable.getTableDescriptor()); this.indexTableMaps = new TreeMap<byte[], Table>(Bytes.BYTES_COMPARATOR); this.cctTableMaps = new TreeMap<byte[], Table>(Bytes.BYTES_COMPARATOR); indexTableMaps.put(IndexConstants.KEY, mainTable); cctTableMaps.put(Bytes.add(IndexConstants.KEY, IndexConstants.CCT_FIX), mainCCTTable); if (indexDesc.getIndexedColumns() != null && indexDesc.getIndexedColumns().length != 0) { for (IndexSpecification spec : indexDesc.getIndexSpecifications()) { indexTableMaps.put(spec.getIndexColumn(), new HTable(conf, spec.getIndexTableName())); cctTableMaps.put(Bytes.add(spec.getIndexColumn(), IndexConstants.CCT_FIX), new HTable(conf, Bytes.add(spec.getIndexTableName().getName(), IndexConstants.CCT_FIX))); } } String tempInfo = mainTable.getTableDescriptor().getValue("DATA_FORMAT"); if (tempInfo != null) { this.columnTypeMap = new TreeMap<byte[], DataType>(Bytes.BYTES_COMPARATOR); String[] temp = tempInfo.split(","); for (int i = 0; i < temp.length; i++) { int loc = temp[i].lastIndexOf(':'); if (loc != -1) { this.columnTypeMap.put(Bytes.toBytes(temp[i].substring(0, loc)), DataType.valueOf(temp[i].substring(loc + 1))); } else { LOG.warn("Failed to read column type!" + temp[i]); } } } this.resultBufferSize = DEFAULT_RESULT_BUFFER_SIZE; this.loadFactor = DEFAULT_LOAD_FACTOR; this.maxScanners = DEFAULT_MAX_SCANNERS; this.maxGetsPerScanner = DEFAULT_MAX_GETS_PER_SCANNER; this.chooser = new SimpleIndexChooser(this); }
public static void main(String[] args) throws IOException { HTable table = new HTable(HBaseConfiguration.create(), "orders"); Scan scan = new Scan(); FilterList f1 = new FilterList(Operator.MUST_PASS_ALL); f1.addFilter(new SingleColumnValueFilter(Bytes.toBytes("f"), Bytes.toBytes("c3"), CompareOp.LESS, Bytes.toBytes(3000.0))); f1.addFilter(new SingleColumnValueFilter(Bytes.toBytes("f"), Bytes.toBytes("c5"), CompareOp.EQUAL, Bytes.toBytes("1-URGENT"))); scan.setFilter(f1); scan.setAttribute(IndexConstants.SCAN_WITH_INDEX, Bytes.toBytes(true)); scan.setAttribute(IndexConstants.MAX_SCAN_SCALE, Bytes.toBytes(0.5f)); scan.setCacheBlocks(false); scan.setCaching(100000); // scan.setStopRow(Bytes.toBytes("13")); ResultScanner scanner=table.getScanner(scan); Result result = null; long startTime = System.currentTimeMillis(); int count = 0; boolean print=false; while ((result = scanner.next()) != null) { count++; if (print) { println(result); } if (count % 100000 == 0) { System.out.println("Time elapsed: " + (System.currentTimeMillis() - startTime) + " ms, result count: " + count); } } long stopTime = System.currentTimeMillis(); System.out.println("Time elapsed: " + (stopTime - startTime) + " ms, result count: " + count); table.close(); }
public static void main(String[] args) throws IOException { HTable table = new HTable(HBaseConfiguration.create(), "izp30e"); Scan scan = new Scan(); FilterList f1 = new FilterList(Operator.MUST_PASS_ALL); f1.addFilter(new SingleColumnValueFilter(Bytes.toBytes("f"), Bytes.toBytes("h"), CompareOp.EQUAL, Bytes.toBytes("www.pqai.com"))); f1.addFilter(new SingleColumnValueFilter(Bytes.toBytes("f"), Bytes.toBytes("y"), CompareOp.EQUAL, Bytes.toBytes("C8"))); scan.setFilter(f1); scan.setAttribute(IndexConstants.SCAN_WITH_INDEX, Bytes.toBytes(true)); scan.setAttribute(IndexConstants.MAX_SCAN_SCALE, Bytes.toBytes(0.3f)); scan.setCacheBlocks(false); scan.setCaching(100000); // scan.setStopRow(Bytes.toBytes("13")); ResultScanner scanner=table.getScanner(scan); Result result = null; long startTime = System.currentTimeMillis(); int count = 0; boolean print=true; while ((result = scanner.next()) != null) { count++; if (print) { println(result); } if (count % 100000 == 0) { System.out.println("Time elapsed: " + (System.currentTimeMillis() - startTime) + " ms, result count: " + count); } } long stopTime = System.currentTimeMillis(); System.out.println("Time elapsed: " + (stopTime - startTime) + " ms, result count: " + count); table.close(); }
Path getIndexPathFromPath(Path hfilePath) { return new Path(new Path(hfilePath.getParent(), IndexConstants.REGION_INDEX_DIR_NAME), hfilePath.getName()); }
public static Path getIndexFilePathFromHFilePath(Path hfilePath) throws IOException { return new Path(hfilePath.toString() + IndexConstants.REGION_INDEX_DIR_NAME); }