@Override protected void checkTable() throws IOException { if (admin.tableExists(tableName)) { System.out.println("coffey TPCHRemotePutBaseClass deleting existing table: " + tableName); admin.disableTable(tableName); admin.deleteTable(tableName); } System.out.println("coffey TPCHRemotePutBaseClass creating ir table: " + tableName); HTableDescriptor tableDesc = new HTableDescriptor(tableName); IndexColumnDescriptor family = new IndexColumnDescriptor(FAMILY_NAME); List<TPCH_CF_INFO> cfs = TPCHConstants.getCFInfo(); for (TPCH_CF_INFO ci : cfs) { if (ci.isIndex) { IndexDescriptor index = new IndexDescriptor(Bytes.toBytes(ci.qualifier), ci.type); family.addIndex(index); System.out.println("coffey TPCHRemotePutBaseClass has ir index on cf: " + ci.qualifier + ", type is: " + ci.type); } } tableDesc.addFamily(family); admin.createTable(tableDesc, splitKeys); System.out.println("coffey TPCHRemotePutBaseClass creating ir table: " + tableName + " finish"); }
@Override protected void checkTable() throws IOException { if (admin.tableExists(tableName)) { System.out.println("coffey PutIR deleting existing table: " + tableName); admin.disableTable(tableName); admin.deleteTable(tableName); } System.out.println("coffey PutIR creating ir table: " + tableName); HTableDescriptor tableDesc = new HTableDescriptor(tableName); IndexColumnDescriptor family = new IndexColumnDescriptor(FAMILY_NAME); List<TPCH_CF_INFO> cfs = TPCHConstants.getCFInfo(); for (TPCH_CF_INFO ci : cfs) { if (ci.isIndex) { IndexDescriptor index = new IndexDescriptor(Bytes.toBytes(ci.qualifier), ci.type); family.addIndex(index); System.out.println("coffey PutIR has ir index on cf: " + ci.qualifier + ", type is: " + ci.type); } } tableDesc.addFamily(family); admin.createTable(tableDesc, splitKeys); System.out.println("coffey PutIR creating ir table: " + tableName + " finish"); }
@Override protected void checkTable() throws IOException { if (admin.tableExists(tableName)) { System.out.println("coffey lcc deleting existing table: " + tableName); admin.disableTable(tableName); admin.deleteTable(tableName); } System.out.println("coffey lcc creating lcc table: " + tableName); HTableDescriptor tableDesc = new HTableDescriptor(tableName); IndexColumnDescriptor family = new IndexColumnDescriptor(FAMILY_NAME, 2); List<CF_INFO> cfs = PutTestConstants.getCFInfo(); for (CF_INFO ci : cfs) { if (ci.isIndex) { IndexDescriptor index = new IndexDescriptor(Bytes.toBytes(ci.qualifier), ci.type); family.addIndex(index); System.out.println("coffey lcc has lcc index on cf: " + ci.qualifier + ", type is: " + ci.type); } } tableDesc.addFamily(family); admin.createTable(tableDesc, splitkeys); System.out.println("coffey lcc creating lcc table: " + tableName + " finish"); }
@Override protected void checkTable() throws IOException { if (admin.tableExists(tableName)) { System.out.println("coffey PutIR deleting existing table: " + tableName); admin.disableTable(tableName); admin.deleteTable(tableName); } System.out.println("coffey PutIR creating ir table: " + tableName); HTableDescriptor tableDesc = new HTableDescriptor(tableName); IndexColumnDescriptor family = new IndexColumnDescriptor(FAMILY_NAME); List<CF_INFO> cfs = PutTestConstants.getCFInfo(); for (CF_INFO ci : cfs) { if (ci.isIndex) { IndexDescriptor index = new IndexDescriptor(Bytes.toBytes(ci.qualifier), ci.type); family.addIndex(index); System.out.println("coffey PutIR has ir index on cf: " + ci.qualifier + ", type is: " + ci.type); } } tableDesc.addFamily(family); admin.createTable(tableDesc, splitkeys); System.out.println("coffey PutIR creating ir table: " + tableName + " finish"); }
@Override protected void checkTable() throws IOException { if (admin.tableExists(tableName)) { System.out.println("coffey lcc deleting existing table: " + tableName); admin.disableTable(tableName); admin.deleteTable(tableName); } System.out.println("coffey lcc creating lcc table: " + tableName); HTableDescriptor tableDesc = new HTableDescriptor(tableName); IndexColumnDescriptor family = new IndexColumnDescriptor(FAMILY_NAME, 2); List<CF_INFO> cfs = PutTestConstants.getCFInfo(); for (CF_INFO ci : cfs) { if (ci.isIndex) { IndexDescriptor index = new IndexDescriptor(Bytes.toBytes(ci.qualifier), ci.type); family.addIndex(index); System.out.println("coffey lcc has lcc index on cf: " + ci.qualifier + ", type is: " + ci.type); } } tableDesc.addFamily(family); // admin.createTable(tableDesc, splitkeys); admin.createTable(tableDesc); System.out.println("coffey lcc creating lcc table: " + tableName + " finish"); }
@Override protected void checkTable() throws IOException { if (admin.tableExists(tableName)) { System.out.println("coffey PutIR deleting existing table: " + tableName); admin.disableTable(tableName); admin.deleteTable(tableName); } System.out.println("coffey PutIR creating ir table: " + tableName); HTableDescriptor tableDesc = new HTableDescriptor(tableName); IndexColumnDescriptor family = new IndexColumnDescriptor(FAMILY_NAME); List<CF_INFO> cfs = PutTestConstants.getCFInfo(); for (CF_INFO ci : cfs) { if (ci.isIndex) { IndexDescriptor index = new IndexDescriptor(Bytes.toBytes(ci.qualifier), ci.type); family.addIndex(index); System.out.println("coffey PutIR has ir index on cf: " + ci.qualifier + ", type is: " + ci.type); } } tableDesc.addFamily(family); admin.createTable(tableDesc); // admin.createTable(tableDesc, splitkeys); System.out.println("coffey PutIR creating ir table: " + tableName + " finish"); }
private void initIRIndex(HBaseAdmin admin) throws IOException { System.out.println("start init IRIndex"); HTableDescriptor tableDesc = new HTableDescriptor(tableName); IndexDescriptor index1 = new IndexDescriptor(Bytes.toBytes("c3"), DataType.DOUBLE); IndexDescriptor index2 = new IndexDescriptor(Bytes.toBytes("c4"), DataType.STRING); IndexDescriptor index3 = new IndexDescriptor(Bytes.toBytes("c5"), DataType.STRING); IndexColumnDescriptor family = new IndexColumnDescriptor("f"); family.addIndex(index1); family.addIndex(index2); family.addIndex(index3); tableDesc.addFamily(family); admin.createTable(tableDesc); }
@Override protected void checkTable() throws IOException { if (admin.tableExists(tableName)) { String localPath = conf.get(LCCIndexConstant.LCINDEX_LOCAL_DIR); if (localPath != null) { String tableDirName = localPath + "/" + tableName; File dir = new File(tableDirName); if (dir.exists()) { FileUtils.deleteDirectory(dir); } String hostnames = conf.get(LCCIndexConstant.LCINDEX_REGIONSERVER_HOSTNAMES); String parts[] = hostnames.split(LCCIndexConstant.LCINDEX_REGIONSERVER_HOSTNAMES_DELIMITER); for (String hostname : parts) { LCCHFileMoverClient cli = new LCCHFileMoverClient(hostname, conf); cli.deleteRemoteFile(tableDirName); } } System.out.println("coffey TPCHRemotePutLCC deleting existing table: " + tableName); admin.disableTable(tableName); admin.deleteTable(tableName); } System.out.println("coffey TPCHRemotePutLCC creating lcc table: " + tableName); HTableDescriptor tableDesc = new HTableDescriptor(tableName); IndexColumnDescriptor family = new IndexColumnDescriptor(FAMILY_NAME, 2); List<TPCH_CF_INFO> cfs = TPCHConstants.getCFInfo(); for (TPCH_CF_INFO ci : cfs) { if (ci.isIndex) { IndexDescriptor index = new IndexDescriptor(Bytes.toBytes(ci.qualifier), ci.type); family.addIndex(index); System.out.println("coffey TPCHRemotePutLCC has lcc index on cf: " + ci.qualifier + ", type is: " + ci.type); } } tableDesc.addFamily(family); fillRangeFromFile(tableDesc); // admin.createTable(tableDesc, splitKeys); admin.createTable(tableDesc); System.out.println("coffey TPCHRemotePutLCC creating lcc table: " + tableName + " finish"); }
@Override protected void checkTable() throws IOException { if (admin.tableExists(tableName)) { String localPath = conf.get(LCCIndexConstant.LCINDEX_LOCAL_DIR); if (localPath != null) { String tableDirName = localPath + "/" + tableName; File dir = new File(tableDirName); if (dir.exists()) { FileUtils.deleteDirectory(dir); } String hostnames = conf.get(LCCIndexConstant.LCINDEX_REGIONSERVER_HOSTNAMES); String parts[] = hostnames.split(LCCIndexConstant.LCINDEX_REGIONSERVER_HOSTNAMES_DELIMITER); for (String hostname : parts) { LCCHFileMoverClient cli = new LCCHFileMoverClient(hostname, conf); cli.deleteRemoteFile(tableDirName); } } System.out.println("coffey lcc deleting existing table: " + tableName); admin.disableTable(tableName); admin.deleteTable(tableName); } System.out.println("coffey lcc creating lcc table: " + tableName); HTableDescriptor tableDesc = new HTableDescriptor(tableName); IndexColumnDescriptor family = new IndexColumnDescriptor(FAMILY_NAME, 2); List<TPCH_CF_INFO> cfs = TPCHConstants.getCFInfo(); for (TPCH_CF_INFO ci : cfs) { if (ci.isIndex) { IndexDescriptor index = new IndexDescriptor(Bytes.toBytes(ci.qualifier), ci.type); family.addIndex(index); System.out.println("coffey lcc has lcc index on cf: " + ci.qualifier + ", type is: " + ci.type); } } tableDesc.addFamily(family); fillRangeFromFile(tableDesc); admin.createTable(tableDesc, splitKeys); System.out.println("coffey lcc creating lcc table: " + tableName + " finish"); }
private void initIRIndex(HBaseAdmin admin) throws IOException { System.out.println("start init IRIndex"); HTableDescriptor tableDesc = new HTableDescriptor(tableName); IndexDescriptor index1 = new IndexDescriptor(Bytes.toBytes("A"), DataType.DOUBLE); IndexDescriptor index2 = new IndexDescriptor(Bytes.toBytes("B"), DataType.INT); IndexColumnDescriptor family = new IndexColumnDescriptor("f", 2); family.addIndex(index1); family.addIndex(index2); tableDesc.addFamily(family); admin.createTable(tableDesc); }
private void initIRIndex(HBaseAdmin admin) throws IOException { System.out.println("start init IRIndex"); HTableDescriptor tableDesc = new HTableDescriptor(tableName); IndexDescriptor index1 = new IndexDescriptor(Bytes.toBytes("A"), DataType.DOUBLE); IndexDescriptor index2 = new IndexDescriptor(Bytes.toBytes("B"), DataType.INT); IndexColumnDescriptor family = new IndexColumnDescriptor("f"); family.addIndex(index1); family.addIndex(index2); tableDesc.addFamily(family); admin.createTable(tableDesc); }
protected void performFlushWithIndex(InternalScanner scanner, StoreFile.Writer writer, long smallestReadPoint, HStore store) throws IOException { int compactionKVMax = conf.getInt(HConstants.COMPACTION_KV_MAX, HConstants.COMPACTION_KV_MAX_DEFAULT); ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build(); List<Cell> kvs = new ArrayList<Cell>(); boolean hasMore; IndexWriter indexWriter = null; TreeSet<IndexKeyValue> irIndexSet = null; if (store.getIndexType() == IndexType.IRIndex) { irIndexSet = new TreeSet<IndexKeyValue>(store.irIndexComparator); for (Map.Entry<byte[], IndexDescriptor> entry : store.irIndexMap.entrySet()) { Log.info("LCINFO: flush with irindex qualifier: " + Bytes.toString(entry.getKey()) + ", index table is: " + entry.getValue()); } } LinkedList<KeyValue> keyvalueQueue = null; if (store.getIndexType() == IndexType.LCIndex || store.getIndexType() == IndexType.LMDIndex_S || store.getIndexType() == IndexType.LMDIndex_D) { // save keyvalue because snapshot will be cleaned soon keyvalueQueue = new LinkedList<>(); } try { do { hasMore = scanner.next(kvs, scannerContext); if (!kvs.isEmpty()) { for (Cell c : kvs) { writer.append(c); // LCTODO: how to handle delete and remove? if (store.getIndexType() == IndexType.IRIndex) { if (store.irIndexMap.containsKey(CellUtil.cloneQualifier(c))) { irIndexSet.add(new IndexKeyValue(c)); } } else if (store.getIndexType() == IndexType.LCIndex) { keyvalueQueue.add((KeyValue) c); } else if (store.getIndexType() == IndexType.LMDIndex_S || store.getIndexType() == IndexType.LMDIndex_D) { keyvalueQueue.add((KeyValue) c); } } kvs.clear(); } } while (hasMore); if (store.getIndexType() == IndexType.IRIndex && !irIndexSet.isEmpty()) { // LCDBG: committing store fromsnapshotTimeRangeTracker // hdfs://localhost:9000/hbase/data/hbase/meta/1588230740/.tmp/c1aaec13c7ab48669e8aeeab66bdf1e1 // to // hdfs://localhost:9000/hbase/data/hbase/meta/1588230740/info/c1aaec13c7ab48669e8aeeab66bdf1e1 indexWriter = store.createIndexWriterInTmp(writer.getPath()); for (IndexKeyValue tmpkv : irIndexSet) { indexWriter.append(tmpkv); } } else if (store.getIndexType() == IndexType.LCIndex) { FlushJobQueue.FlushJob flushJob = new FlushJobQueue.FlushJob(store, keyvalueQueue, writer.getPath(), writer.timeRangeTracker); FlushJobQueue.getInstance().addJob(flushJob); } else if (store.getIndexType() == IndexType.LMDIndex_S || store.getIndexType() == IndexType.LMDIndex_D) { LMDIndexWriter lmdIndexWriter = new LMDIndexWriter(store, writer.getPath(), writer.timeRangeTracker, "FLUSH"); lmdIndexWriter.processKeyValueQueue(keyvalueQueue); } } catch (IOException e) { throw e; } finally { if (indexWriter != null) indexWriter.close(); } }
public static void main(String[] args) throws IOException { Configuration conf = HBaseConfiguration.create(); HBaseAdmin admin = new HBaseAdmin(conf); HTable table = null; // if (admin.tableExists(tableName)) { // indexadmin.disableTable(tableName); // indexadmin.deleteTable(tableName); // } if (!admin.tableExists(tableName)) { HTableDescriptor tableDesc = new HTableDescriptor(tableName); IndexDescriptor index1 = new IndexDescriptor(Bytes.toBytes("c1"), DataType.DOUBLE); IndexColumnDescriptor family = new IndexColumnDescriptor("f"); family.addIndex(index1); tableDesc.addFamily(family); admin.createTable(tableDesc); table = new HTable(conf, tableName); Put put = new Put(Bytes.toBytes("001")); put.add(Bytes.toBytes("f"), Bytes.toBytes("c1"), 10, Bytes.toBytes(120.0)); put.add(Bytes.toBytes("f"), Bytes.toBytes("c2"), 10, Bytes.toBytes("a")); table.put(put); put = new Put(Bytes.toBytes("001")); put.add(Bytes.toBytes("f"), Bytes.toBytes("c1"), 20, Bytes.toBytes(90.0)); put.add(Bytes.toBytes("f"), Bytes.toBytes("c2"), 20, Bytes.toBytes("a")); table.put(put); put = new Put(Bytes.toBytes("002")); put.add(Bytes.toBytes("f"), Bytes.toBytes("c1"), 30, Bytes.toBytes(150.0)); put.add(Bytes.toBytes("f"), Bytes.toBytes("c2"), 30, Bytes.toBytes("a")); table.put(put); } admin.close(); if (table == null) { table = new HTable(conf, tableName); } Scan scan = new Scan(); RangeList list = new RangeList(); list.addRange(new Range(Bytes.toBytes("f:c1"), Bytes.toBytes(100.0), CompareOp.GREATER, null, CompareOp.NO_OP)); SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("f"), Bytes.toBytes("c1"), CompareOp.GREATER, Bytes.toBytes(100.0)); filter.setLatestVersionOnly(false); scan.setFilter(filter); // scan.setAttribute(IndexConstants.SCAN_INDEX_RANGE, Writables.getBytes(list)); scan.setMaxVersions(); Result result = null; ResultScanner rs = table.getScanner(scan); while ((result = rs.next()) != null) { System.out.println(result); } rs.close(); table.close(); }
public IzpPutThroughputTest() throws IOException { Configuration conf = HBaseConfiguration.create(); HBaseAdmin admin = new HBaseAdmin(conf); fs =new Path(filePath).getFileSystem(conf); // if(admin.tableExists(tableName)){ // indexadmin.disableTable(tableName); // indexadmin.deleteTable(tableName); // } if (!admin.tableExists(tableName)) { HTableDescriptor tableDesc = new HTableDescriptor(tableName); if (index == 1) { IndexDescriptor index1 = new IndexDescriptor(Bytes.toBytes("f"), DataType.STRING); IndexDescriptor index2 = new IndexDescriptor(Bytes.toBytes("h"), DataType.STRING); IndexDescriptor index3 = new IndexDescriptor(Bytes.toBytes("a"), DataType.STRING); IndexDescriptor index4 = new IndexDescriptor(Bytes.toBytes("y"), DataType.STRING); IndexDescriptor index5 = new IndexDescriptor(Bytes.toBytes("r"), DataType.STRING); IndexDescriptor index6 = new IndexDescriptor(Bytes.toBytes("g"), DataType.STRING); IndexDescriptor index7 = new IndexDescriptor(Bytes.toBytes("p"), DataType.STRING); IndexDescriptor index8 = new IndexDescriptor(Bytes.toBytes("o"), DataType.STRING); IndexDescriptor index9 = new IndexDescriptor(Bytes.toBytes("s"), DataType.INT); IndexColumnDescriptor family1 = new IndexColumnDescriptor("f"); family1.addIndex(index1); family1.addIndex(index2); family1.addIndex(index3); family1.addIndex(index4); family1.addIndex(index5); family1.addIndex(index6); family1.addIndex(index7); family1.addIndex(index8); family1.addIndex(index9); HColumnDescriptor family2 = new HColumnDescriptor("q"); tableDesc.addFamily(family1); tableDesc.addFamily(family2); byte[][] splitkeys=new byte[regionNum][]; for(int i=0;i<regionNum;i++){ splitkeys[i]=Bytes.toBytes(String.format("%03d", i)); } admin.createTable(tableDesc, splitkeys); } else { HColumnDescriptor family = new HColumnDescriptor("f"); tableDesc.addFamily(family); admin.createTable(tableDesc, Bytes.toBytes("000"), Bytes.toBytes("099"), 101); } } admin.close(); }
public static void main(String[] args) throws IOException { Configuration conf = HBaseConfiguration.create(); HBaseAdmin admin = new HBaseAdmin(conf); HTable table = null; // if (admin.tableExists(tableName)) { // indexadmin.disableTable(tableName); // indexadmin.deleteTable(tableName); // } if (!admin.tableExists(tableName)) { HTableDescriptor tableDesc = new HTableDescriptor(tableName); IndexDescriptor index1 = new IndexDescriptor(Bytes.toBytes("c1"), DataType.DOUBLE); IndexColumnDescriptor family = new IndexColumnDescriptor("f"); family.addIndex(index1); tableDesc.addFamily(family); admin.createTable(tableDesc); table = new HTable(conf, tableName); Put put = new Put(Bytes.toBytes("001")); put.add(Bytes.toBytes("f"), Bytes.toBytes("c1"), 10, Bytes.toBytes(120.0)); put.add(Bytes.toBytes("f"), Bytes.toBytes("c2"), 10, Bytes.toBytes("a")); table.put(put); put = new Put(Bytes.toBytes("001")); put.add(Bytes.toBytes("f"), Bytes.toBytes("c1"), 20, Bytes.toBytes(90.0)); put.add(Bytes.toBytes("f"), Bytes.toBytes("c2"), 20, Bytes.toBytes("a")); table.put(put); put = new Put(Bytes.toBytes("002")); put.add(Bytes.toBytes("f"), Bytes.toBytes("c1"), 30, Bytes.toBytes(150.0)); put.add(Bytes.toBytes("f"), Bytes.toBytes("c2"), 30, Bytes.toBytes("a")); table.put(put); } admin.close(); if (table == null) { table = new HTable(conf, tableName); } Scan scan = new Scan(); RangeList list = new RangeList(); list.addRange(new Range(Bytes.toBytes("f:c1"), Bytes.toBytes(100.0), CompareOp.GREATER, null, CompareOp.NO_OP)); SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("f"), Bytes.toBytes("c1"), CompareOp.GREATER, Bytes.toBytes(100.0)); filter.setLatestVersionOnly(false); scan.setFilter(filter); scan.setAttribute(IndexConstants.SCAN_WITH_INDEX, Writables.getBytes(list)); scan.setMaxVersions(); Result result = null; ResultScanner rs = table.getScanner(scan); while ((result = rs.next()) != null) { System.out.println(result); } rs.close(); table.close(); }