protected void prepareForLoadTest() throws IOException { LOG.info("Starting load test: dataBlockEncoding=" + dataBlockEncoding + ", isMultiPut=" + isMultiPut); numKeys = numKeys(); Admin admin = new HBaseAdmin(conf); while (admin.getClusterStatus().getServers().size() < NUM_RS) { LOG.info("Sleeping until " + NUM_RS + " RSs are online"); Threads.sleepWithoutInterrupt(1000); } admin.close(); HTableDescriptor htd = new HTableDescriptor(TABLE); HColumnDescriptor hcd = new HColumnDescriptor(CF) .setCompressionType(compression) .setDataBlockEncoding(dataBlockEncoding); createPreSplitLoadTestTable(htd, hcd); LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator(CF); writerThreads = prepareWriterThreads(dataGen, conf, TABLE); readerThreads = prepareReaderThreads(dataGen, conf, TABLE, 100); }
protected void prepareForLoadTest() throws IOException { LOG.info("Starting load test: dataBlockEncoding=" + dataBlockEncoding + ", isMultiPut=" + isMultiPut); numKeys = numKeys(); HBaseAdmin admin = new HBaseAdmin(conf); while (admin.getClusterStatus().getServers().size() < NUM_RS) { LOG.info("Sleeping until " + NUM_RS + " RSs are online"); Threads.sleepWithoutInterrupt(1000); } admin.close(); HTableDescriptor htd = new HTableDescriptor(TABLE); HColumnDescriptor hcd = new HColumnDescriptor(CF) .setCompressionType(compression) .setDataBlockEncoding(dataBlockEncoding); createPreSplitLoadTestTable(htd, hcd); LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator(CF); writerThreads = prepareWriterThreads(dataGen, conf, TABLE); readerThreads = prepareReaderThreads(dataGen, conf, TABLE, 100); }
protected void prepareForLoadTest() throws IOException { LOG.info("Starting load test: dataBlockEncoding=" + dataBlockEncoding + ", isMultiPut=" + isMultiPut); numKeys = numKeys(); Admin admin = TEST_UTIL.getAdmin(); while (admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)) .getLiveServerMetrics().size() < NUM_RS) { LOG.info("Sleeping until " + NUM_RS + " RSs are online"); Threads.sleepWithoutInterrupt(1000); } admin.close(); HTableDescriptor htd = new HTableDescriptor(TABLE); HColumnDescriptor hcd = new HColumnDescriptor(CF) .setCompressionType(compression) .setDataBlockEncoding(dataBlockEncoding); createPreSplitLoadTestTable(htd, hcd); LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator(CF); writerThreads = prepareWriterThreads(dataGen, conf, TABLE); readerThreads = prepareReaderThreads(dataGen, conf, TABLE, 100); }
protected void prepareForLoadTest() throws IOException { LOG.info("Starting load test: dataBlockEncoding=" + dataBlockEncoding + ", isMultiPut=" + isMultiPut); numKeys = numKeys(); HBaseAdmin admin = new HBaseAdmin(conf); while (admin.getClusterStatus().getServers().size() < NUM_RS) { LOG.info("Sleeping until " + NUM_RS + " RSs are online"); Threads.sleepWithoutInterrupt(1000); } admin.close(); int numRegions = HBaseTestingUtility.createPreSplitLoadTestTable(conf, TABLE, CF, compression, dataBlockEncoding); TEST_UTIL.waitUntilAllRegionsAssigned(numRegions); LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator(CF); writerThreads = new MultiThreadedWriter(dataGen, conf, TABLE); writerThreads.setMultiPut(isMultiPut); readerThreads = new MultiThreadedReader(dataGen, conf, TABLE, 100); }
public MultiThreadedAction(LoadTestDataGenerator dataGen, Configuration conf, TableName tableName, String actionLetter) throws IOException { this.conf = conf; this.dataGenerator = dataGen; this.tableName = tableName; this.actionLetter = actionLetter; this.connection = HConnectionManager.createConnection(conf); }
private LoadTestDataGenerator getLoadGeneratorInstance(String clazzName) throws IOException { try { Class<?> clazz = Class.forName(clazzName); Constructor<?> constructor = clazz.getConstructor(int.class, int.class, int.class, int.class, byte[][].class); return (LoadTestDataGenerator) constructor.newInstance(minColDataSize, maxColDataSize, minColsPerKey, maxColsPerKey, families); } catch (Exception e) { throw new IOException(e); } }
private MultiThreadedWriter getMultiThreadedWriterInstance(String clazzName , LoadTestDataGenerator dataGen) throws IOException { try { Class<?> clazz = Class.forName(clazzName); Constructor<?> constructor = clazz.getConstructor( LoadTestDataGenerator.class, Configuration.class, TableName.class); return (MultiThreadedWriter) constructor.newInstance(dataGen, conf, tableName); } catch (Exception e) { throw new IOException(e); } }
private MultiThreadedUpdater getMultiThreadedUpdaterInstance(String clazzName , LoadTestDataGenerator dataGen) throws IOException { try { Class<?> clazz = Class.forName(clazzName); Constructor<?> constructor = clazz.getConstructor( LoadTestDataGenerator.class, Configuration.class, TableName.class, double.class); return (MultiThreadedUpdater) constructor.newInstance( dataGen, conf, tableName, updatePercent); } catch (Exception e) { throw new IOException(e); } }
private MultiThreadedReader getMultiThreadedReaderInstance(String clazzName , LoadTestDataGenerator dataGen) throws IOException { try { Class<?> clazz = Class.forName(clazzName); Constructor<?> constructor = clazz.getConstructor( LoadTestDataGenerator.class, Configuration.class, TableName.class, double.class); return (MultiThreadedReader) constructor.newInstance(dataGen, conf, tableName, verifyPercent); } catch (Exception e) { throw new IOException(e); } }
public MultiThreadedUpdaterWithACL(LoadTestDataGenerator dataGen, Configuration conf, TableName tableName, double updatePercent, User userOwner, String userNames) throws IOException { super(dataGen, conf, tableName, updatePercent); this.userOwner = userOwner; this.userNames = userNames.split(COMMA); }
/** Loads data into the table using the multi-threaded writer. */ private void loadData() throws IOException { long startKey = 0; long endKey = 100000; int minColsPerKey = 5; int maxColsPerKey = 15; int minColDataSize = 256; int maxColDataSize = 256 * 3; int numThreads = 10; // print out the arguments System.out.printf("Key range %d .. %d\n", startKey, endKey); System.out.printf("Number of Columns/Key: %d..%d\n", minColsPerKey, maxColsPerKey); System.out.printf("Data Size/Column: %d..%d bytes\n", minColDataSize, maxColDataSize); System.out.printf("Client Threads: %d\n", numThreads); // start the writers LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator( minColDataSize, maxColDataSize, minColsPerKey, maxColsPerKey, LoadTestTool.DEFAULT_COLUMN_FAMILY); MultiThreadedWriter writer = new MultiThreadedWriter(dataGen, conf, TABLE_NAME); writer.setMultiPut(true); writer.start(startKey, endKey, numThreads); System.out.printf("Started loading data..."); writer.waitForFinish(); System.out.printf("Finished loading data..."); }
public TimeBoundedMultiThreadedReader(LoadTestDataGenerator dataGen, Configuration conf, TableName tableName, double verifyPercent) throws IOException { super(dataGen, conf, tableName, verifyPercent); long timeoutMs = conf.getLong( String.format("%s.%s", TEST_NAME, GET_TIMEOUT_KEY), DEFAULT_GET_TIMEOUT); timeoutNano = timeoutMs * 1000000; LOG.info("Timeout for gets: " + timeoutMs); String runTimeKey = String.format(RUN_TIME_KEY, this.getClass().getSimpleName()); this.runTime = conf.getLong(runTimeKey, -1); if (this.runTime <= 0) { throw new IllegalArgumentException("Please configure " + runTimeKey); } }
private LoadTestDataGenerator getLoadGeneratorInstance(String clazzName) throws IOException { try { Class<?> clazz = Class.forName(clazzName); Constructor<?> constructor = clazz.getConstructor(int.class, int.class, int.class, int.class, byte[][].class); return (LoadTestDataGenerator) constructor.newInstance(minColDataSize, maxColDataSize, minColsPerKey, maxColsPerKey, COLUMN_FAMILIES); } catch (Exception e) { throw new IOException(e); } }
/** Loads data into the table using the multi-threaded writer. */ private void loadData() throws IOException { long startKey = 0; long endKey = 100000; int minColsPerKey = 5; int maxColsPerKey = 15; int minColDataSize = 256; int maxColDataSize = 256 * 3; int numThreads = 10; // print out the arguments System.out.printf("Key range %d .. %d\n", startKey, endKey); System.out.printf("Number of Columns/Key: %d..%d\n", minColsPerKey, maxColsPerKey); System.out.printf("Data Size/Column: %d..%d bytes\n", minColDataSize, maxColDataSize); System.out.printf("Client Threads: %d\n", numThreads); // start the writers LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator( minColDataSize, maxColDataSize, minColsPerKey, maxColsPerKey, LoadTestTool.COLUMN_FAMILY); MultiThreadedWriter writer = new MultiThreadedWriter(dataGen, conf, TABLE_NAME); writer.setMultiPut(true); writer.start(startKey, endKey, numThreads); System.out.printf("Started loading data..."); writer.waitForFinish(); System.out.printf("Finished loading data..."); }
public MultiThreadedAction(LoadTestDataGenerator dataGen, Configuration conf, TableName tableName, String actionLetter) { this.conf = conf; this.dataGenerator = dataGen; this.tableName = tableName; this.actionLetter = actionLetter; }
public MultiThreadedAction(LoadTestDataGenerator dataGen, Configuration conf, TableName tableName, String actionLetter) throws IOException { this.conf = conf; this.dataGenerator = dataGen; this.tableName = tableName; this.actionLetter = actionLetter; this.connection = (ClusterConnection) ConnectionFactory.createConnection(conf); }
/** Loads data into the table using the multi-threaded writer. */ private void loadData() throws IOException { long startKey = 0; long endKey = 100000; int minColsPerKey = 5; int maxColsPerKey = 15; int minColDataSize = 256; int maxColDataSize = 256 * 3; int numThreads = 10; // print out the arguments System.out.printf("Key range %d .. %d\n", startKey, endKey); System.out.printf("Number of Columns/Key: %d..%d\n", minColsPerKey, maxColsPerKey); System.out.printf("Data Size/Column: %d..%d bytes\n", minColDataSize, maxColDataSize); System.out.printf("Client Threads: %d\n", numThreads); // start the writers LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator( minColDataSize, maxColDataSize, minColsPerKey, maxColsPerKey, HFileTestUtil.DEFAULT_COLUMN_FAMILY); MultiThreadedWriter writer = new MultiThreadedWriter(dataGen, conf, TABLE_NAME); writer.setMultiPut(true); writer.start(startKey, endKey, numThreads); System.out.printf("Started loading data..."); writer.waitForFinish(); System.out.printf("Finished loading data..."); }
public MultiThreadedAction(LoadTestDataGenerator dataGen, Configuration conf, byte[] tableName, String actionLetter) { this.conf = conf; this.dataGenerator = dataGen; this.tableName = tableName; this.actionLetter = actionLetter; }
public MultiThreadedWriterBase(LoadTestDataGenerator dataGen, Configuration conf, TableName tableName, String actionLetter) throws IOException { super(dataGen, conf, tableName, actionLetter); this.wroteKeys = createWriteKeysQueue(conf); }
public MultiThreadedWriter(LoadTestDataGenerator dataGen, Configuration conf, TableName tableName) throws IOException { super(dataGen, conf, tableName, "W"); }