private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName, final byte[] family, final int count, EnvironmentEdge ee, final WAL wal, final HTableDescriptor htd, final MultiVersionConcurrencyControl mvcc) throws IOException { String familyStr = Bytes.toString(family); long txid = -1; for (int j = 0; j < count; j++) { byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j)); byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j)); WALEdit edit = new WALEdit(); edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes)); // uses WALKey instead of HLogKey on purpose. will only work for tests where we don't care // about legacy coprocessors txid = wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, ee.currentTime(), mvcc), edit, true); } if (-1 != txid) { wal.sync(txid); } }
private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName, final byte[] family, final int count, EnvironmentEdge ee, final WAL wal, final HTableDescriptor htd, final MultiVersionConcurrencyControl mvcc) throws IOException { String familyStr = Bytes.toString(family); for (int j = 0; j < count; j++) { byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j)); byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j)); WALEdit edit = new WALEdit(); edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes)); wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName,999, mvcc), edit, true); } wal.sync(); }
private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName, final byte[] family, final int count, EnvironmentEdge ee, final WAL wal, final HTableDescriptor htd, final AtomicLong sequenceId) throws IOException { String familyStr = Bytes.toString(family); long txid = -1; for (int j = 0; j < count; j++) { byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j)); byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j)); WALEdit edit = new WALEdit(); edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes)); // uses WALKey instead of HLogKey on purpose. will only work for tests where we don't care // about legacy coprocessors txid = wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, ee.currentTime()), edit, sequenceId, true, null); } if (-1 != txid) { wal.sync(txid); } }
private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName, final byte[] family, final int count, EnvironmentEdge ee, final WAL wal, final HTableDescriptor htd, final AtomicLong sequenceId) throws IOException { String familyStr = Bytes.toString(family); for (int j = 0; j < count; j++) { byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j)); byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j)); WALEdit edit = new WALEdit(); edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes)); wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, ee.currentTime()), edit, sequenceId, true, null); } wal.sync(); }
private void addWALEdits(final TableName tableName, final RegionInfo hri, final byte[] rowName, final byte[] family, final int count, EnvironmentEdge ee, final WAL wal, final NavigableMap<byte[], Integer> scopes, final MultiVersionConcurrencyControl mvcc) throws IOException { String familyStr = Bytes.toString(family); long txid = -1; for (int j = 0; j < count; j++) { byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j)); byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j)); WALEdit edit = new WALEdit(); edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes)); // uses WALKeyImpl instead of HLogKey on purpose. will only work for tests where we don't care // about legacy coprocessors txid = wal.append(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, ee.currentTime(), mvcc), edit, true); } if (-1 != txid) { wal.sync(txid); } }
private void waitForRegion() throws InterruptedIOException { if (busyRegions.isEmpty()) { return; } EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate(); final long start = ee.currentTime(); while ((ee.currentTime() - start) <= MAX_WAITING_TIME) { for (byte[] region : busyRegions) { AtomicInteger count = taskCounterPerRegion.get(region); if (count == null || count.get() < maxConcurrentTasksPerRegion) { return; } } try { synchronized (tasksInProgress) { tasksInProgress.wait(10); } } catch (InterruptedException e) { throw new InterruptedIOException("Interrupted." + " tasksInProgress=" + tasksInProgress); } } }
static List<Put> addRegionEdits (final byte [] rowName, final byte [] family, final int count, EnvironmentEdge ee, final Region r, final String qualifierPrefix) throws IOException { List<Put> puts = new ArrayList<Put>(); for (int j = 0; j < count; j++) { byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j)); Put p = new Put(rowName); p.add(family, qualifier, ee.currentTime(), rowName); r.put(p); puts.add(p); } return puts; }
private void addWALEdits (final byte [] tableName, final HRegionInfo hri, final byte [] rowName, final byte [] family, final int count, EnvironmentEdge ee, final HLog wal, final HTableDescriptor htd) throws IOException { String familyStr = Bytes.toString(family); for (int j = 0; j < count; j++) { byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j)); byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j)); WALEdit edit = new WALEdit(); edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTimeMillis(), columnBytes)); wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd); } }
private void addRegionEdits (final byte [] rowName, final byte [] family, final int count, EnvironmentEdge ee, final HRegion r, final String qualifierPrefix) throws IOException { for (int j = 0; j < count; j++) { byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j)); Put p = new Put(rowName); p.add(family, qualifier, ee.currentTimeMillis(), rowName); r.put(p); } }
static List<Put> addRegionEdits (final byte [] rowName, final byte [] family, final int count, EnvironmentEdge ee, final HRegion r, final String qualifierPrefix) throws IOException { List<Put> puts = new ArrayList<Put>(); for (int j = 0; j < count; j++) { byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j)); Put p = new Put(rowName); p.add(family, qualifier, ee.currentTime(), rowName); r.put(p); puts.add(p); } return puts; }
private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName, final byte[] family, final int count, EnvironmentEdge ee, final HLog wal, final HTableDescriptor htd, final AtomicLong sequenceId) throws IOException { String familyStr = Bytes.toString(family); for (int j = 0; j < count; j++) { byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j)); byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j)); WALEdit edit = new WALEdit(); edit.add(new KeyValue(rowName, family, qualifierBytes, ee .currentTimeMillis(), columnBytes)); wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd, sequenceId); } }
private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName, final byte[] family, final int count, EnvironmentEdge ee, final HLog wal, final HTableDescriptor htd, final AtomicLong sequenceId) throws IOException { String familyStr = Bytes.toString(family); for (int j = 0; j < count; j++) { byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j)); byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j)); WALEdit edit = new WALEdit(); edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTimeMillis(), columnBytes)); wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd, sequenceId); } }
@Test public void testOverconsumptionFixedIntervalRefillStrategy() throws InterruptedException { RateLimiter limiter = new FixedIntervalRateLimiter(); limiter.set(10, TimeUnit.SECONDS); // fix the current time in order to get the precise value of interval EnvironmentEdge edge = new EnvironmentEdge() { private final long ts = System.currentTimeMillis(); @Override public long currentTime() { return ts; } }; EnvironmentEdgeManager.injectEdge(edge); // 10 resources are available, but we need to consume 20 resources // Verify that we have to wait at least 1.1sec to have 1 resource available assertTrue(limiter.canExecute()); limiter.consume(20); // To consume 1 resource also wait for 1000ms assertEquals(1000, limiter.waitInterval(1)); // To consume 10 resource wait for 100ms assertEquals(1000, limiter.waitInterval(10)); EnvironmentEdgeManager.reset(); limiter.setNextRefillTime(limiter.getNextRefillTime() - 900); // Verify that after 1sec also no resource should be available assertFalse(limiter.canExecute(1)); limiter.setNextRefillTime(limiter.getNextRefillTime() - 100); // Verify that after 1sec the 10 resource is available assertTrue(limiter.canExecute()); assertEquals(0, limiter.waitInterval()); }
@BeforeClass public static void setEnvironmentEdge() throws Exception { EnvironmentEdge ee = new TimeOffsetEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(ee); Configuration conf = TEST_UTIL.getConfiguration(); conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000); TEST_UTIL.startMiniCluster(1); }
private WALEdit createWALEdit(final byte[] rowName, final byte[] family, EnvironmentEdge ee, int index) { byte[] qualifierBytes = Bytes.toBytes(Integer.toString(index)); byte[] columnBytes = Bytes.toBytes(Bytes.toString(family) + ":" + Integer.toString(index)); WALEdit edit = new WALEdit(); edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes)); return edit; }
private FSWALEntry createFSWALEntry(HTableDescriptor htd, HRegionInfo hri, long sequence, byte[] rowName, byte[] family, EnvironmentEdge ee, MultiVersionConcurrencyControl mvcc, int index, NavigableMap<byte[], Integer> scopes) throws IOException { FSWALEntry entry = new FSWALEntry(sequence, createWALKey(htd.getTableName(), hri, mvcc, scopes), createWALEdit( rowName, family, ee, index), hri, true); entry.stampRegionSequenceId(mvcc.begin()); return entry; }
private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName, final byte[] family, final int count, EnvironmentEdge ee, final WAL wal, final HTableDescriptor htd, final MultiVersionConcurrencyControl mvcc, NavigableMap<byte[], Integer> scopes) throws IOException { for (int j = 0; j < count; j++) { wal.append(hri, createWALKey(tableName, hri, mvcc, scopes), createWALEdit(rowName, family, ee, j), true); } wal.sync(); }
static List<Put> addRegionEdits(final byte[] rowName, final byte[] family, final int count, EnvironmentEdge ee, final Region r, final String qualifierPrefix) throws IOException { List<Put> puts = new ArrayList<>(); for (int j = 0; j < count; j++) { byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j)); Put p = new Put(rowName); p.addColumn(family, qualifier, ee.currentTime(), rowName); r.put(p); puts.add(p); } return puts; }
static List<Put> addRegionEdits (final byte [] rowName, final byte [] family, final int count, EnvironmentEdge ee, final HRegion r, final String qualifierPrefix) throws IOException { List<Put> puts = new ArrayList<Put>(); for (int j = 0; j < count; j++) { byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j)); Put p = new Put(rowName); p.add(family, qualifier, ee.currentTimeMillis(), rowName); r.put(p); puts.add(p); } return puts; }
private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName, final byte[] family, final int count, EnvironmentEdge ee, final HLog wal, final HTableDescriptor htd) throws IOException { String familyStr = Bytes.toString(family); for (int j = 0; j < count; j++) { byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j)); byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j)); WALEdit edit = new WALEdit(); edit.add(new KeyValue(rowName, family, qualifierBytes, ee .currentTimeMillis(), columnBytes)); wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd); } }
private void addWALEdits (final TableName tableName, final HRegionInfo hri, final byte [] rowName, final byte [] family, final int count, EnvironmentEdge ee, final HLog wal, final HTableDescriptor htd) throws IOException { String familyStr = Bytes.toString(family); for (int j = 0; j < count; j++) { byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j)); byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j)); WALEdit edit = new WALEdit(); edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTimeMillis(), columnBytes)); wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd); } }