Java 类org.apache.hadoop.hbase.regionserver.StoreFileScanner 实例源码

项目：ditb 文件：Compactor.java

protected InternalScanner preCreateCoprocScanner(final CompactionRequest request,
    final ScanType scanType, final long earliestPutTs, final List<StoreFileScanner> scanners,
    User user) throws IOException {
  if (store.getCoprocessorHost() == null) return null;
  if (user == null) {
    return store.getCoprocessorHost()
        .preCompactScannerOpen(store, scanners, scanType, earliestPutTs, request);
  } else {
    try {
      return user.getUGI().doAs(new PrivilegedExceptionAction<InternalScanner>() {
        @Override public InternalScanner run() throws Exception {
          return store.getCoprocessorHost()
              .preCompactScannerOpen(store, scanners, scanType, earliestPutTs, request);
        }
      });
    } catch (InterruptedException ie) {
      InterruptedIOException iioe = new InterruptedIOException();
      iioe.initCause(ie);
      throw iioe;
    }
  }
}

项目：ditb 文件：LMDIndexDirectStoreFileScanner.java

private List<byte[]> initRowKeyList(FileSystem fileSystem, CacheConfig cacheConf,
    Configuration conf, TreeMap<byte[], TreeSet<byte[]>> indexFamilyMap,
    ScanRange.ScanRangeList rangeList) throws IOException {
  // init
  StoreFile bucketStoreFile =
      new StoreFile(fileSystem, LMDIndexParameters.getTmpBucketFilePath(file.getPath()), conf,
          cacheConf, BloomType.NONE);
  StoreFile secondaryStoreFile =
      new StoreFile(fileSystem, LMDIndexParameters.getTmpSecondaryFilePath(file.getPath()), conf,
          cacheConf, BloomType.NONE);
  StoreFileScanner bucketScanner = getStoreFileScanner(bucketStoreFile);
  StoreFileScanner secondaryScanner = getStoreFileScanner(secondaryStoreFile);
  // get hit buckets
  MDRange[] ranges = getRanges(indexFamilyMap, rangeList);
  List<LMDBucket> bucketList = getBucketRanges(bucketScanner, ranges);
  // scan rowkeys based on the buckets
  List<byte[]> rowkeyList = getRawRowkeyList(secondaryScanner, bucketList, ranges);
  // deinit
  bucketScanner.close();
  bucketStoreFile.closeReader(true);
  secondaryScanner.close();
  secondaryStoreFile.closeReader(true);
  return rowkeyList;
}

项目：hbase 文件：MobFile.java

/**
 * Reads a cell from the mob file.
 * @param search The cell need to be searched in the mob file.
 * @param cacheMobBlocks Should this scanner cache blocks.
 * @param readPt the read point.
 * @return The cell in the mob file.
 * @throws IOException
 */
public Cell readCell(Cell search, boolean cacheMobBlocks, long readPt) throws IOException {
  Cell result = null;
  StoreFileScanner scanner = null;
  List<HStoreFile> sfs = new ArrayList<>();
  sfs.add(sf);
  try {
    List<StoreFileScanner> sfScanners = StoreFileScanner.getScannersForStoreFiles(sfs,
      cacheMobBlocks, true, false, false, readPt);
    if (!sfScanners.isEmpty()) {
      scanner = sfScanners.get(0);
      if (scanner.seek(search)) {
        result = scanner.peek();
      }
    }
  } finally {
    if (scanner != null) {
      scanner.close();
    }
  }
  return result;
}

项目：hbase 文件：TestMobFile.java

@Test
public void testGetScanner() throws Exception {
  Path testDir = TEST_UTIL.getDataTestDir();
  FileSystem fs = testDir.getFileSystem(conf);
  HFileContext meta = new HFileContextBuilder().withBlockSize(8*1024).build();
  StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, fs)
          .withOutputDir(testDir)
          .withFileContext(meta)
          .build();
  MobTestUtil.writeStoreFile(writer, testName.getMethodName());

  MobFile mobFile =
      new MobFile(new HStoreFile(fs, writer.getPath(), conf, cacheConf, BloomType.NONE, true));
  assertNotNull(mobFile.getScanner());
  assertTrue(mobFile.getScanner() instanceof StoreFileScanner);
}

项目：hbase 文件：TestPartitionedMobCompactor.java

/**
 * Gets the number of del cell in the del files
 * @param paths the del file paths
 * @return the cell size
 */
private int countDelCellsInDelFiles(List<Path> paths) throws IOException {
  List<HStoreFile> sfs = new ArrayList<>();
  int size = 0;
  for (Path path : paths) {
    HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true);
    sfs.add(sf);
  }
  List<KeyValueScanner> scanners = new ArrayList<>(StoreFileScanner.getScannersForStoreFiles(sfs,
    false, true, false, false, HConstants.LATEST_TIMESTAMP));
  long timeToPurgeDeletes = Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
  long ttl = HStore.determineTTLFromFamily(hcd);
  ScanInfo scanInfo = new ScanInfo(conf, hcd, ttl, timeToPurgeDeletes, CellComparatorImpl.COMPARATOR);
  StoreScanner scanner = new StoreScanner(scanInfo, ScanType.COMPACT_RETAIN_DELETES, scanners);
  List<Cell> results = new ArrayList<>();
  boolean hasMore = true;

  while (hasMore) {
    hasMore = scanner.next(results);
    size += results.size();
    results.clear();
  }
  scanner.close();
  return size;
}

项目：HBase-LOB 文件：MobFile.java

/**
 * Reads a cell from the mob file.
 * @param search The KeyValue need to be searched in the mob file.
 * @param cacheMobBlocks Should this scanner cache blocks.
 * @return The KeyValue in the mob file.
 * @throws IOException
 */
public KeyValue readCell(KeyValue search, boolean cacheMobBlocks) throws IOException {
  KeyValue result = null;
  StoreFileScanner scanner = null;
  List<StoreFile> sfs = new ArrayList<StoreFile>();
  sfs.add(sf);
  try {
    List<StoreFileScanner> sfScanners = StoreFileScanner.getScannersForStoreFiles(sfs,
        cacheMobBlocks, true, false, null, sf.getMaxMemstoreTS());
    if (!sfScanners.isEmpty()) {
      scanner = sfScanners.get(0);
      if (scanner.seek(search)) {
        result = scanner.peek();
      }
    }
  } finally {
    if (scanner != null) {
      scanner.close();
    }
  }
  return result;
}

项目：HBase-LOB 文件：TestMobFile.java

@Test
public void testGetScanner() throws Exception {
  FileSystem fs = FileSystem.get(conf);
  Path testDir = FSUtils.getRootDir(conf);
  Path outputDir = new Path(new Path(testDir, TABLE), FAMILY);
  HFileContext meta = new HFileContextBuilder().withBlockSize(8*1024).build();
  StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs)
          .withOutputDir(outputDir)
          .withFileContext(meta)
          .build();
  MobTestUtil.writeStoreFile(writer, getName());

  MobFile mobFile = new MobFile(new StoreFile(fs, writer.getPath(),
      conf, cacheConf, BloomType.NONE));
  assertNotNull(mobFile.getScanner());
  assertTrue(mobFile.getScanner() instanceof StoreFileScanner);
}

项目：ditb 文件：Compactor.java

/**
 * Creates file scanners for compaction.
 *
 * @param filesToCompact Files.
 * @return Scanners.
 */
protected List<StoreFileScanner> createFileScanners(final Collection<StoreFile> filesToCompact,
    long smallestReadPoint, boolean useDropBehind) throws IOException {
  return StoreFileScanner.getScannersForStoreFiles(filesToCompact,
  /* cache blocks = */false,
  /* use pread = */false,
  /* is compaction */true,
  /* use Drop Behind */useDropBehind, smallestReadPoint);
}

项目：ditb 文件：CompactJobQueue.java

private void winterTestingStoreFile(StoreFile sf) throws IOException {
  StoreFileScanner compactedFileScanner = sf.getReader().getStoreFileScanner(false, false);
  KeyValue startKey =
      KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW, HConstants.LATEST_TIMESTAMP);
  compactedFileScanner.seek(startKey);
  KeyValue kv;
  int n = 0;
  while ((kv = (KeyValue) compactedFileScanner.next()) != null) {
    LOG.info("LCDBG, show kv: " + Bytes.toInt(kv.getRow()));
    ++n;
  }
  LOG.info("LCDBG, reader has: " + n + " in " + sf.getPath());
  compactedFileScanner.close();
}

项目：ditb 文件：LMDIndexDirectStoreFileScanner.java

private StoreFileScanner getStoreFileScanner(StoreFile storeFile) throws IOException {
  StoreFile.Reader r = storeFile.createReader(canUseDrop);
  r.setReplicaStoreFile(isPrimaryReplica);
  StoreFileScanner scanner = r.getStoreFileScanner(cacheBlocks, usePread, isCompaction, readPt);
  scanner.setScanQueryMatcher(matcher);
  return scanner;
}

项目：ditb 文件：LMDIndexDirectStoreFileScanner.java

private List<byte[]> getRawRowkeyList(StoreFileScanner secondaryScanner,
    List<LMDBucket> bucketList, MDRange[] ranges) throws IOException {
  List<byte[]> rowkeyList = new ArrayList<>();
  for (LMDBucket bucket : bucketList) {
    Cell peekCell = secondaryScanner.peek();
    if (peekCell != null && Bytes.compareTo(bucket.getStartKey(), peekCell.getRow()) == 0) {
    } else {
      secondaryScanner.reseek(new KeyValue(bucket.getStartKey(), LMDIndexConstants.FAMILY,
          LMDIndexConstants.QUALIFIER));
    }
    Cell cell;
    while ((cell = secondaryScanner.peek()) != null) {
      if (Bytes.compareTo(bucket.getStopKey(), cell.getRow()) < 0) {
        break;
      }
      boolean included = true;
      int[] values = MDUtils.bitwiseUnzip(cell.getRow(), ranges.length);
      for (int i = 0; i < ranges.length; i++) {
        if (!ranges[i].include(values[i])) {
          included = false;
          break;
        }
      }
      if (included) {
        //          System.out.println("adding key: " + Bytes.toInt(cell.getQualifier()));
        rowkeyList.add(cell.getQualifier());
        secondaryScanner.next();
      } else {
        //          System.out.println("skipped key: " + Bytes.toInt(cell.getQualifier()));
        secondaryScanner.reseek(
            new KeyValue(cell.getRow(), LMDIndexConstants.FAMILY, LMDIndexConstants.QUALIFIER));
      }
    }
  }
  return rowkeyList;
}

项目：ditb 文件：TestStripeCompactionPolicy.java

private static StoreFile createFile(long size) throws Exception {
  StoreFile sf = mock(StoreFile.class);
  when(sf.getPath()).thenReturn(new Path("moo"));
  StoreFile.Reader r = mock(StoreFile.Reader.class);
  when(r.getEntries()).thenReturn(size);
  when(r.length()).thenReturn(size);
  when(r.getBloomFilterType()).thenReturn(BloomType.NONE);
  when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class));
  when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong())).thenReturn(
    mock(StoreFileScanner.class));
  when(sf.getReader()).thenReturn(r);
  when(sf.createReader(anyBoolean())).thenReturn(r);
  when(sf.createReader()).thenReturn(r);
  return sf;
}

项目：LCIndex-HBase-0.94.16 文件：ReadHFile.java

private void readHFile(Configuration hadoopConf, Configuration hbaseConf, String fsStr,
    String fileName) throws IOException {
  CacheConfig tmpCacheConfig = new CacheConfig(hbaseConf);
  FileSystem fs = null;
  if (fsStr.equalsIgnoreCase("local")) {
    fs = LocalFileSystem.getLocal(hadoopConf);
  } else {
    fs = FileSystem.get(hadoopConf);
  }
  Path path = new Path(fileName);
  if (!fs.exists(path)) {
    System.out.println("WinterTestAID file not exists: " + path);
  } else {
    System.out.println("WinterTestAID reading lccindex hfile: " + path);
    StoreFile sf = new StoreFile(fs, path, hbaseConf, tmpCacheConfig, BloomType.NONE, null);
    Reader reader = sf.createReader();
    System.out.println("WinterTestAID store file attr: " + sf.mWinterGetAttribute());
    StoreFileScanner sss = reader.getStoreFileScanner(false, false);
    sss.seek(KeyValue.LOWESTKEY);
    System.out.println("WinterTestAID store peek value: "
        + LCCIndexConstant.mWinterToPrint(sss.peek()));
    KeyValue kv;
    int counter = 0, printInterval = 1, totalSize = 0;
    while ((kv = sss.next()) != null) {
      if (counter == 0) {
        counter = printInterval;
        System.out
            .println("WinterTestAID hfile keyvalue: " + LCCIndexConstant.mWinterToPrint(kv));
      }
      --counter;
      ++totalSize;
    }
    sss.close();
    reader.close(false);
    System.out.println("WinterTestAID total size: " + totalSize);
    System.out.println("WinterTestAID winter inner mWinterGetScannersForStoreFiles start: "
        + LCCIndexConstant.convertUnknownBytes(reader.getFirstKey()));
  }
}

项目：LCIndex-HBase-0.94.16 文件：WinterTestAID.java

public static void readHFile(Configuration hbaseConf, Path hfilePath) throws IOException {
  CacheConfig tmpCacheConfig = new CacheConfig(hbaseConf);
  FileSystem hdfs = getHDFS();
  if (!hdfs.exists(hfilePath)) {
    System.out.println("WinterTestAID file not exists: " + hfilePath);
  } else {
    System.out.println("WinterTestAID reading lccindex hfile: " + hfilePath);
    StoreFile sf = new StoreFile(hdfs, hfilePath, hbaseConf, tmpCacheConfig, BloomType.NONE, null);
    Reader reader = sf.createReader();
    System.out.println("WinterTestAID store file attr: " + sf.mWinterGetAttribute());
    StoreFileScanner sss = reader.getStoreFileScanner(false, false);
    sss.seek(KeyValue.LOWESTKEY);
    System.out.println("WinterTestAID store peek value: "
        + LCCIndexConstant.mWinterToPrint(sss.peek()));
    KeyValue kv;
    int counter = 0, printInterval = 1, totalSize = 0;
    while ((kv = sss.next()) != null) {
      if (counter == 0) {
        counter = printInterval;
        System.out
            .println("WinterTestAID hfile keyvalue: " + LCCIndexConstant.mWinterToPrint(kv));
      }
      --counter;
      ++totalSize;
    }
    sss.close();
    reader.close(false);
    System.out.println("WinterTestAID total size: " + totalSize);
    System.out.println("WinterTestAID winter inner mWinterGetScannersForStoreFiles start: "
        + LCCIndexConstant.convertUnknownBytes(reader.getFirstKey()));
  }
}

项目：pbase 文件：TestStripeCompactionPolicy.java

private static StoreFile createFile(long size) throws Exception {
  StoreFile sf = mock(StoreFile.class);
  when(sf.getPath()).thenReturn(new Path("moo"));
  StoreFile.Reader r = mock(StoreFile.Reader.class);
  when(r.getEntries()).thenReturn(size);
  when(r.length()).thenReturn(size);
  when(r.getBloomFilterType()).thenReturn(BloomType.NONE);
  when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class));
  when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong())).thenReturn(
    mock(StoreFileScanner.class));
  when(sf.getReader()).thenReturn(r);
  when(sf.createReader()).thenReturn(r);
  return sf;
}

项目：hbase 文件：PartitionedMobCompactor.java

/**
 * Creates a store scanner.
 * @param filesToCompact The files to be compacted.
 * @param scanType The scan type.
 * @return The store scanner.
 * @throws IOException if IO failure is encountered
 */
private StoreScanner createScanner(List<HStoreFile> filesToCompact, ScanType scanType)
    throws IOException {
  List<StoreFileScanner> scanners = StoreFileScanner.getScannersForStoreFiles(filesToCompact,
    false, true, false, false, HConstants.LATEST_TIMESTAMP);
  long ttl = HStore.determineTTLFromFamily(column);
  ScanInfo scanInfo = new ScanInfo(conf, column, ttl, 0, CellComparator.getInstance());
  return new StoreScanner(scanInfo, scanType, scanners);
}

项目：hbase 文件：MobFile.java

/**
 * Internal use only. This is used by the sweeper.
 *
 * @return The store file scanner.
 * @throws IOException
 */
public StoreFileScanner getScanner() throws IOException {
  List<HStoreFile> sfs = new ArrayList<>();
  sfs.add(sf);
  List<StoreFileScanner> sfScanners = StoreFileScanner.getScannersForStoreFiles(sfs, false, true,
      false, false, sf.getMaxMemStoreTS());

  return sfScanners.get(0);
}

项目：hbase 文件：StripeCompactor.java

@Override
public InternalScanner createScanner(ScanInfo scanInfo, List<StoreFileScanner> scanners,
    ScanType scanType, FileDetails fd, long smallestReadPoint) throws IOException {
  return (majorRangeFromRow == null)
      ? StripeCompactor.this.createScanner(store, scanInfo, scanners, scanType,
        smallestReadPoint, fd.earliestPutTs)
      : StripeCompactor.this.createScanner(store, scanInfo, scanners, smallestReadPoint,
        fd.earliestPutTs, majorRangeFromRow, majorRangeToRow);
}

项目：hbase 文件：TestStripeCompactionPolicy.java

private static HStoreFile createFile(long size) throws Exception {
  HStoreFile sf = mock(HStoreFile.class);
  when(sf.getPath()).thenReturn(new Path("moo"));
  StoreFileReader r = mock(StoreFileReader.class);
  when(r.getEntries()).thenReturn(size);
  when(r.length()).thenReturn(size);
  when(r.getBloomFilterType()).thenReturn(BloomType.NONE);
  when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class));
  when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyLong(),
    anyBoolean())).thenReturn(mock(StoreFileScanner.class));
  when(sf.getReader()).thenReturn(r);
  when(sf.getBulkLoadTimestamp()).thenReturn(OptionalLong.empty());
  return sf;
}

项目：hbase 文件：TestCompactor.java

public static HStoreFile createDummyStoreFile(long maxSequenceId) throws Exception {
  // "Files" are totally unused, it's Scanner class below that gives compactor fake KVs.
  // But compaction depends on everything under the sun, so stub everything with dummies.
  HStoreFile sf = mock(HStoreFile.class);
  StoreFileReader r = mock(StoreFileReader.class);
  when(r.length()).thenReturn(1L);
  when(r.getBloomFilterType()).thenReturn(BloomType.NONE);
  when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class));
  when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyLong(),
    anyBoolean())).thenReturn(mock(StoreFileScanner.class));
  when(sf.getReader()).thenReturn(r);
  when(sf.getMaxSequenceId()).thenReturn(maxSequenceId);
  return sf;
}

项目：bigbase 文件：BlockCacheStorageSimpleRegionTests.java

/**
 * _test store file scanner.
 *
 * @throws IOException Signals that an I/O exception has occurred.
 */
public void testStoreFileScanner() throws IOException
{
    LOG.info("StoreFileScanner full starts");
    long start = System.currentTimeMillis();
    Map<byte[], Store> storeMap = region.getStores();
    Collection<Store> stores = storeMap.values();
    Store store = stores.iterator().next();
    Collection<StoreFile> files = store.getStorefiles();
    start = System.currentTimeMillis();        
    int count = 0;
    for(StoreFile file: files){
      LOG.info(file.getPath());
      StoreFile.Reader reader = file.createReader();
      StoreFileScanner scanner = reader.getStoreFileScanner(false, false);          
      scanner.seek(KeyValue.LOWESTKEY);
      while(scanner.next() != null){
        count++; 
      }
      scanner.close();
      reader.close(false);
    }

    long end = System.currentTimeMillis();
    LOG.info("StoreFileScanner full finished in "+(end-start)+"ms. Found "+count+" records");


}

项目：bigbase 文件：BlockCacheStorageSimpleRegionTests.java

/**
 * _test random direct scanners.
 * 
 * FAIL after compaction
 *
 * @throws IOException Signals that an I/O exception has occurred.
 */
public void _testRandomDirectScanners() throws IOException
{
  LOG.info("Random StoreFile scanners . Running "+(N/10)+ " of size "+M+ " scanners");
  Random r = new Random();
  long totalScanned =0;
  long start = System.currentTimeMillis();
  Map<byte[], Store> storeMap = region.getStores();
  Collection<Store> stores = storeMap.values();
  Store store = stores.iterator().next();
  Collection<StoreFile> files = store.getStorefiles();
  StoreFile[] array = new StoreFile[files.size()];
  files.toArray(array);
  for(int i =0; i < N/10; i++){

    StoreFile file = array[r.nextInt(files.size())];        
    byte[] row = (ROW_PREFIX+r.nextInt(N)).getBytes();         
    StoreFile.Reader reader = file.createReader();
    StoreFileScanner scanner = reader.getStoreFileScanner(false, false);
    KeyValue kv = new KeyValue(row, CF, CQQ[0]);
    //LOG.info(i+" Seek "+kv);
    scanner.seek(kv);
    int total = 0;             
    while(total ++ < M && scanner.next() != null){
      totalScanned++; 
    }
    if(i % 100000 == 0 && i > 0){
      LOG.info("Scanner "+i+" scanned="+totalScanned+" avg per scanner="+(totalScanned/i));
    }
    scanner.close();
    reader.close(false);

  }
  LOG.info("Random StoreFile scanners done. "+(N/10)+" in "+
      (System.currentTimeMillis() - start)+"ms. Total scanned="+totalScanned+" Avg. ="+((totalScanned * 10)/ N));
}

项目：bigbase 文件：BlockCacheSimpleRegionTests.java

/**
 * _test store file scanner.
 *
 * @throws IOException Signals that an I/O exception has occurred.
 */
public void _testStoreFileScanner() throws IOException
{
    LOG.info("StoreFileScanner full starts");
    long start = System.currentTimeMillis();
    Map<byte[], Store> storeMap = region.getStores();
    Collection<Store> stores = storeMap.values();
    Store store = stores.iterator().next();
    Collection<StoreFile> files = store.getStorefiles();
    start = System.currentTimeMillis();        
    int count = 0;
    for(StoreFile file: files){
      LOG.info(file.getPath());
      StoreFile.Reader reader = file.createReader();
      StoreFileScanner scanner = reader.getStoreFileScanner(false, false);          
      scanner.seek(KeyValue.LOWESTKEY);
      while(scanner.next() != null){
        count++; 
      }
      scanner.close();
      reader.close(true);
    }

    long end = System.currentTimeMillis();
    LOG.info("StoreFileScanner full finished in "+(end-start)+"ms. Found "+count+" records");


}

项目：bigbase 文件：BlockCacheSimpleRegionTests.java

/**
 * _test random direct scanners.
 *
 * @throws IOException Signals that an I/O exception has occurred.
 */
public void _testRandomDirectScanners() throws IOException
{
  LOG.info("Random StoreFile scanners . Running "+(N/10)+ " of size "+M+ " scanners");
  Random r = new Random();
  long totalScanned =0;
  long start = System.currentTimeMillis();
  Map<byte[], Store> storeMap = region.getStores();
  Collection<Store> stores = storeMap.values();
  Store store = stores.iterator().next();
  Collection<StoreFile> files = store.getStorefiles();
  StoreFile[] array = new StoreFile[files.size()];
  files.toArray(array);
  for(int i =0; i < N/10; i++){

    StoreFile file = array[r.nextInt(files.size())];        
    byte[] row = (ROW_PREFIX+r.nextInt(N)).getBytes();         
    StoreFile.Reader reader = file.createReader();
    StoreFileScanner scanner = reader.getStoreFileScanner(false, false);
    KeyValue kv = new KeyValue(row, CF, CQQ[0]);
    scanner.seek(kv);
    int total = 0;             
    while(total ++ < M && scanner.next() != null){
      totalScanned++; 
    }
    if(i % 100000 == 0 && i > 0){
      LOG.info("Scanner "+i+" scanned="+totalScanned+" avg per scanner="+(totalScanned/i));
    }
    scanner.close();
    reader.close(false);

  }
  LOG.info("Random StoreFile scanners done. "+(N/10)+" in "+
      (System.currentTimeMillis() - start)+"ms. Total scanned="+totalScanned+" Avg. ="+((totalScanned * 10)/ N));
}

项目：c5 文件：Compactor.java

/**
 * @param scanners Store file scanners.
 * @param scanType Scan type.
 * @param smallestReadPoint Smallest MVCC read point.
 * @param earliestPutTs Earliest put across all files.
 * @return A compaction scanner.
 */
protected InternalScanner createScanner(Store store, List<StoreFileScanner> scanners,
    ScanType scanType, long smallestReadPoint, long earliestPutTs) throws IOException {
  Scan scan = new Scan();
  scan.setMaxVersions(store.getFamily().getMaxVersions());
  return new StoreScanner(store, store.getScanInfo(), scan, scanners,
      scanType, smallestReadPoint, earliestPutTs);
}

项目：HBase-LOB 文件：MobFile.java

/**
 * Internal use only. This is used by the sweeper.
 *
 * @return The store file scanner.
 * @throws IOException
 */
public StoreFileScanner getScanner() throws IOException {
  List<StoreFile> sfs = new ArrayList<StoreFile>();
  sfs.add(sf);
  List<StoreFileScanner> sfScanners = StoreFileScanner.getScannersForStoreFiles(sfs, false, true,
      false, null, sf.getMaxMemstoreTS());

  return sfScanners.get(0);
}

项目：pbase 文件：Compactor.java

/**
 * Creates file scanners for compaction.
 * @param filesToCompact Files.
 * @return Scanners.
 */
protected List<StoreFileScanner> createFileScanners(
    final Collection<StoreFile> filesToCompact, long smallestReadPoint) throws IOException {
  return StoreFileScanner.getScannersForStoreFiles(filesToCompact, false, false, true,
    smallestReadPoint);
}

项目：HIndex 文件：Compactor.java

/**
 * Creates file scanners for compaction.
 * @param filesToCompact Files.
 * @return Scanners.
 */
protected List<StoreFileScanner> createFileScanners(
    final Collection<StoreFile> filesToCompact, long smallestReadPoint) throws IOException {
  return StoreFileScanner.getScannersForStoreFiles(filesToCompact, false, false, true,
    smallestReadPoint);
}

项目：HIndex 文件：DefaultCompactor.java

/**
 * Do a minor/major compaction on an explicit set of storefiles from a Store.
 */
public List<Path> compact(final CompactionRequest request) throws IOException {
  FileDetails fd = getFileDetails(request.getFiles(), request.isMajor());
  this.progress = new CompactionProgress(fd.maxKeyCount);

  // Find the smallest read point across all the Scanners.
  long smallestReadPoint = getSmallestReadPoint();
  List<StoreFileScanner> scanners = createFileScanners(request.getFiles(), smallestReadPoint);

  StoreFile.Writer writer = null;
  List<Path> newFiles = new ArrayList<Path>();
  try {
    InternalScanner scanner = null;
    try {
      /* Include deletes, unless we are doing a major compaction */
      ScanType scanType =
          request.isMajor() ? ScanType.COMPACT_DROP_DELETES : ScanType.COMPACT_RETAIN_DELETES;
      scanner = preCreateCoprocScanner(request, scanType, fd.earliestPutTs, scanners);
      if (scanner == null) {
        scanner = createScanner(store, scanners, scanType, smallestReadPoint, fd.earliestPutTs);
      }
      scanner = postCreateCoprocScanner(request, scanType, scanner);
      if (scanner == null) {
        // NULL scanner returned from coprocessor hooks means skip normal processing.
        return newFiles;
      }
      // Create the writer even if no kv(Empty store file is also ok),
      // because we need record the max seq id for the store file, see HBASE-6059
      writer = store.createWriterInTmp(fd.maxKeyCount, this.compactionCompression, true,
          fd.maxMVCCReadpoint >= smallestReadPoint, fd.maxTagsLength > 0);
      boolean finished = performCompaction(scanner, writer, smallestReadPoint);
      if (!finished) {
        writer.close();
        store.getFileSystem().delete(writer.getPath(), false);
        writer = null;
        throw new InterruptedIOException( "Aborting compaction of store " + store +
            " in region " + store.getRegionInfo().getRegionNameAsString() +
            " because it was interrupted.");
       }
     } finally {
       if (scanner != null) {
         scanner.close();
       }
    }
  } finally {
    if (writer != null) {
      writer.appendMetadata(fd.maxSeqId, request.isMajor());
      writer.close();
      newFiles.add(writer.getPath());
    }
  }
  return newFiles;
}

项目：hbase 文件：DefaultMobStoreCompactor.java

@Override
public InternalScanner createScanner(ScanInfo scanInfo, List<StoreFileScanner> scanners,
    ScanType scanType, FileDetails fd, long smallestReadPoint) throws IOException {
  return new StoreScanner(store, scanInfo, scanners, scanType, smallestReadPoint,
      fd.earliestPutTs);
}

项目：hbase 文件：Compactor.java

/**
 * Creates file scanners for compaction.
 * @param filesToCompact Files.
 * @return Scanners.
 */
private List<StoreFileScanner> createFileScanners(Collection<HStoreFile> filesToCompact,
    long smallestReadPoint, boolean useDropBehind) throws IOException {
  return StoreFileScanner.getScannersForCompaction(filesToCompact, useDropBehind,
    smallestReadPoint);
}

项目：hbase 文件：Compactor.java

InternalScanner createScanner(ScanInfo scanInfo, List<StoreFileScanner> scanners, ScanType scanType,
FileDetails fd, long smallestReadPoint) throws IOException;

项目：hbase 文件：Compactor.java

@Override
public InternalScanner createScanner(ScanInfo scanInfo, List<StoreFileScanner> scanners,
    ScanType scanType, FileDetails fd, long smallestReadPoint) throws IOException {
  return Compactor.this.createScanner(store, scanInfo, scanners, scanType, smallestReadPoint,
    fd.earliestPutTs);
}

项目：PyroDB 文件：Compactor.java

/**
 * Creates file scanners for compaction.
 * @param filesToCompact Files.
 * @return Scanners.
 */
protected List<StoreFileScanner> createFileScanners(
    final Collection<StoreFile> filesToCompact, long smallestReadPoint) throws IOException {
  return StoreFileScanner.getScannersForStoreFiles(filesToCompact, false, false, true,
    smallestReadPoint);
}

项目：PyroDB 文件：DefaultCompactor.java

/**
 * Do a minor/major compaction on an explicit set of storefiles from a Store.
 */
public List<Path> compact(final CompactionRequest request) throws IOException {
  FileDetails fd = getFileDetails(request.getFiles(), request.isAllFiles());
  this.progress = new CompactionProgress(fd.maxKeyCount);

  // Find the smallest read point across all the Scanners.
  long smallestReadPoint = getSmallestReadPoint();
  List<StoreFileScanner> scanners = createFileScanners(request.getFiles(), smallestReadPoint);

  StoreFile.Writer writer = null;
  List<Path> newFiles = new ArrayList<Path>();
  try {
    InternalScanner scanner = null;
    try {
      /* Include deletes, unless we are doing a compaction of all files */
      ScanType scanType =
          request.isAllFiles() ? ScanType.COMPACT_DROP_DELETES : ScanType.COMPACT_RETAIN_DELETES;
      scanner = preCreateCoprocScanner(request, scanType, fd.earliestPutTs, scanners);
      if (scanner == null) {
        scanner = createScanner(store, scanners, scanType, smallestReadPoint, fd.earliestPutTs);
      }
      scanner = postCreateCoprocScanner(request, scanType, scanner);
      if (scanner == null) {
        // NULL scanner returned from coprocessor hooks means skip normal processing.
        return newFiles;
      }
      // Create the writer even if no kv(Empty store file is also ok),
      // because we need record the max seq id for the store file, see HBASE-6059
      writer = store.createWriterInTmp(fd.maxKeyCount, this.compactionCompression, true,
          fd.maxMVCCReadpoint >= smallestReadPoint, fd.maxTagsLength > 0);
      boolean finished = performCompaction(scanner, writer, smallestReadPoint);
      if (!finished) {
        writer.close();
        store.getFileSystem().delete(writer.getPath(), false);
        writer = null;
        throw new InterruptedIOException( "Aborting compaction of store " + store +
            " in region " + store.getRegionInfo().getRegionNameAsString() +
            " because it was interrupted.");
       }
     } finally {
       if (scanner != null) {
         scanner.close();
       }
    }
  } finally {
    if (writer != null) {
      writer.appendMetadata(fd.maxSeqId, request.isAllFiles());
      writer.close();
      newFiles.add(writer.getPath());
    }
  }
  return newFiles;
}

项目：c5 文件：Compactor.java

protected List<StoreFileScanner> createFileScanners(
    final Collection<StoreFile> filesToCompact) throws IOException {
  return StoreFileScanner.getScannersForStoreFiles(filesToCompact, false, false, true);
}

项目：c5 文件：Compactor.java

protected InternalScanner preCreateCoprocScanner(final CompactionRequest request,
    ScanType scanType, long earliestPutTs,  List<StoreFileScanner> scanners) throws IOException {
  if (store.getCoprocessorHost() == null) return null;
  return store.getCoprocessorHost()
      .preCompactScannerOpen(store, scanners, scanType, earliestPutTs, request);
}

项目：c5 文件：DefaultCompactor.java

/**
 * Do a minor/major compaction on an explicit set of storefiles from a Store.
 */
public List<Path> compact(final CompactionRequest request) throws IOException {
  FileDetails fd = getFileDetails(request.getFiles(), request.isMajor());
  this.progress = new CompactionProgress(fd.maxKeyCount);

  List<StoreFileScanner> scanners = createFileScanners(request.getFiles());

  StoreFile.Writer writer = null;
  List<Path> newFiles = new ArrayList<Path>();
  // Find the smallest read point across all the Scanners.
  long smallestReadPoint = setSmallestReadPoint();
  try {
    InternalScanner scanner = null;
    try {
      /* Include deletes, unless we are doing a major compaction */
      ScanType scanType =
          request.isMajor() ? ScanType.COMPACT_DROP_DELETES : ScanType.COMPACT_RETAIN_DELETES;
      scanner = preCreateCoprocScanner(request, scanType, fd.earliestPutTs, scanners);
      if (scanner == null) {
        scanner = createScanner(store, scanners, scanType, smallestReadPoint, fd.earliestPutTs);
      }
      scanner = postCreateCoprocScanner(request, scanType, scanner);
      if (scanner == null) {
        // NULL scanner returned from coprocessor hooks means skip normal processing.
        return newFiles;
      }
      // Create the writer even if no kv(Empty store file is also ok),
      // because we need record the max seq id for the store file, see HBASE-6059
      writer = store.createWriterInTmp(fd.maxKeyCount, this.compactionCompression, true,
          fd.maxMVCCReadpoint >= smallestReadPoint);
      boolean finished = performCompaction(scanner, writer, smallestReadPoint);
      if (!finished) {
        abortWriter(writer);
        writer = null;
        throw new InterruptedIOException( "Aborting compaction of store " + store +
            " in region " + store.getRegionInfo().getRegionNameAsString() +
            " because it was interrupted.");
       }
     } finally {
       if (scanner != null) {
         scanner.close();
       }
    }
  } finally {
    if (writer != null) {
      writer.appendMetadata(fd.maxSeqId, request.isMajor());
      writer.close();
      newFiles.add(writer.getPath());
    }
  }
  return newFiles;
}

项目：ditb 文件：Compactor.java

/**
 * @param store             store
 * @param scanners          Store file scanners.
 * @param scanType          Scan type.
 * @param smallestReadPoint Smallest MVCC read point.
 * @param earliestPutTs     Earliest put across all files.
 * @return A compaction scanner.
 */
protected InternalScanner createScanner(Store store, List<StoreFileScanner> scanners,
    ScanType scanType, long smallestReadPoint, long earliestPutTs) throws IOException {
  Scan scan = new Scan();
  scan.setMaxVersions(store.getFamily().getMaxVersions());
  return new StoreScanner(store, store.getScanInfo(), scan, scanners, scanType, smallestReadPoint,
      earliestPutTs);
}

项目：ditb 文件：Compactor.java

/**
 * @param store              The store.
 * @param scanners           Store file scanners.
 * @param smallestReadPoint  Smallest MVCC read point.
 * @param earliestPutTs      Earliest put across all files.
 * @param dropDeletesFromRow Drop deletes starting with this row, inclusive. Can be null.
 * @param dropDeletesToRow   Drop deletes ending with this row, exclusive. Can be null.
 * @return A compaction scanner.
 */
protected InternalScanner createScanner(Store store, List<StoreFileScanner> scanners,
    long smallestReadPoint, long earliestPutTs, byte[] dropDeletesFromRow,
    byte[] dropDeletesToRow) throws IOException {
  Scan scan = new Scan();
  scan.setMaxVersions(store.getFamily().getMaxVersions());
  return new StoreScanner(store, store.getScanInfo(), scan, scanners, smallestReadPoint,
      earliestPutTs, dropDeletesFromRow, dropDeletesToRow);
}