1. 包列表
  2. 类列表
  3. org.apache.hadoop.hdfs.server.common.Util

Java 类org.apache.hadoop.hdfs.server.common.Util 实例源码

项目：hadoop    文件：NNStorage.java    
/**
 * Return the storage directory corresponding to the passed URI
 * @param uri URI of a storage directory
 * @return The matching storage directory or null if none found
 */
StorageDirectory getStorageDirectory(URI uri) {
  try {
    uri = Util.fileAsURI(new File(uri));
    Iterator<StorageDirectory> it = dirIterator();
    for (; it.hasNext(); ) {
      StorageDirectory sd = it.next();
      if (Util.fileAsURI(sd.getRoot()).equals(uri)) {
        return sd;
      }
    }
  } catch (IOException ioe) {
    LOG.warn("Error converting file to URI", ioe);
  }
  return null;
}
项目：hadoop    文件：NNStorage.java    
/**
 * Return the list of locations being used for a specific purpose.
 * i.e. Image or edit log storage.
 *
 * @param dirType Purpose of locations requested.
 * @throws IOException
 */
Collection<URI> getDirectories(NameNodeDirType dirType)
    throws IOException {
  ArrayList<URI> list = new ArrayList<URI>();
  Iterator<StorageDirectory> it = (dirType == null) ? dirIterator() :
                                  dirIterator(dirType);
  for ( ;it.hasNext(); ) {
    StorageDirectory sd = it.next();
    try {
      list.add(Util.fileAsURI(sd.getRoot()));
    } catch (IOException e) {
      throw new IOException("Exception while processing " +
          "StorageDirectory " + sd.getRoot(), e);
    }
  }
  return list;
}
项目：aliyun-oss-hadoop-fs    文件：NNStorage.java    
/**
 * Return the storage directory corresponding to the passed URI
 * @param uri URI of a storage directory
 * @return The matching storage directory or null if none found
 */
StorageDirectory getStorageDirectory(URI uri) {
  try {
    uri = Util.fileAsURI(new File(uri));
    Iterator<StorageDirectory> it = dirIterator();
    for (; it.hasNext(); ) {
      StorageDirectory sd = it.next();
      if (Util.fileAsURI(sd.getRoot()).equals(uri)) {
        return sd;
      }
    }
  } catch (IOException ioe) {
    LOG.warn("Error converting file to URI", ioe);
  }
  return null;
}
项目：aliyun-oss-hadoop-fs    文件：NNStorage.java    
/**
 * Return the list of locations being used for a specific purpose.
 * i.e. Image or edit log storage.
 *
 * @param dirType Purpose of locations requested.
 * @throws IOException
 */
Collection<URI> getDirectories(NameNodeDirType dirType)
    throws IOException {
  ArrayList<URI> list = new ArrayList<URI>();
  Iterator<StorageDirectory> it = (dirType == null) ? dirIterator() :
                                  dirIterator(dirType);
  for ( ;it.hasNext(); ) {
    StorageDirectory sd = it.next();
    try {
      list.add(Util.fileAsURI(sd.getRoot()));
    } catch (IOException e) {
      throw new IOException("Exception while processing " +
          "StorageDirectory " + sd.getRoot(), e);
    }
  }
  return list;
}
项目：big-c    文件：NNStorage.java    
/**
 * Return the storage directory corresponding to the passed URI
 * @param uri URI of a storage directory
 * @return The matching storage directory or null if none found
 */
StorageDirectory getStorageDirectory(URI uri) {
  try {
    uri = Util.fileAsURI(new File(uri));
    Iterator<StorageDirectory> it = dirIterator();
    for (; it.hasNext(); ) {
      StorageDirectory sd = it.next();
      if (Util.fileAsURI(sd.getRoot()).equals(uri)) {
        return sd;
      }
    }
  } catch (IOException ioe) {
    LOG.warn("Error converting file to URI", ioe);
  }
  return null;
}
项目：big-c    文件：NNStorage.java    
/**
 * Return the list of locations being used for a specific purpose.
 * i.e. Image or edit log storage.
 *
 * @param dirType Purpose of locations requested.
 * @throws IOException
 */
Collection<URI> getDirectories(NameNodeDirType dirType)
    throws IOException {
  ArrayList<URI> list = new ArrayList<URI>();
  Iterator<StorageDirectory> it = (dirType == null) ? dirIterator() :
                                  dirIterator(dirType);
  for ( ;it.hasNext(); ) {
    StorageDirectory sd = it.next();
    try {
      list.add(Util.fileAsURI(sd.getRoot()));
    } catch (IOException e) {
      throw new IOException("Exception while processing " +
          "StorageDirectory " + sd.getRoot(), e);
    }
  }
  return list;
}
项目：hadoop-2.6.0-cdh5.4.3    文件：NNStorage.java    
/**
 * Return the storage directory corresponding to the passed URI
 * @param uri URI of a storage directory
 * @return The matching storage directory or null if none found
 */
StorageDirectory getStorageDirectory(URI uri) {
  try {
    uri = Util.fileAsURI(new File(uri));
    Iterator<StorageDirectory> it = dirIterator();
    for (; it.hasNext(); ) {
      StorageDirectory sd = it.next();
      if (Util.fileAsURI(sd.getRoot()).equals(uri)) {
        return sd;
      }
    }
  } catch (IOException ioe) {
    LOG.warn("Error converting file to URI", ioe);
  }
  return null;
}
项目：hadoop-2.6.0-cdh5.4.3    文件：NNStorage.java    
/**
 * Return the list of locations being used for a specific purpose.
 * i.e. Image or edit log storage.
 *
 * @param dirType Purpose of locations requested.
 * @throws IOException
 */
Collection<URI> getDirectories(NameNodeDirType dirType)
    throws IOException {
  ArrayList<URI> list = new ArrayList<URI>();
  Iterator<StorageDirectory> it = (dirType == null) ? dirIterator() :
                                  dirIterator(dirType);
  for ( ;it.hasNext(); ) {
    StorageDirectory sd = it.next();
    try {
      list.add(Util.fileAsURI(sd.getRoot()));
    } catch (IOException e) {
      throw new IOException("Exception while processing " +
          "StorageDirectory " + sd.getRoot(), e);
    }
  }
  return list;
}
项目：hadoop-2.6.0-cdh5.4.3    文件：StorageLocation.java    
/**
 * Attempt to parse a storage uri with storage class and URI. The storage
 * class component of the uri is case-insensitive.
 *
 * @param rawLocation Location string of the format [type]uri, where [type] is
 *                    optional.
 * @return A StorageLocation object if successfully parsed, null otherwise.
 *         Does not throw any exceptions.
 */
public static StorageLocation parse(String rawLocation)
    throws IOException, SecurityException {
  Matcher matcher = regex.matcher(rawLocation);
  StorageType storageType = StorageType.DEFAULT;
  String location = rawLocation;

  if (matcher.matches()) {
    String classString = matcher.group(1);
    location = matcher.group(2);
    if (!classString.isEmpty()) {
      storageType = StorageType.valueOf(classString.toUpperCase());
    }
  }

  return new StorageLocation(storageType, Util.stringAsURI(location));
}
项目：hadoop-EAR    文件：TestBlockReplacement.java    
public void testThrottler() throws IOException {
  Configuration conf = new Configuration();
  FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
  long bandwidthPerSec = 1024*1024L;
  final long TOTAL_BYTES =6*bandwidthPerSec; 
  long bytesToSend = TOTAL_BYTES; 
  long start = Util.now();
  DataTransferThrottler throttler = new DataTransferThrottler(bandwidthPerSec);
  long totalBytes = 0L;
  long bytesSent = 1024*512L; // 0.5MB
  throttler.throttle(bytesSent);
  bytesToSend -= bytesSent;
  bytesSent = 1024*768L; // 0.75MB
  throttler.throttle(bytesSent);
  bytesToSend -= bytesSent;
  try {
    Thread.sleep(1000);
  } catch (InterruptedException ignored) {}
  throttler.throttle(bytesToSend);
  long end = Util.now();
  assertTrue(totalBytes*1000/(end-start)<=bandwidthPerSec);
}
项目：hadoop-EAR    文件：TestPreTransactionalServerLogReader.java    
@Test
public void testOneOperation() throws Exception {
  File editsDir = createEditsDir();
  DummyServerCore core = new DummyServerCore();
  EditLogFileOutputStream out = initEdits(editsDir);
  ServerLogReaderPreTransactional logReader = new ServerLogReaderPreTransactional(core,
      Util.stringAsURI(editsDir.getAbsolutePath()));
  core.logReader = logReader;
  Thread coreThread, logReaderThread;

  coreThread = new Thread(core);
  logReaderThread = new Thread(logReader);

  logReaderThread.start();
  coreThread.start();
  writeOperation(out, 1000, true);
  Thread.sleep(500);
  core.shutdown();
  logReaderThread.join();
  coreThread.join();

  Assert.assertEquals(1, core.notifications.size());
  Assert.assertEquals(1000, core.notifications.poll().txId);
}
项目：hadoop-EAR    文件：TestServerLogReaderVersion.java    
@Test
public void testMissingVersion() throws IOException {
  // -37 is pre-transactional layout
  int lv = 12345;
  StorageInfo si = new StorageInfo(lv, 10, 0);
  StorageDirectory sd = new NNStorage(si).new StorageDirectory(editsDir);
  format(sd);

  URI editsURI = Util.stringAsURI(sd.getRoot().getAbsolutePath());

  // remove verision file
  sd.getVersionFile().delete();
  try {
    NotifierUtils.getVersion(editsURI);
    fail("Should fail");
  } catch (Exception e) {
    LOG.info("expected exception: " + e.getMessage());
  }
}
项目：hadoop-EAR    文件：SnapshotNode.java    
@Override
public void createSnapshot(String snapshotId, boolean updateLeases) throws IOException {
  // Create new SnapshotStore
  SnapshotStorage ssStore = new SnapshotStorage(conf, Util.stringAsURI(tempDir));

  // Download image & edit files from namenode
  downloadSnapshotFiles(ssStore);

  // Merge image and edit files
  doMerge(ssStore);

  // Update file lengths for leased files (optional)
  if (updateLeases) {
    updateLeasedFiles(ssStore);
  }

  // Save snapshot
  saveSnapshot(ssStore, snapshotId);
  ssStore.close();
}
项目：hadoop-EAR    文件：TestAvatarStorageSetup.java    
@Before
public void setUp() throws Exception {
  Collection<String> dirList = new ArrayList<String>();
  dirList.add(imageLclName);
  dirList.add(imageShdName0);
  dirList.add(imageShdName1);
  dirList.add(editsLclName);
  dirList.add(editsShdName0);
  dirList.add(editsShdName1);  

  for(String name : dirList) {
    File dir = new File(name);
    FileUtil.fullyDelete(dir);
    dir.mkdirs();
  }

  imageLcl = Util.stringAsURI(imageLclName);
  imageShd0 = Util.stringAsURI(imageShdName0);
  imageShd1 = Util.stringAsURI(imageShdName1);

  editsLcl = Util.stringAsURI(editsLclName);
  editsShd0 = Util.stringAsURI(editsShdName0);
  editsShd1 = Util.stringAsURI(editsShdName1);

  conf = new Configuration();
}
项目：hadoop-EAR    文件：NNStorage.java    
/**
 * Return the storage directory corresponding to the passed URI
 * @param uri URI of a storage directory
 * @return The matching storage directory or null if none found
 */
StorageDirectory getStorageDirectory(URI uri) {
  try {
    uri = Util.fileAsURI(new File(uri));
    Iterator<StorageDirectory> it = dirIterator();
    for (; it.hasNext(); ) {
      StorageDirectory sd = it.next();
      if (Util.fileAsURI(sd.getRoot()).equals(uri)) {
        return sd;
      }
    }
  } catch (IOException ioe) {
    LOG.warn("Error converting file to URI", ioe);
  }
  return null;
}
项目：hadoop-EAR    文件：Balancer.java    
private long dispatchBlockMoves() throws InterruptedException {
  long bytesLastMoved = bytesMoved.get();

  Future<?>[] futures = new Future<?>[plan.sources.size()];
  int i=0;
  for (Source source : plan.sources) {
    futures[i++] = dispatcherExecutor.submit(
                      source.new BlockMoveDispatcher(Util.now()));
  }

  // wait for all dispatcher threads to finish
  for (Future<?> future : futures) {
    try {
      future.get();
    } catch (ExecutionException e) {
      LOG.warn("Dispatcher thread failed", e.getCause());
    }
  }

  // wait for all block moving to be done
  waitForMoveCompletion();

  return bytesMoved.get()-bytesLastMoved;
}
项目：hadoop-plus    文件：NNStorage.java    
/**
 * Return the storage directory corresponding to the passed URI
 * @param uri URI of a storage directory
 * @return The matching storage directory or null if none found
 */
StorageDirectory getStorageDirectory(URI uri) {
  try {
    uri = Util.fileAsURI(new File(uri));
    Iterator<StorageDirectory> it = dirIterator();
    for (; it.hasNext(); ) {
      StorageDirectory sd = it.next();
      if (Util.fileAsURI(sd.getRoot()).equals(uri)) {
        return sd;
      }
    }
  } catch (IOException ioe) {
    LOG.warn("Error converting file to URI", ioe);
  }
  return null;
}
项目：hadoop-plus    文件：NNStorage.java    
/**
 * Return the list of locations being used for a specific purpose.
 * i.e. Image or edit log storage.
 *
 * @param dirType Purpose of locations requested.
 * @throws IOException
 */
Collection<URI> getDirectories(NameNodeDirType dirType)
    throws IOException {
  ArrayList<URI> list = new ArrayList<URI>();
  Iterator<StorageDirectory> it = (dirType == null) ? dirIterator() :
                                  dirIterator(dirType);
  for ( ;it.hasNext(); ) {
    StorageDirectory sd = it.next();
    try {
      list.add(Util.fileAsURI(sd.getRoot()));
    } catch (IOException e) {
      throw new IOException("Exception while processing " +
          "StorageDirectory " + sd.getRoot(), e);
    }
  }
  return list;
}
项目：FlexMap    文件：NNStorage.java    
/**
 * Return the storage directory corresponding to the passed URI
 * @param uri URI of a storage directory
 * @return The matching storage directory or null if none found
 */
StorageDirectory getStorageDirectory(URI uri) {
  try {
    uri = Util.fileAsURI(new File(uri));
    Iterator<StorageDirectory> it = dirIterator();
    for (; it.hasNext(); ) {
      StorageDirectory sd = it.next();
      if (Util.fileAsURI(sd.getRoot()).equals(uri)) {
        return sd;
      }
    }
  } catch (IOException ioe) {
    LOG.warn("Error converting file to URI", ioe);
  }
  return null;
}
项目：FlexMap    文件：NNStorage.java    
/**
 * Return the list of locations being used for a specific purpose.
 * i.e. Image or edit log storage.
 *
 * @param dirType Purpose of locations requested.
 * @throws IOException
 */
Collection<URI> getDirectories(NameNodeDirType dirType)
    throws IOException {
  ArrayList<URI> list = new ArrayList<URI>();
  Iterator<StorageDirectory> it = (dirType == null) ? dirIterator() :
                                  dirIterator(dirType);
  for ( ;it.hasNext(); ) {
    StorageDirectory sd = it.next();
    try {
      list.add(Util.fileAsURI(sd.getRoot()));
    } catch (IOException e) {
      throw new IOException("Exception while processing " +
          "StorageDirectory " + sd.getRoot(), e);
    }
  }
  return list;
}
项目：FlexMap    文件：StorageLocation.java    
/**
 * Attempt to parse a storage uri with storage class and URI. The storage
 * class component of the uri is case-insensitive.
 *
 * @param rawLocation Location string of the format [type]uri, where [type] is
 *                    optional.
 * @return A StorageLocation object if successfully parsed, null otherwise.
 *         Does not throw any exceptions.
 */
public static StorageLocation parse(String rawLocation)
    throws IOException, SecurityException {
  Matcher matcher = regex.matcher(rawLocation);
  StorageType storageType = StorageType.DEFAULT;
  String location = rawLocation;

  if (matcher.matches()) {
    String classString = matcher.group(1);
    location = matcher.group(2);
    if (!classString.isEmpty()) {
      storageType = StorageType.valueOf(classString.toUpperCase());
    }
  }

  return new StorageLocation(storageType, Util.stringAsURI(location));
}
项目：hadoop-TCP    文件：NNStorage.java    
/**
 * Return the storage directory corresponding to the passed URI
 * @param uri URI of a storage directory
 * @return The matching storage directory or null if none found
 */
StorageDirectory getStorageDirectory(URI uri) {
  try {
    uri = Util.fileAsURI(new File(uri));
    Iterator<StorageDirectory> it = dirIterator();
    for (; it.hasNext(); ) {
      StorageDirectory sd = it.next();
      if (Util.fileAsURI(sd.getRoot()).equals(uri)) {
        return sd;
      }
    }
  } catch (IOException ioe) {
    LOG.warn("Error converting file to URI", ioe);
  }
  return null;
}
项目：hadoop-TCP    文件：NNStorage.java    
/**
 * Return the list of locations being used for a specific purpose.
 * i.e. Image or edit log storage.
 *
 * @param dirType Purpose of locations requested.
 * @throws IOException
 */
Collection<URI> getDirectories(NameNodeDirType dirType)
    throws IOException {
  ArrayList<URI> list = new ArrayList<URI>();
  Iterator<StorageDirectory> it = (dirType == null) ? dirIterator() :
                                  dirIterator(dirType);
  for ( ;it.hasNext(); ) {
    StorageDirectory sd = it.next();
    try {
      list.add(Util.fileAsURI(sd.getRoot()));
    } catch (IOException e) {
      throw new IOException("Exception while processing " +
          "StorageDirectory " + sd.getRoot(), e);
    }
  }
  return list;
}
项目：hadoop-on-lustre    文件：TestBlockReplacement.java    
public void testThrottler() throws IOException {
  Configuration conf = new Configuration();
  FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
  long bandwidthPerSec = 1024*1024L;
  final long TOTAL_BYTES =6*bandwidthPerSec; 
  long bytesToSend = TOTAL_BYTES; 
  long start = Util.now();
  BlockTransferThrottler throttler = new BlockTransferThrottler(bandwidthPerSec);
  long totalBytes = 0L;
  long bytesSent = 1024*512L; // 0.5MB
  throttler.throttle(bytesSent);
  bytesToSend -= bytesSent;
  bytesSent = 1024*768L; // 0.75MB
  throttler.throttle(bytesSent);
  bytesToSend -= bytesSent;
  try {
    Thread.sleep(1000);
  } catch (InterruptedException ignored) {}
  throttler.throttle(bytesToSend);
  long end = Util.now();
  assertTrue(totalBytes*1000/(end-start)<=bandwidthPerSec);
}
项目：hardfs    文件：NNStorage.java    
/**
 * Return the storage directory corresponding to the passed URI
 * @param uri URI of a storage directory
 * @return The matching storage directory or null if none found
 */
StorageDirectory getStorageDirectory(URI uri) {
  try {
    uri = Util.fileAsURI(new File(uri));
    Iterator<StorageDirectory> it = dirIterator();
    for (; it.hasNext(); ) {
      StorageDirectory sd = it.next();
      if (Util.fileAsURI(sd.getRoot()).equals(uri)) {
        return sd;
      }
    }
  } catch (IOException ioe) {
    LOG.warn("Error converting file to URI", ioe);
  }
  return null;
}
项目：hardfs    文件：NNStorage.java    
/**
 * Return the list of locations being used for a specific purpose.
 * i.e. Image or edit log storage.
 *
 * @param dirType Purpose of locations requested.
 * @throws IOException
 */
Collection<URI> getDirectories(NameNodeDirType dirType)
    throws IOException {
  ArrayList<URI> list = new ArrayList<URI>();
  Iterator<StorageDirectory> it = (dirType == null) ? dirIterator() :
                                  dirIterator(dirType);
  for ( ;it.hasNext(); ) {
    StorageDirectory sd = it.next();
    try {
      list.add(Util.fileAsURI(sd.getRoot()));
    } catch (IOException e) {
      throw new IOException("Exception while processing " +
          "StorageDirectory " + sd.getRoot(), e);
    }
  }
  return list;
}
项目：hadoop-on-lustre2    文件：NNStorage.java    
/**
 * Return the storage directory corresponding to the passed URI
 * @param uri URI of a storage directory
 * @return The matching storage directory or null if none found
 */
StorageDirectory getStorageDirectory(URI uri) {
  try {
    uri = Util.fileAsURI(new File(uri));
    Iterator<StorageDirectory> it = dirIterator();
    for (; it.hasNext(); ) {
      StorageDirectory sd = it.next();
      if (Util.fileAsURI(sd.getRoot()).equals(uri)) {
        return sd;
      }
    }
  } catch (IOException ioe) {
    LOG.warn("Error converting file to URI", ioe);
  }
  return null;
}
项目：hadoop-on-lustre2    文件：NNStorage.java    
/**
 * Return the list of locations being used for a specific purpose.
 * i.e. Image or edit log storage.
 *
 * @param dirType Purpose of locations requested.
 * @throws IOException
 */
Collection<URI> getDirectories(NameNodeDirType dirType)
    throws IOException {
  ArrayList<URI> list = new ArrayList<URI>();
  Iterator<StorageDirectory> it = (dirType == null) ? dirIterator() :
                                  dirIterator(dirType);
  for ( ;it.hasNext(); ) {
    StorageDirectory sd = it.next();
    try {
      list.add(Util.fileAsURI(sd.getRoot()));
    } catch (IOException e) {
      throw new IOException("Exception while processing " +
          "StorageDirectory " + sd.getRoot(), e);
    }
  }
  return list;
}
项目：hadoop-on-lustre2    文件：StorageLocation.java    
/**
 * Attempt to parse a storage uri with storage class and URI. The storage
 * class component of the uri is case-insensitive.
 *
 * @param rawLocation Location string of the format [type]uri, where [type] is
 *                    optional.
 * @return A StorageLocation object if successfully parsed, null otherwise.
 *         Does not throw any exceptions.
 */
static StorageLocation parse(String rawLocation) throws IOException {
  Matcher matcher = regex.matcher(rawLocation);
  StorageType storageType = StorageType.DEFAULT;
  String location = rawLocation;

  if (matcher.matches()) {
    String classString = matcher.group(1);
    location = matcher.group(2);
    if (!classString.isEmpty()) {
      storageType = StorageType.valueOf(classString.toUpperCase());
    }
  }

  return new StorageLocation(storageType, Util.stringAsURI(location));
}
项目：cumulus    文件：FSImage.java    
Collection<URI> getDirectories(NameNodeDirType dirType) 
    throws IOException {
  ArrayList<URI> list = new ArrayList<URI>();
  Iterator<StorageDirectory> it = (dirType == null) ? dirIterator() :
                                  dirIterator(dirType);
  for ( ;it.hasNext(); ) {
    StorageDirectory sd = it.next();
    try {
      list.add(Util.fileAsURI(sd.getRoot()));
    } catch (IOException e) {
      throw new IOException("Exception while processing " +
          "StorageDirectory " + sd.getRoot(), e);
    }
  }
  return list;
}
项目：cumulus    文件：TestBlockReplacement.java    
public void testThrottler() throws IOException {
  Configuration conf = new HdfsConfiguration();
  FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
  long bandwidthPerSec = 1024*1024L;
  final long TOTAL_BYTES =6*bandwidthPerSec; 
  long bytesToSend = TOTAL_BYTES; 
  long start = Util.now();
  DataTransferThrottler throttler = new DataTransferThrottler(bandwidthPerSec);
  long totalBytes = 0L;
  long bytesSent = 1024*512L; // 0.5MB
  throttler.throttle(bytesSent);
  bytesToSend -= bytesSent;
  bytesSent = 1024*768L; // 0.75MB
  throttler.throttle(bytesSent);
  bytesToSend -= bytesSent;
  try {
    Thread.sleep(1000);
  } catch (InterruptedException ignored) {}
  throttler.throttle(bytesToSend);
  long end = Util.now();
  assertTrue(totalBytes*1000/(end-start)<=bandwidthPerSec);
}