1. 包列表
  2. 类列表
  3. org.apache.hadoop.hdfs.MiniDFSCluster.Builder

Java 类org.apache.hadoop.hdfs.MiniDFSCluster.Builder 实例源码

项目：hadoop    文件：TestCopyFiles.java    
/** tests basedir option copying files from dfs file system to dfs file system */
public void testBasedir() throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/basedir/middle/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-basedir",
                                       "/basedir",
                                       namenode+"/basedir/middle/srcdat",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(hdfs, "/destdat/middle/srcdat", files));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/basedir");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：gemfirexd-oss    文件：HdfsSortedOplogOrganizerJUnitTest.java    
private MiniDFSCluster initMiniHACluster(int nn1port, int nn2port)
    throws IOException {
  Configuration confForMiniDFS = new Configuration();

  Builder builder = new MiniDFSCluster.Builder(confForMiniDFS)
  .nnTopology(new MiniDFSNNTopology()
  .addNameservice(new MiniDFSNNTopology.NSConf("ns1")
  .addNN(new MiniDFSNNTopology.NNConf("nn1").setIpcPort(nn1port))
  .addNN(new MiniDFSNNTopology.NNConf("nn2").setIpcPort(nn2port))
      ))
      .numDataNodes(1);

  MiniDFSCluster cluster = builder.build();
  cluster.waitActive();

  NameNode nnode1 = cluster.getNameNode(0);
  assertTrue(nnode1.isStandbyState());
  NameNode nnode2 = cluster.getNameNode(1);
  assertTrue(nnode2.isStandbyState());

  cluster.transitionToActive(0);
  assertFalse(nnode1.isStandbyState());
  return cluster;
}
项目：big-c    文件：TestCopyFiles.java    
/** tests basedir option copying files from dfs file system to dfs file system */
public void testBasedir() throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/basedir/middle/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-basedir",
                                       "/basedir",
                                       namenode+"/basedir/middle/srcdat",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(hdfs, "/destdat/middle/srcdat", files));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/basedir");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：hadoop-2.6.0-cdh5.4.3    文件：TestCopyFiles.java    
/** tests basedir option copying files from dfs file system to dfs file system */
public void testBasedir() throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/basedir/middle/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-basedir",
                                       "/basedir",
                                       namenode+"/basedir/middle/srcdat",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(hdfs, "/destdat/middle/srcdat", files));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/basedir");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：gemfirexd-oss    文件：HdfsSortedOplogOrganizerJUnitTest.java    
private MiniDFSCluster initMiniHACluster(int nn1port, int nn2port)
    throws IOException {
  Configuration confForMiniDFS = new Configuration();

  Builder builder = new MiniDFSCluster.Builder(confForMiniDFS)
  .nnTopology(new MiniDFSNNTopology()
  .addNameservice(new MiniDFSNNTopology.NSConf("ns1")
  .addNN(new MiniDFSNNTopology.NNConf("nn1").setIpcPort(nn1port))
  .addNN(new MiniDFSNNTopology.NNConf("nn2").setIpcPort(nn2port))
      ))
      .numDataNodes(1);

  MiniDFSCluster cluster = builder.build();
  cluster.waitActive();

  NameNode nnode1 = cluster.getNameNode(0);
  assertTrue(nnode1.isStandbyState());
  NameNode nnode2 = cluster.getNameNode(1);
  assertTrue(nnode2.isStandbyState());

  cluster.transitionToActive(0);
  assertFalse(nnode1.isStandbyState());
  return cluster;
}
项目：hadoop-on-lustre2    文件：TestCopyFiles.java    
/** tests basedir option copying files from dfs file system to dfs file system */
public void testBasedir() throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/basedir/middle/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-basedir",
                                       "/basedir",
                                       namenode+"/basedir/middle/srcdat",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(hdfs, "/destdat/middle/srcdat", files));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/basedir");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：hadoop    文件：TestDataNodeRollingUpgrade.java    
private void startCluster() throws IOException {
  conf = new HdfsConfiguration();
  conf.setInt("dfs.blocksize", 1024*1024);
  cluster = new Builder(conf).numDataNodes(REPL_FACTOR).build();
  cluster.waitActive();
  fs = cluster.getFileSystem();
  nn = cluster.getNameNode(0);
  assertNotNull(nn);
  dn0 = cluster.getDataNodes().get(0);
  assertNotNull(dn0);
  blockPoolId = cluster.getNameNode(0).getNamesystem().getBlockPoolId();
}
项目：hadoop    文件：TestCopyFiles.java    
/** copy files from dfs file system to dfs file system */
public void testCopyFromDfsToDfs() throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       namenode+"/logs",
                                       namenode+"/srcdat",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(hdfs, "/destdat", files));
      FileSystem fs = FileSystem.get(URI.create(namenode+"/logs"), conf);
      assertTrue("Log directory does not exist.",
                 fs.exists(new Path(namenode+"/logs")));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/srcdat");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：hadoop    文件：TestCopyFiles.java    
/** copy empty directory on dfs file system */
public void testEmptyDir() throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {

      FileSystem fs = FileSystem.get(URI.create(namenode), new Configuration());
      fs.mkdirs(new Path("/empty"));

      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       namenode+"/logs",
                                       namenode+"/empty",
                                       namenode+"/dest"});
      fs = FileSystem.get(URI.create(namenode+"/destdat"), conf);
      assertTrue("Destination directory does not exist.",
                 fs.exists(new Path(namenode+"/dest")));
      deldir(hdfs, "/dest");
      deldir(hdfs, "/empty");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：hadoop    文件：TestCopyFiles.java    
/** copy files from local file system to dfs file system */
public void testCopyFromLocalToDfs() throws Exception {
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).build();
    final FileSystem hdfs = cluster.getFileSystem();
    final String namenode = hdfs.getUri().toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       namenode+"/logs",
                                       "file:///"+TEST_ROOT_DIR+"/srcdat",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(cluster.getFileSystem(), "/destdat", files));
      assertTrue("Log directory does not exist.",
                  hdfs.exists(new Path(namenode+"/logs")));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/logs");
      deldir(FileSystem.get(LOCAL_FS, conf), TEST_ROOT_DIR+"/srcdat");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：hadoop    文件：TestCopyFiles.java    
/** copy files from dfs file system to local file system */
public void testCopyFromDfsToLocal() throws Exception {
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    final FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
    cluster = new MiniDFSCluster.Builder(conf).build();
    final FileSystem hdfs = cluster.getFileSystem();
    final String namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       "/logs",
                                       namenode+"/srcdat",
                                       "file:///"+TEST_ROOT_DIR+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files));
      assertTrue("Log directory does not exist.",
                  hdfs.exists(new Path("/logs")));
      deldir(localfs, TEST_ROOT_DIR+"/destdat");
      deldir(hdfs, "/logs");
      deldir(hdfs, "/srcdat");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：hadoop    文件：TestCopyFiles.java    
/**
 * verify that -delete option works for other {@link FileSystem}
 * implementations. See MAPREDUCE-1285 */
public void testDeleteLocal() throws Exception {
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    final FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
    cluster = new MiniDFSCluster.Builder(conf).build();
    final FileSystem hdfs = cluster.getFileSystem();
    final String namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
      String destdir = TEST_ROOT_DIR + "/destdat";
      MyFile[] localFiles = createFiles(localfs, destdir);
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-delete",
                                       "-update",
                                       "-log",
                                       "/logs",
                                       namenode+"/srcdat",
                                       "file:///"+TEST_ROOT_DIR+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(localfs, destdir, files));
      assertTrue("Log directory does not exist.",
                  hdfs.exists(new Path("/logs")));
      deldir(localfs, destdir);
      deldir(hdfs, "/logs");
      deldir(hdfs, "/srcdat");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：hadoop    文件：TestCopyFiles.java    
/** test globbing  */
public void testGlobbing() throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       namenode+"/logs",
                                       namenode+"/srcdat/*",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(hdfs, "/destdat", files));
      FileSystem fs = FileSystem.get(URI.create(namenode+"/logs"), conf);
      assertTrue("Log directory does not exist.",
                 fs.exists(new Path(namenode+"/logs")));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/srcdat");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：aliyun-oss-hadoop-fs    文件：TestDataNodeRollingUpgrade.java    
private void startCluster() throws IOException {
  conf = new HdfsConfiguration();
  conf.setInt("dfs.blocksize", 1024*1024);
  cluster = new Builder(conf).numDataNodes(REPL_FACTOR).build();
  cluster.waitActive();
  fs = cluster.getFileSystem();
  nn = cluster.getNameNode(0);
  assertNotNull(nn);
  dn0 = cluster.getDataNodes().get(0);
  assertNotNull(dn0);
  blockPoolId = cluster.getNameNode(0).getNamesystem().getBlockPoolId();
}
项目：gemfirexd-oss    文件：CreateHDFSStoreTest.java    
public static MiniDFSCluster initMiniCluster(int port, int numDN, HashMap<String, String> map) throws Exception {
  System.setProperty("test.build.data", "hdfs-test-cluster");
  Configuration hconf = new HdfsConfiguration();
  for (Entry<String, String> entry : map.entrySet()) {
    hconf.set(entry.getKey(), entry.getValue());
  }

  Builder builder = new MiniDFSCluster.Builder(hconf);
  builder.numDataNodes(numDN);
  builder.nameNodePort(port);
  MiniDFSCluster cluster = builder.build();
  return cluster;
}
项目：gemfirexd-oss    文件：BaseHoplogTestCase.java    
public static MiniDFSCluster initMiniCluster(int port, int numDN, HashMap<String, String> map) throws Exception {
  System.setProperty("test.build.data", "hdfs-test-cluster");
  Configuration hconf = new HdfsConfiguration();
  for (Entry<String, String> entry : map.entrySet()) {
    hconf.set(entry.getKey(), entry.getValue());
  }

  hconf.set("dfs.namenode.fs-limits.min-block-size", "1024");

  Builder builder = new MiniDFSCluster.Builder(hconf);
  builder.numDataNodes(numDN);
  builder.nameNodePort(port);
  MiniDFSCluster cluster = builder.build();
  return cluster;
}
项目：gemfirexd-oss    文件：HdfsErrorHandlingJunitTest.java    
private void initMiniCluster(Configuration hconf, int numDataNodes)
    throws IOException {
  Builder builder = new MiniDFSCluster.Builder(hconf);
  builder.numDataNodes(numDataNodes);
  builder.nameNodePort(CLUSTER_PORT);
  cluster = builder.build();
}
项目：big-c    文件：TestDataNodeRollingUpgrade.java    
private void startCluster() throws IOException {
  conf = new HdfsConfiguration();
  conf.setInt("dfs.blocksize", 1024*1024);
  cluster = new Builder(conf).numDataNodes(REPL_FACTOR).build();
  cluster.waitActive();
  fs = cluster.getFileSystem();
  nn = cluster.getNameNode(0);
  assertNotNull(nn);
  dn0 = cluster.getDataNodes().get(0);
  assertNotNull(dn0);
  blockPoolId = cluster.getNameNode(0).getNamesystem().getBlockPoolId();
}
项目：big-c    文件：TestCopyFiles.java    
/** copy files from dfs file system to dfs file system */
public void testCopyFromDfsToDfs() throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       namenode+"/logs",
                                       namenode+"/srcdat",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(hdfs, "/destdat", files));
      FileSystem fs = FileSystem.get(URI.create(namenode+"/logs"), conf);
      assertTrue("Log directory does not exist.",
                 fs.exists(new Path(namenode+"/logs")));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/srcdat");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：big-c    文件：TestCopyFiles.java    
/** copy empty directory on dfs file system */
public void testEmptyDir() throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {

      FileSystem fs = FileSystem.get(URI.create(namenode), new Configuration());
      fs.mkdirs(new Path("/empty"));

      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       namenode+"/logs",
                                       namenode+"/empty",
                                       namenode+"/dest"});
      fs = FileSystem.get(URI.create(namenode+"/destdat"), conf);
      assertTrue("Destination directory does not exist.",
                 fs.exists(new Path(namenode+"/dest")));
      deldir(hdfs, "/dest");
      deldir(hdfs, "/empty");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：big-c    文件：TestCopyFiles.java    
/** copy files from local file system to dfs file system */
public void testCopyFromLocalToDfs() throws Exception {
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).build();
    final FileSystem hdfs = cluster.getFileSystem();
    final String namenode = hdfs.getUri().toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       namenode+"/logs",
                                       "file:///"+TEST_ROOT_DIR+"/srcdat",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(cluster.getFileSystem(), "/destdat", files));
      assertTrue("Log directory does not exist.",
                  hdfs.exists(new Path(namenode+"/logs")));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/logs");
      deldir(FileSystem.get(LOCAL_FS, conf), TEST_ROOT_DIR+"/srcdat");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：big-c    文件：TestCopyFiles.java    
/** copy files from dfs file system to local file system */
public void testCopyFromDfsToLocal() throws Exception {
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    final FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
    cluster = new MiniDFSCluster.Builder(conf).build();
    final FileSystem hdfs = cluster.getFileSystem();
    final String namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       "/logs",
                                       namenode+"/srcdat",
                                       "file:///"+TEST_ROOT_DIR+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files));
      assertTrue("Log directory does not exist.",
                  hdfs.exists(new Path("/logs")));
      deldir(localfs, TEST_ROOT_DIR+"/destdat");
      deldir(hdfs, "/logs");
      deldir(hdfs, "/srcdat");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：big-c    文件：TestCopyFiles.java    
/**
 * verify that -delete option works for other {@link FileSystem}
 * implementations. See MAPREDUCE-1285 */
public void testDeleteLocal() throws Exception {
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    final FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
    cluster = new MiniDFSCluster.Builder(conf).build();
    final FileSystem hdfs = cluster.getFileSystem();
    final String namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
      String destdir = TEST_ROOT_DIR + "/destdat";
      MyFile[] localFiles = createFiles(localfs, destdir);
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-delete",
                                       "-update",
                                       "-log",
                                       "/logs",
                                       namenode+"/srcdat",
                                       "file:///"+TEST_ROOT_DIR+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(localfs, destdir, files));
      assertTrue("Log directory does not exist.",
                  hdfs.exists(new Path("/logs")));
      deldir(localfs, destdir);
      deldir(hdfs, "/logs");
      deldir(hdfs, "/srcdat");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：big-c    文件：TestCopyFiles.java    
/** test globbing  */
public void testGlobbing() throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       namenode+"/logs",
                                       namenode+"/srcdat/*",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(hdfs, "/destdat", files));
      FileSystem fs = FileSystem.get(URI.create(namenode+"/logs"), conf);
      assertTrue("Log directory does not exist.",
                 fs.exists(new Path(namenode+"/logs")));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/srcdat");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：hadoop-2.6.0-cdh5.4.3    文件：TestDataNodeRollingUpgrade.java    
private void startCluster() throws IOException {
  conf = new HdfsConfiguration();
  conf.setInt("dfs.blocksize", 1024*1024);
  cluster = new Builder(conf).numDataNodes(REPL_FACTOR).build();
  cluster.waitActive();
  fs = cluster.getFileSystem();
  nn = cluster.getNameNode(0);
  assertNotNull(nn);
  dn0 = cluster.getDataNodes().get(0);
  assertNotNull(dn0);
  blockPoolId = cluster.getNameNode(0).getNamesystem().getBlockPoolId();
}
项目：hadoop-2.6.0-cdh5.4.3    文件：TestCopyFiles.java    
/** copy files from dfs file system to dfs file system */
public void testCopyFromDfsToDfs() throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       namenode+"/logs",
                                       namenode+"/srcdat",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(hdfs, "/destdat", files));
      FileSystem fs = FileSystem.get(URI.create(namenode+"/logs"), conf);
      assertTrue("Log directory does not exist.",
                 fs.exists(new Path(namenode+"/logs")));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/srcdat");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：hadoop-2.6.0-cdh5.4.3    文件：TestCopyFiles.java    
/** copy empty directory on dfs file system */
public void testEmptyDir() throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {

      FileSystem fs = FileSystem.get(URI.create(namenode), new Configuration());
      fs.mkdirs(new Path("/empty"));

      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       namenode+"/logs",
                                       namenode+"/empty",
                                       namenode+"/dest"});
      fs = FileSystem.get(URI.create(namenode+"/destdat"), conf);
      assertTrue("Destination directory does not exist.",
                 fs.exists(new Path(namenode+"/dest")));
      deldir(hdfs, "/dest");
      deldir(hdfs, "/empty");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：hadoop-2.6.0-cdh5.4.3    文件：TestCopyFiles.java    
/** copy files from local file system to dfs file system */
public void testCopyFromLocalToDfs() throws Exception {
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).build();
    final FileSystem hdfs = cluster.getFileSystem();
    final String namenode = hdfs.getUri().toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       namenode+"/logs",
                                       "file:///"+TEST_ROOT_DIR+"/srcdat",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(cluster.getFileSystem(), "/destdat", files));
      assertTrue("Log directory does not exist.",
                  hdfs.exists(new Path(namenode+"/logs")));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/logs");
      deldir(FileSystem.get(LOCAL_FS, conf), TEST_ROOT_DIR+"/srcdat");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：hadoop-2.6.0-cdh5.4.3    文件：TestCopyFiles.java    
/** copy files from dfs file system to local file system */
public void testCopyFromDfsToLocal() throws Exception {
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    final FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
    cluster = new MiniDFSCluster.Builder(conf).build();
    final FileSystem hdfs = cluster.getFileSystem();
    final String namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       "/logs",
                                       namenode+"/srcdat",
                                       "file:///"+TEST_ROOT_DIR+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files));
      assertTrue("Log directory does not exist.",
                  hdfs.exists(new Path("/logs")));
      deldir(localfs, TEST_ROOT_DIR+"/destdat");
      deldir(hdfs, "/logs");
      deldir(hdfs, "/srcdat");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：hadoop-2.6.0-cdh5.4.3    文件：TestCopyFiles.java    
/**
 * verify that -delete option works for other {@link FileSystem}
 * implementations. See MAPREDUCE-1285 */
public void testDeleteLocal() throws Exception {
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    final FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
    cluster = new MiniDFSCluster.Builder(conf).build();
    final FileSystem hdfs = cluster.getFileSystem();
    final String namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
      String destdir = TEST_ROOT_DIR + "/destdat";
      MyFile[] localFiles = createFiles(localfs, destdir);
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-delete",
                                       "-update",
                                       "-log",
                                       "/logs",
                                       namenode+"/srcdat",
                                       "file:///"+TEST_ROOT_DIR+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(localfs, destdir, files));
      assertTrue("Log directory does not exist.",
                  hdfs.exists(new Path("/logs")));
      deldir(localfs, destdir);
      deldir(hdfs, "/logs");
      deldir(hdfs, "/srcdat");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：hadoop-2.6.0-cdh5.4.3    文件：TestCopyFiles.java    
/** test globbing  */
public void testGlobbing() throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       namenode+"/logs",
                                       namenode+"/srcdat/*",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(hdfs, "/destdat", files));
      FileSystem fs = FileSystem.get(URI.create(namenode+"/logs"), conf);
      assertTrue("Log directory does not exist.",
                 fs.exists(new Path(namenode+"/logs")));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/srcdat");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：gemfirexd-oss    文件：CreateHDFSStoreTest.java    
public static MiniDFSCluster initMiniCluster(int port, int numDN, HashMap<String, String> map) throws Exception {
  System.setProperty("test.build.data", "hdfs-test-cluster");
  Configuration hconf = new HdfsConfiguration();
  for (Entry<String, String> entry : map.entrySet()) {
    hconf.set(entry.getKey(), entry.getValue());
  }

  Builder builder = new MiniDFSCluster.Builder(hconf);
  builder.numDataNodes(numDN);
  builder.nameNodePort(port);
  MiniDFSCluster cluster = builder.build();
  return cluster;
}
项目：gemfirexd-oss    文件：BaseHoplogTestCase.java    
public static MiniDFSCluster initMiniCluster(int port, int numDN, HashMap<String, String> map) throws Exception {
  System.setProperty("test.build.data", "hdfs-test-cluster");
  Configuration hconf = new HdfsConfiguration();
  for (Entry<String, String> entry : map.entrySet()) {
    hconf.set(entry.getKey(), entry.getValue());
  }

  hconf.set("dfs.namenode.fs-limits.min-block-size", "1024");

  Builder builder = new MiniDFSCluster.Builder(hconf);
  builder.numDataNodes(numDN);
  builder.nameNodePort(port);
  MiniDFSCluster cluster = builder.build();
  return cluster;
}
项目：gemfirexd-oss    文件：HdfsErrorHandlingJunitTest.java    
private void initMiniCluster(Configuration hconf, int numDataNodes)
    throws IOException {
  Builder builder = new MiniDFSCluster.Builder(hconf);
  builder.numDataNodes(numDataNodes);
  builder.nameNodePort(CLUSTER_PORT);
  cluster = builder.build();
}
项目：FlexMap    文件：TestDataNodeRollingUpgrade.java    
private void startCluster() throws IOException {
  conf = new HdfsConfiguration();
  conf.setInt("dfs.blocksize", 1024*1024);
  cluster = new Builder(conf).numDataNodes(REPL_FACTOR).build();
  cluster.waitActive();
  fs = cluster.getFileSystem();
  nn = cluster.getNameNode(0);
  assertNotNull(nn);
  dn0 = cluster.getDataNodes().get(0);
  assertNotNull(dn0);
  blockPoolId = cluster.getNameNode(0).getNamesystem().getBlockPoolId();
}
项目：hops    文件：TestCopyFiles.java    
/** copy files from dfs file system to dfs file system */
@SuppressWarnings("deprecation")
public void testCopyFromDfsToDfs() throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       namenode+"/logs",
                                       namenode+"/srcdat",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(hdfs, "/destdat", files));
      FileSystem fs = FileSystem.get(URI.create(namenode+"/logs"), conf);
      assertTrue("Log directory does not exist.",
                 fs.exists(new Path(namenode+"/logs")));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/srcdat");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：hops    文件：TestCopyFiles.java    
/** copy empty directory on dfs file system */
@SuppressWarnings("deprecation")
public void testEmptyDir() throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {

      FileSystem fs = FileSystem.get(URI.create(namenode), new Configuration());
      fs.mkdirs(new Path("/empty"));

      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       namenode+"/logs",
                                       namenode+"/empty",
                                       namenode+"/dest"});
      fs = FileSystem.get(URI.create(namenode+"/destdat"), conf);
      assertTrue("Destination directory does not exist.",
                 fs.exists(new Path(namenode+"/dest")));
      deldir(hdfs, "/dest");
      deldir(hdfs, "/empty");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：hops    文件：TestCopyFiles.java    
/** copy files from local file system to dfs file system */
@SuppressWarnings("deprecation")
public void testCopyFromLocalToDfs() throws Exception {
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).build();
    final FileSystem hdfs = cluster.getFileSystem();
    final String namenode = hdfs.getUri().toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       namenode+"/logs",
                                       "file:///"+TEST_ROOT_DIR+"/srcdat",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(cluster.getFileSystem(), "/destdat", files));
      assertTrue("Log directory does not exist.",
                  hdfs.exists(new Path(namenode+"/logs")));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/logs");
      deldir(FileSystem.get(LOCAL_FS, conf), TEST_ROOT_DIR+"/srcdat");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：hops    文件：TestCopyFiles.java    
/** copy files from dfs file system to local file system */
@SuppressWarnings("deprecation")
public void testCopyFromDfsToLocal() throws Exception {
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    final FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
    cluster = new MiniDFSCluster.Builder(conf).build();
    final FileSystem hdfs = cluster.getFileSystem();
    final String namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-log",
                                       "/logs",
                                       namenode+"/srcdat",
                                       "file:///"+TEST_ROOT_DIR+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files));
      assertTrue("Log directory does not exist.",
                  hdfs.exists(new Path("/logs")));
      deldir(localfs, TEST_ROOT_DIR+"/destdat");
      deldir(hdfs, "/logs");
      deldir(hdfs, "/srcdat");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}
项目：hops    文件：TestCopyFiles.java    
/** tests basedir option copying files from dfs file system to dfs file system */
@SuppressWarnings("deprecation")
public void testBasedir() throws Exception {
  String namenode = null;
  MiniDFSCluster cluster = null;
  try {
    Configuration conf = new Configuration();
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    final FileSystem hdfs = cluster.getFileSystem();
    namenode = FileSystem.getDefaultUri(conf).toString();
    if (namenode.startsWith("hdfs://")) {
      MyFile[] files = createFiles(URI.create(namenode), "/basedir/middle/srcdat");
      ToolRunner.run(new DistCpV1(conf), new String[] {
                                       "-basedir",
                                       "/basedir",
                                       namenode+"/basedir/middle/srcdat",
                                       namenode+"/destdat"});
      assertTrue("Source and destination directories do not match.",
                 checkFiles(hdfs, "/destdat/middle/srcdat", files));
      deldir(hdfs, "/destdat");
      deldir(hdfs, "/basedir");
      deldir(hdfs, "/logs");
    }
  } finally {
    if (cluster != null) { cluster.shutdown(); }
  }
}