@Test public void testGoodClassOrNull() throws Exception { String NAME = "ClassWithNoPackage"; ClassLoader cl = TestClassWithNoPackage.class.getClassLoader(); String JAR = JarFinder.getJar(cl.loadClass(NAME)); // Add testjob jar file to classpath. Configuration conf = new Configuration(); conf.setClassLoader(new URLClassLoader(new URL[]{new URL("file", null, JAR)}, null)); // Get class with no package name. String defaultPackage = this.getClass().getPackage().getName(); Class c = StreamUtil.goodClassOrNull(conf, NAME, defaultPackage); assertNotNull("Class " + NAME + " not found!", c); }
public static MiniMRClientCluster create(Class<?> caller, String identifier, int noOfNMs, Configuration conf) throws IOException { if (conf == null) { conf = new Configuration(); } FileSystem fs = FileSystem.get(conf); Path testRootDir = new Path("target", identifier + "-tmpDir") .makeQualified(fs); Path appJar = new Path(testRootDir, "MRAppJar.jar"); // Copy MRAppJar and make it private. Path appMasterJar = new Path(MiniMRYarnCluster.APPJAR); fs.copyFromLocalFile(appMasterJar, appJar); fs.setPermission(appJar, new FsPermission("744")); Job job = Job.getInstance(conf); job.addFileToClassPath(appJar); Path callerJar = new Path(JarFinder.getJar(caller)); Path remoteCallerJar = new Path(testRootDir, callerJar.getName()); fs.copyFromLocalFile(callerJar, remoteCallerJar); fs.setPermission(remoteCallerJar, new FsPermission("744")); job.addFileToClassPath(remoteCallerJar); MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(identifier, noOfNMs); job.getConfiguration().set("minimrclientcluster.caller.name", identifier); job.getConfiguration().setInt("minimrclientcluster.nodemanagers.number", noOfNMs); miniMRYarnCluster.init(job.getConfiguration()); miniMRYarnCluster.start(); return new MiniMRYarnClusterAdapter(miniMRYarnCluster); }
public static MiniMRClientCluster create(Class<?> caller, String identifier, int noOfNMs, Configuration conf, boolean formatDB) throws IOException { if (conf == null) { conf = new Configuration(); } FileSystem fs = FileSystem.get(conf); Path testRootDir = new Path("target", identifier + "-tmpDir") .makeQualified(fs); Path appJar = new Path(testRootDir, "MRAppJar.jar"); // Copy MRAppJar and make it private. Path appMasterJar = new Path(MiniMRYarnCluster.APPJAR); fs.copyFromLocalFile(appMasterJar, appJar); fs.setPermission(appJar, new FsPermission("744")); Job job = Job.getInstance(conf); job.addFileToClassPath(appJar); Path callerJar = new Path(JarFinder.getJar(caller)); Path remoteCallerJar = new Path(testRootDir, callerJar.getName()); fs.copyFromLocalFile(callerJar, remoteCallerJar); fs.setPermission(remoteCallerJar, new FsPermission("744")); job.addFileToClassPath(remoteCallerJar); MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(identifier, noOfNMs, formatDB); job.getConfiguration().set("minimrclientcluster.caller.name", identifier); job.getConfiguration().setInt("minimrclientcluster.nodemanagers.number", noOfNMs); miniMRYarnCluster.init(job.getConfiguration()); miniMRYarnCluster.start(); return new MiniMRYarnClusterAdapter(miniMRYarnCluster); }
public void _testDistributedCache(String jobJarPath) throws Exception { if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) { LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR + " not found. Not running test."); return; } // Create a temporary file of length 1. Path first = createTempFile("distributed.first", "x"); // Create two jars with a single file inside them. Path second = makeJar(new Path(TEST_ROOT_DIR, "distributed.second.jar"), 2); Path third = makeJar(new Path(TEST_ROOT_DIR, "distributed.third.jar"), 3); Path fourth = makeJar(new Path(TEST_ROOT_DIR, "distributed.fourth.jar"), 4); Job job = Job.getInstance(mrCluster.getConfig()); // Set the job jar to a new "dummy" jar so we can check that its extracted // properly job.setJar(jobJarPath); // Because the job jar is a "dummy" jar, we need to include the jar with // DistributedCacheChecker or it won't be able to find it Path distributedCacheCheckerJar = new Path( JarFinder.getJar(DistributedCacheChecker.class)); job.addFileToClassPath(distributedCacheCheckerJar.makeQualified( localFs.getUri(), distributedCacheCheckerJar.getParent())); job.setMapperClass(DistributedCacheChecker.class); job.setOutputFormatClass(NullOutputFormat.class); FileInputFormat.setInputPaths(job, first); // Creates the Job Configuration job.addCacheFile( new URI(first.toUri().toString() + "#distributed.first.symlink")); job.addFileToClassPath(second); // The AppMaster jar itself job.addFileToClassPath( APP_JAR.makeQualified(localFs.getUri(), APP_JAR.getParent())); job.addArchiveToClassPath(third); job.addCacheArchive(fourth.toUri()); job.setMaxMapAttempts(1); // speed up failures job.submit(); String trackingUrl = job.getTrackingURL(); String jobId = job.getJobID().toString(); Assert.assertTrue(job.waitForCompletion(false)); Assert.assertTrue("Tracking URL was " + trackingUrl + " but didn't Match Job ID " + jobId , trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/")); }
public static MiniMRClientCluster create(Class<?> caller, String identifier, int noOfNMs, Configuration conf, File testWorkDir) throws IOException { if (conf == null) { conf = new Configuration(); } FileSystem fs = FileSystem.get(conf); Path testRootDir = new Path(testWorkDir.getPath(), identifier + "-tmpDir") .makeQualified(fs); Path appJar = new Path(testRootDir, "MRAppJar.jar"); // Copy MRAppJar and make it private. Path appMasterJar = new Path(MiniMRYarnCluster.APPJAR); fs.copyFromLocalFile(appMasterJar, appJar); fs.setPermission(appJar, new FsPermission("744")); Job job = Job.getInstance(conf); job.addFileToClassPath(appJar); Path callerJar = new Path(JarFinder.getJar(caller)); Path remoteCallerJar = new Path(testRootDir, callerJar.getName()); fs.copyFromLocalFile(callerJar, remoteCallerJar); fs.setPermission(remoteCallerJar, new FsPermission("744")); job.addFileToClassPath(remoteCallerJar); MiniMRYarnCluster miniMRYarnCluster; try { miniMRYarnCluster = new MiniMRYarnCluster(identifier, noOfNMs, testWorkDir); } catch (Exception e) { throw new RuntimeException(e); } job.getConfiguration().set("minimrclientcluster.caller.name", identifier); job.getConfiguration().setInt("minimrclientcluster.nodemanagers.number", noOfNMs); miniMRYarnCluster.init(job.getConfiguration()); miniMRYarnCluster.start(); return new MiniMRYarnClusterAdapter(miniMRYarnCluster, testWorkDir); }
public YarnComponent withApplicationMasterClass(Class clazz){ appmasterJar = JarFinder.getJar(clazz); return this; }
