@Test public void testFilesCountAndSizesForSpecifiedFilePool() throws Exception { conf = remoteClient.getDaemonConf(); final long inputSize = clusterSize * 200; int [] fileSizesInMB = {50, 100, 400, 50, 300, 10, 60, 40, 20 ,10 , 500}; long targetSize = Long.MAX_VALUE; final int expFileCount = 13; String [] runtimeValues ={"LOADJOB", SubmitterUserResolver.class.getName(), "STRESS", inputSize+"m", "file:///dev/null"}; int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir, conf,GridMixRunMode.DATA_GENERATION, runtimeValues); Assert.assertEquals("Data generation has failed.", 0 , exitCode); // create files for given sizes. createFiles(new Path(gridmixDir,"input"),fileSizesInMB); conf.setLong(FilePool.GRIDMIX_MIN_FILE, 100 * 1024 * 1024); FilePool fpool = new FilePool(conf,new Path(gridmixDir,"input")); fpool.refresh(); verifyFilesSizeAndCountForSpecifiedPool(expFileCount,targetSize, fpool); }
private void verifyFilesSizeAndCountForSpecifiedPool(int expFileCount, long minFileSize, FilePool pool) throws IOException { final ArrayList<FileStatus> files = new ArrayList<FileStatus>(); long actFilesSize = pool.getInputFiles(minFileSize, files)/(1024 * 1024); long expFilesSize = 3100 ; Assert.assertEquals("Files Size has not matched for specified pool.", expFilesSize, actFilesSize); int actFileCount = files.size(); Assert.assertEquals("File count has not matched.", expFileCount, actFileCount); int count = 0; for (FileStatus fstat : files) { String fp = fstat.getPath().toString(); count = count + ((fp.indexOf("datafile_") > 0)? 0 : 1); } Assert.assertEquals("Total folders are not matched with cluster size", clusterSize, count); }
private void verifyFilesSizeAndCountForSpecifiedPool(int expFileCount, long minFileSize, FilePool pool) throws IOException { final ArrayList<FileStatus> files = new ArrayList<FileStatus>(); long filesSizeInBytes = pool.getInputFiles(minFileSize, files); long actFilesSizeInMB = filesSizeInBytes / (1024 * 1024); long expFilesSizeInMB = (clusterSize * 200) + 1300; Assert.assertEquals("Files Size has not matched for specified pool.", expFilesSizeInMB, actFilesSizeInMB); int actFileCount = files.size(); Assert.assertEquals("File count has not matched.", expFileCount, actFileCount); int count = 0; for (FileStatus fstat : files) { String fp = fstat.getPath().toString(); count = count + ((fp.indexOf("datafile_") > 0)? 0 : 1); } Assert.assertEquals("Total folders are not matched with cluster size", clusterSize, count); }
@Test public void testFilesCountAndSizesForSpecifiedFilePool() throws Exception { conf = remoteClient.getDaemonConf(); final long inputSizeInMB = clusterSize * 200; int [] fileSizesInMB = {50, 100, 400, 50, 300, 10, 60, 40, 20 ,10 , 500}; long targetSize = Long.MAX_VALUE; final int expFileCount = clusterSize + 4; String [] runtimeValues ={"LOADJOB", SubmitterUserResolver.class.getName(), "STRESS", inputSizeInMB + "m", "file:///dev/null"}; String [] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; // Generate the input data by using gridmix framework. int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir, conf, GridMixRunMode.DATA_GENERATION.getValue(), runtimeValues, otherArgs); Assert.assertEquals("Data generation has failed.", 0 , exitCode); // Create the files without using gridmix input generation with // above mentioned sizes in a array. createFiles(new Path(gridmixDir, "input"), fileSizesInMB); conf.setLong(FilePool.GRIDMIX_MIN_FILE, 100 * 1024 * 1024); FilePool fpool = new FilePool(conf, new Path(gridmixDir, "input")); fpool.refresh(); verifyFilesSizeAndCountForSpecifiedPool(expFileCount, targetSize, fpool); }