/** * Create a <code>version</code> file for datanode inside the specified parent * directory. If such a file already exists, it will be overwritten. * The given version string will be written to the file as the layout * version. None of the parameters may be null. * * @param parent directory where namenode VERSION file is stored * @param version StorageInfo to create VERSION file from * @param bpid Block pool Id * @param bpidToWrite Block pool Id to write into the version file */ public static void createDataNodeVersionFile(File[] parent, StorageInfo version, String bpid, String bpidToWrite) throws IOException { DataStorage storage = new DataStorage(version); storage.setDatanodeUuid("FixedDatanodeUuid"); File[] versionFiles = new File[parent.length]; for (int i = 0; i < parent.length; i++) { File versionFile = new File(parent[i], "VERSION"); StorageDirectory sd = new StorageDirectory(parent[i].getParentFile()); storage.createStorageID(sd, false); storage.writeProperties(versionFile, sd); versionFiles[i] = versionFile; File bpDir = BlockPoolSliceStorage.getBpRoot(bpid, parent[i]); createBlockPoolVersionFile(bpDir, version, bpidToWrite); } }
/** * Create a <code>version</code> file for datanode inside the specified parent * directory. If such a file already exists, it will be overwritten. * The given version string will be written to the file as the layout * version. None of the parameters may be null. * * @param parent directory where namenode VERSION file is stored * @param version StorageInfo to create VERSION file from * @param bpid Block pool Id * @param bpidToWrite Block pool Id to write into the version file */ public static void createDataNodeVersionFile(File[] parent, StorageInfo version, String bpid, String bpidToWrite) throws IOException { DataStorage storage = new DataStorage(version); storage.setDatanodeUuid("FixedDatanodeUuid"); File[] versionFiles = new File[parent.length]; for (int i = 0; i < parent.length; i++) { File versionFile = new File(parent[i], "VERSION"); StorageDirectory sd = new StorageDirectory(parent[i].getParentFile()); storage.createStorageID(sd); storage.writeProperties(versionFile, sd); versionFiles[i] = versionFile; File bpDir = BlockPoolSliceStorage.getBpRoot(bpid, parent[i]); createBlockPoolVersionFile(bpDir, version, bpidToWrite); } }
/** * Verify that the current directory exists and that the previous directory * does not exist. Verify that current hasn't been modified by comparing * the checksum of all it's containing files with their original checksum. */ static void checkResult(String[] nameNodeDirs, String[] dataNodeDirs, String bpid) throws Exception { List<File> dirs = Lists.newArrayList(); for (int i = 0; i < nameNodeDirs.length; i++) { File curDir = new File(nameNodeDirs[i], "current"); dirs.add(curDir); FSImageTestUtil.assertReasonableNameCurrentDir(curDir); } FSImageTestUtil.assertParallelFilesAreIdentical( dirs, Collections.<String>emptySet()); File dnCurDirs[] = new File[dataNodeDirs.length]; for (int i = 0; i < dataNodeDirs.length; i++) { dnCurDirs[i] = new File(dataNodeDirs[i],"current"); assertEquals(UpgradeUtilities.checksumContents(DATA_NODE, dnCurDirs[i], false), UpgradeUtilities.checksumMasterDataNodeContents()); } for (int i = 0; i < nameNodeDirs.length; i++) { assertFalse(new File(nameNodeDirs[i],"previous").isDirectory()); } if (bpid == null) { for (int i = 0; i < dataNodeDirs.length; i++) { assertFalse(new File(dataNodeDirs[i],"previous").isDirectory()); } } else { for (int i = 0; i < dataNodeDirs.length; i++) { File bpRoot = BlockPoolSliceStorage.getBpRoot(bpid, dnCurDirs[i]); assertFalse(new File(bpRoot,"previous").isDirectory()); File bpCurFinalizeDir = new File(bpRoot,"current/"+DataStorage.STORAGE_DIR_FINALIZED); assertEquals(UpgradeUtilities.checksumContents(DATA_NODE, bpCurFinalizeDir, true), UpgradeUtilities.checksumMasterBlockPoolFinalizedContents()); } } }
/** Disable the logs that are not very useful for snapshot related tests. */ public static void disableLogs() { final String[] lognames = { "org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner", "org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl", "org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetAsyncDiskService", }; for(String n : lognames) { GenericTestUtils.disableLog(LogFactory.getLog(n)); } GenericTestUtils.disableLog(LogFactory.getLog(UserGroupInformation.class)); GenericTestUtils.disableLog(LogFactory.getLog(BlockManager.class)); GenericTestUtils.disableLog(LogFactory.getLog(FSNamesystem.class)); GenericTestUtils.disableLog(LogFactory.getLog(DirectoryScanner.class)); GenericTestUtils.disableLog(LogFactory.getLog(MetricsSystemImpl.class)); GenericTestUtils.disableLog(BlockScanner.LOG); GenericTestUtils.disableLog(HttpServer2.LOG); GenericTestUtils.disableLog(DataNode.LOG); GenericTestUtils.disableLog(BlockPoolSliceStorage.LOG); GenericTestUtils.disableLog(LeaseManager.LOG); GenericTestUtils.disableLog(NameNode.stateChangeLog); GenericTestUtils.disableLog(NameNode.blockStateChangeLog); GenericTestUtils.disableLog(DFSClient.LOG); GenericTestUtils.disableLog(Server.LOG); }
public static void createBlockPoolVersionFile(File bpDir, StorageInfo version, String bpid) throws IOException { // Create block pool version files if (DataNodeLayoutVersion.supports( LayoutVersion.Feature.FEDERATION, version.layoutVersion)) { File bpCurDir = new File(bpDir, Storage.STORAGE_DIR_CURRENT); BlockPoolSliceStorage bpStorage = new BlockPoolSliceStorage(version, bpid); File versionFile = new File(bpCurDir, "VERSION"); StorageDirectory sd = new StorageDirectory(bpDir); bpStorage.writeProperties(versionFile, sd); } }
/** Disable the logs that are not very useful for snapshot related tests. */ public static void disableLogs() { final String[] lognames = { "org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner", "org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl", "org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetAsyncDiskService", }; for(String n : lognames) { setLevel2OFF(LogFactory.getLog(n)); } setLevel2OFF(LogFactory.getLog(UserGroupInformation.class)); setLevel2OFF(LogFactory.getLog(BlockManager.class)); setLevel2OFF(LogFactory.getLog(FSNamesystem.class)); setLevel2OFF(LogFactory.getLog(DirectoryScanner.class)); setLevel2OFF(LogFactory.getLog(MetricsSystemImpl.class)); setLevel2OFF(DataBlockScanner.LOG); setLevel2OFF(HttpServer.LOG); setLevel2OFF(DataNode.LOG); setLevel2OFF(BlockPoolSliceStorage.LOG); setLevel2OFF(LeaseManager.LOG); setLevel2OFF(NameNode.stateChangeLog); setLevel2OFF(NameNode.blockStateChangeLog); setLevel2OFF(DFSClient.LOG); setLevel2OFF(Server.LOG); }
/** * Create a <code>version</code> file for datanode inside the specified parent * directory. If such a file already exists, it will be overwritten. * The given version string will be written to the file as the layout * version. None of the parameters may be null. * * @param parent directory where namenode VERSION file is stored * @param version StorageInfo to create VERSION file from * @param bpid Block pool Id * @param bpidToWrite Block pool Id to write into the version file */ public static void createDataNodeVersionFile(File[] parent, StorageInfo version, String bpid, String bpidToWrite) throws IOException { DataStorage storage = new DataStorage(version, "doNotCare"); File[] versionFiles = new File[parent.length]; for (int i = 0; i < parent.length; i++) { File versionFile = new File(parent[i], "VERSION"); StorageDirectory sd = new StorageDirectory(parent[i].getParentFile()); storage.writeProperties(versionFile, sd); versionFiles[i] = versionFile; File bpDir = BlockPoolSliceStorage.getBpRoot(bpid, parent[i]); createBlockPoolVersionFile(bpDir, version, bpidToWrite); } }
public static void createBlockPoolVersionFile(File bpDir, StorageInfo version, String bpid) throws IOException { // Create block pool version files if (LayoutVersion.supports(Feature.FEDERATION, version.layoutVersion)) { File bpCurDir = new File(bpDir, Storage.STORAGE_DIR_CURRENT); BlockPoolSliceStorage bpStorage = new BlockPoolSliceStorage(version, bpid); File versionFile = new File(bpCurDir, "VERSION"); StorageDirectory sd = new StorageDirectory(bpDir); bpStorage.writeProperties(versionFile, sd); } }
/** Disable the logs that are not very useful for snapshot related tests. */ public static void disableLogs() { final String[] lognames = { "org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner", "org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl", "org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetAsyncDiskService", }; for(String n : lognames) { setLevel2OFF(LogFactory.getLog(n)); } setLevel2OFF(LogFactory.getLog(UserGroupInformation.class)); setLevel2OFF(LogFactory.getLog(BlockManager.class)); setLevel2OFF(LogFactory.getLog(FSNamesystem.class)); setLevel2OFF(LogFactory.getLog(DirectoryScanner.class)); setLevel2OFF(LogFactory.getLog(MetricsSystemImpl.class)); setLevel2OFF(DataBlockScanner.LOG); setLevel2OFF(HttpServer2.LOG); setLevel2OFF(DataNode.LOG); setLevel2OFF(BlockPoolSliceStorage.LOG); setLevel2OFF(LeaseManager.LOG); setLevel2OFF(NameNode.stateChangeLog); setLevel2OFF(NameNode.blockStateChangeLog); setLevel2OFF(DFSClient.LOG); setLevel2OFF(Server.LOG); }
/** * Initialize the data structures used by this class. * IMPORTANT NOTE: This method must be called once before calling * any other public method on this class. * <p/> * Creates a singleton master populated storage * directory for a Namenode (contains edits, fsimage, * version, and time files) and a Datanode (contains version and * block files). This can be a lengthy operation. */ public static void initialize() throws Exception { createEmptyDirs(new String[]{TEST_ROOT_DIR.toString()}); Configuration config = new HdfsConfiguration(); config.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, datanodeStorage.toString()); MiniDFSCluster cluster = null; try { // format data-node createEmptyDirs(new String[]{datanodeStorage.toString()}); // format and start NameNode and start DataNode DFSTestUtil.formatNameNode(config); cluster = new MiniDFSCluster.Builder(config).numDataNodes(1) .startupOption(StartupOption.REGULAR).format(false) .manageDataDfsDirs(false).manageNameDfsDirs(false).build(); NamenodeProtocols namenode = cluster.getNameNodeRpc(); namenodeStorageNamespaceID = namenode.versionRequest().getNamespaceID(); namenodeStorageFsscTime = namenode.versionRequest().getCTime(); namenodeStorageClusterID = namenode.versionRequest().getClusterID(); namenodeStorageBlockPoolID = namenode.versionRequest().getBlockPoolID(); FileSystem fs = FileSystem.get(config); Path baseDir = new Path("/TestUpgrade"); fs.mkdirs(baseDir); // write some files int bufferSize = 4096; byte[] buffer = new byte[bufferSize]; for (int i = 0; i < bufferSize; i++) { buffer[i] = (byte) ('0' + i % 50); } writeFile(fs, new Path(baseDir, "file1"), buffer, bufferSize); writeFile(fs, new Path(baseDir, "file2"), buffer, bufferSize); // write more files writeFile(fs, new Path(baseDir, "file3"), buffer, bufferSize); writeFile(fs, new Path(baseDir, "file4"), buffer, bufferSize); } finally { // shutdown if (cluster != null) { cluster.shutdown(); } FileUtil.fullyDelete(new File(namenodeStorage, "in_use.lock")); FileUtil.fullyDelete(new File(datanodeStorage, "in_use.lock")); } File dnCurDir = new File(datanodeStorage, "current"); datanodeStorageChecksum = checksumContents(DATA_NODE, dnCurDir); String bpid = cluster.getNamesystem(0).getBlockPoolId(); File bpCurDir = new File(BlockPoolSliceStorage.getBpRoot(bpid, dnCurDir), "current"); blockPoolStorageChecksum = checksumContents(DATA_NODE, bpCurDir); File bpCurFinalizeDir = new File(BlockPoolSliceStorage.getBpRoot(bpid, dnCurDir), "current/" + DataStorage.STORAGE_DIR_FINALIZED); blockPoolFinalizedStorageChecksum = checksumContents(DATA_NODE, bpCurFinalizeDir); File bpCurRbwDir = new File(BlockPoolSliceStorage.getBpRoot(bpid, dnCurDir), "current/" + DataStorage.STORAGE_DIR_RBW); blockPoolRbwStorageChecksum = checksumContents(DATA_NODE, bpCurRbwDir); }
/** * Create a <code>version</code> file for datanode inside the specified * parent * directory. If such a file already exists, it will be overwritten. * The given version string will be written to the file as the layout * version. None of the parameters may be null. * * @param parent * directory where namenode VERSION file is stored * @param version * StorageInfo to create VERSION file from * @param bpid * Block pool Id * @param bpidToWrite * Block pool Id to write into the version file */ public static void createDataNodeVersionFile(File[] parent, StorageInfo version, String bpid, String bpidToWrite) throws IOException { DataStorage storage = new DataStorage(version, "doNotCare"); File[] versionFiles = new File[parent.length]; for (int i = 0; i < parent.length; i++) { File versionFile = new File(parent[i], "VERSION"); StorageDirectory sd = new StorageDirectory(parent[i].getParentFile()); storage.writeProperties(versionFile, sd); versionFiles[i] = versionFile; File bpDir = BlockPoolSliceStorage.getBpRoot(bpid, parent[i]); createBlockPoolVersionFile(bpDir, version, bpidToWrite); } }
