/** * Add the specified store file to the stats * @param region region encoded Name * @param family family name * @param storeFile store file name * @return the store file information */ FileInfo addStoreFile(final HRegionInfo region, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { HFileLink link = HFileLink.build(conf, snapshotTable, region.getEncodedName(), family, storeFile.getName()); boolean isCorrupted = false; boolean inArchive = false; long size = -1; try { if ((inArchive = fs.exists(link.getArchivePath()))) { size = fs.getFileStatus(link.getArchivePath()).getLen(); hfileArchiveSize.addAndGet(size); hfileArchiveCount.incrementAndGet(); } else { size = link.getFileStatus(fs).getLen(); hfileSize.addAndGet(size); hfilesCount.incrementAndGet(); } isCorrupted = (storeFile.hasFileSize() && storeFile.getFileSize() != size); if (isCorrupted) hfilesCorrupted.incrementAndGet(); } catch (FileNotFoundException e) { hfilesMissing.incrementAndGet(); } return new FileInfo(inArchive, size, isCorrupted); }
/** * Returns a StoreFileInfo from the given FileStatus. Secondary replicas refer to the * files of the primary region, so an HFileLink is used to construct the StoreFileInfo. This * way ensures that the secondary will be able to continue reading the store files even if * they are moved to archive after compaction * @throws IOException */ public static StoreFileInfo getStoreFileInfo(Configuration conf, FileSystem fs, HRegionInfo regionInfo, HRegionInfo regionInfoForFs, String familyName, Path path) throws IOException { // if this is a primary region, just return the StoreFileInfo constructed from path if (regionInfo.equals(regionInfoForFs)) { return new StoreFileInfo(conf, fs, path); } // else create a store file link. The link file does not exists on filesystem though. HFileLink link = HFileLink.build(conf, regionInfoForFs.getTable(), regionInfoForFs.getEncodedName(), familyName, path.getName()); if (StoreFileInfo.isReference(path)) { Reference reference = Reference.read(fs, path); return new StoreFileInfo(conf, fs, link.getFileStatus(fs), reference); } return new StoreFileInfo(conf, fs, link.getFileStatus(fs), link); }
/** * Return if the specified file is a valid store file or not. * * @param fileStatus The {@link FileStatus} of the file * @return <tt>true</tt> if the file is valid */ public static boolean isValid(final FileStatus fileStatus) throws IOException { final Path p = fileStatus.getPath(); if (fileStatus.isDirectory()) return false; // Check for empty hfile. Should never be the case but can happen // after data loss in hdfs for whatever reason (upgrade, etc.): HBASE-646 // NOTE: that the HFileLink is just a name, so it's an empty file. if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0) { LOG.warn("Skipping " + p + " because it is empty. HBASE-646 DATA LOSS?"); return false; } return validateStoreFileName(p.getName()); }
@Test public void testHFileLink() throws Exception { // pass a link, and verify that correct paths are returned. Path rootDir = FSUtils.getRootDir(TEST_UTIL.getConfiguration()); Path aFileLink = new Path(rootDir, "table/2086db948c48/cf/table=21212abcdc33-0906db948c48"); Path preNamespaceTablePath = new Path(rootDir, "table/21212abcdc33/cf/0906db948c48"); Path preNamespaceArchivePath = new Path(rootDir, ".archive/table/21212abcdc33/cf/0906db948c48"); Path preNamespaceTempPath = new Path(rootDir, ".tmp/table/21212abcdc33/cf/0906db948c48"); boolean preNSTablePathExists = false; boolean preNSArchivePathExists = false; boolean preNSTempPathExists = false; assertTrue(HFileLink.isHFileLink(aFileLink)); HFileLink hFileLink = HFileLink.buildFromHFileLinkPattern(TEST_UTIL.getConfiguration(), aFileLink); assertTrue(hFileLink.getArchivePath().toString().startsWith(rootDir.toString())); HFileV1Detector t = new HFileV1Detector(); t.setConf(TEST_UTIL.getConfiguration()); FileLink fileLink = t.getFileLinkWithPreNSPath(aFileLink); //assert it has 6 paths (2 NS, 2 Pre NS, and 2 .tmp) to look. assertTrue(fileLink.getLocations().length == 6); for (Path p : fileLink.getLocations()) { if (p.equals(preNamespaceArchivePath)) preNSArchivePathExists = true; if (p.equals(preNamespaceTablePath)) preNSTablePathExists = true; if (p.equals(preNamespaceTempPath)) preNSTempPathExists = true; } assertTrue(preNSArchivePathExists & preNSTablePathExists & preNSTempPathExists); }
private void verifyRestore(final Path rootDir, final HTableDescriptor sourceHtd, final HTableDescriptor htdClone) throws IOException { List<String> files = SnapshotTestingUtils.listHFileNames(fs, FSUtils.getTableDir(rootDir, htdClone.getTableName())); assertEquals(12, files.size()); for (int i = 0; i < files.size(); i += 2) { String linkFile = files.get(i); String refFile = files.get(i+1); assertTrue(linkFile + " should be a HFileLink", HFileLink.isHFileLink(linkFile)); assertTrue(refFile + " should be a Referene", StoreFileInfo.isReference(refFile)); assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(linkFile)); Path refPath = getReferredToFile(refFile); LOG.debug("get reference name for file " + refFile + " = " + refPath); assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName())); assertEquals(linkFile, refPath.getName()); } }
@Test public void testEqualsWithLink() throws IOException { Path origin = new Path("/origin"); Path tmp = new Path("/tmp"); Path archive = new Path("/archive"); HFileLink link1 = new HFileLink(new Path(origin, "f1"), new Path(tmp, "f1"), new Path(archive, "f1")); HFileLink link2 = new HFileLink(new Path(origin, "f1"), new Path(tmp, "f1"), new Path(archive, "f1")); StoreFileInfo info1 = new StoreFileInfo(TEST_UTIL.getConfiguration(), TEST_UTIL.getTestFileSystem(), null, link1); StoreFileInfo info2 = new StoreFileInfo(TEST_UTIL.getConfiguration(), TEST_UTIL.getTestFileSystem(), null, link2); assertEquals(info1, info2); assertEquals(info1.hashCode(), info2.hashCode()); }
/** * Verify the validity of the snapshot * * @param conf The current {@link Configuration} instance. * @param fs {@link FileSystem} * @param snapshotDir {@link Path} to the Snapshot directory of the snapshot to verify * @param snapshotDesc the {@link SnapshotDescription} of the snapshot to verify * @throws CorruptedSnapshotException if the snapshot is corrupted * @throws IOException if an error occurred while scanning the directory */ public static void verifySnapshot(final Configuration conf, final FileSystem fs, final Path snapshotDir, final SnapshotDescription snapshotDesc) throws IOException { final String table = snapshotDesc.getTable(); visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() { public void storeFile (final String region, final String family, final String hfile) throws IOException { HFileLink link = HFileLink.create(conf, table, region, family, hfile); try { link.getFileStatus(fs); } catch (FileNotFoundException e) { throw new CorruptedSnapshotException("Corrupted snapshot '" + snapshotDesc + "'", e); } } }); }
/** * Returns the store file names in the snapshot. * * @param fs {@link FileSystem} * @param snapshotDir {@link Path} to the Snapshot directory * @throws IOException if an error occurred while scanning the directory * @return the names of hfiles in the specified snaphot */ public static Set<String> getHFileNames(final FileSystem fs, final Path snapshotDir) throws IOException { final Set<String> names = new HashSet<String>(); visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() { public void storeFile (final String region, final String family, final String hfile) throws IOException { if (HFileLink.isHFileLink(hfile)) { names.add(HFileLink.getReferencedHFileName(hfile)); } else { names.add(hfile); } } }); return names; }
/** * Returns the location where the inputPath will be copied. * - hfiles are encoded as hfile links hfile-region-table * - logs are encoded as serverName/logName */ private Path getOutputPath(final Path inputPath) throws IOException { Path path; if (HFileLink.isHFileLink(inputPath) || StoreFile.isReference(inputPath)) { String family = inputPath.getParent().getName(); String table = HFileLink.getReferencedTableName(inputPath.getName()); String region = HFileLink.getReferencedRegionName(inputPath.getName()); String hfile = HFileLink.getReferencedHFileName(inputPath.getName()); path = new Path(table, new Path(region, new Path(family, hfile))); } else if (isHLogLinkPath(inputPath)) { String logName = inputPath.getName(); path = new Path(new Path(outputRoot, HConstants.HREGION_OLDLOGDIR_NAME), logName); } else { path = inputPath; } return new Path(outputArchive, path); }
private FileStatus getFileStatus(final FileSystem fs, final Path path) { try { if (HFileLink.isHFileLink(path) || StoreFile.isReference(path)) { HFileLink link = new HFileLink(inputRoot, inputArchive, path); return link.getFileStatus(fs); } else if (isHLogLinkPath(path)) { String serverName = path.getParent().getName(); String logName = path.getName(); return new HLogLink(inputRoot, serverName, logName).getFileStatus(fs); } return fs.getFileStatus(path); } catch (IOException e) { LOG.warn("Unable to get the status for file=" + path); return null; } }
/** * Add the specified store file to the stats * @param region region encoded Name * @param family family name * @param hfile store file name * @return the store file information */ FileInfo addStoreFile(final String region, final String family, final String hfile) throws IOException { String table = this.snapshot.getTable(); HFileLink link = HFileLink.create(conf, table, region, family, hfile); boolean inArchive = false; long size = -1; try { if ((inArchive = fs.exists(link.getArchivePath()))) { size = fs.getFileStatus(link.getArchivePath()).getLen(); hfileArchiveSize += size; hfileArchiveCount++; } else { size = link.getFileStatus(fs).getLen(); hfileSize += size; hfilesCount++; } } catch (FileNotFoundException e) { hfilesMissing++; } return new FileInfo(inArchive, size); }
public Reader(FileSystem fs, Path path, HFileLink hfileLink, long size, CacheConfig cacheConf, DataBlockEncoding preferredEncodingInCache, boolean closeIStream) throws IOException { super(path); pWinterPath = path; FSDataInputStream in = hfileLink.open(fs); FSDataInputStream inNoChecksum = in; if (fs instanceof HFileSystem) { FileSystem noChecksumFs = ((HFileSystem) fs).getNoChecksumFs(); inNoChecksum = hfileLink.open(noChecksumFs); } reader = HFile.createReaderWithEncoding(fs, path, in, inNoChecksum, size, cacheConf, preferredEncodingInCache, closeIStream); bloomFilterType = BloomType.NONE; }
/** * Corrupt the specified snapshot by deleting some files. * * @param util {@link HBaseTestingUtility} * @param snapshotName name of the snapshot to corrupt * @return array of the corrupted HFiles * @throws IOException on unexecpted error reading the FS */ public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName) throws IOException { final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem(); final FileSystem fs = mfs.getFileSystem(); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, mfs.getRootDir()); SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); final String table = snapshotDesc.getTable(); final ArrayList corruptedFiles = new ArrayList(); SnapshotReferenceUtil.visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() { public void storeFile (final String region, final String family, final String hfile) throws IOException { HFileLink link = HFileLink.create(util.getConfiguration(), table, region, family, hfile); if (corruptedFiles.size() % 2 == 0) { fs.delete(link.getAvailablePath(fs)); corruptedFiles.add(hfile); } } }); assertTrue(corruptedFiles.size() > 0); return corruptedFiles; }
/** * Returns the location where the inputPath will be copied. */ private Path getOutputPath(final SnapshotFileInfo inputInfo) throws IOException { Path path = null; switch (inputInfo.getType()) { case HFILE: Path inputPath = new Path(inputInfo.getHfile()); String family = inputPath.getParent().getName(); TableName table =HFileLink.getReferencedTableName(inputPath.getName()); String region = HFileLink.getReferencedRegionName(inputPath.getName()); String hfile = HFileLink.getReferencedHFileName(inputPath.getName()); path = new Path(FSUtils.getTableDir(new Path("./"), table), new Path(region, new Path(family, hfile))); break; case WAL: Path oldLogsDir = new Path(outputRoot, HConstants.HREGION_OLDLOGDIR_NAME); path = new Path(oldLogsDir, inputInfo.getWalName()); break; default: throw new IOException("Invalid File Type: " + inputInfo.getType().toString()); } return new Path(outputArchive, path); }
/** * Add the specified store file to the stats * @param region region encoded Name * @param family family name * @param hfile store file name * @return the store file information */ FileInfo addStoreFile(final HRegionInfo region, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { HFileLink link = HFileLink.create(conf, snapshotTable, region.getEncodedName(), family, storeFile.getName()); boolean isCorrupted = false; boolean inArchive = false; long size = -1; try { if ((inArchive = fs.exists(link.getArchivePath()))) { size = fs.getFileStatus(link.getArchivePath()).getLen(); hfileArchiveSize.addAndGet(size); hfileArchiveCount.incrementAndGet(); } else { size = link.getFileStatus(fs).getLen(); hfileSize.addAndGet(size); hfilesCount.incrementAndGet(); } isCorrupted = (storeFile.hasFileSize() && storeFile.getFileSize() != size); if (isCorrupted) hfilesCorrupted.incrementAndGet(); } catch (FileNotFoundException e) { hfilesMissing.incrementAndGet(); } return new FileInfo(inArchive, size, isCorrupted); }
/** * Returns a StoreFileInfo from the given FileStatus. Secondary replicas refer to the * files of the primary region, so an HFileLink is used to construct the StoreFileInfo. This * way ensures that the secondary will be able to continue reading the store files even if * they are moved to archive after compaction * @throws IOException */ public static StoreFileInfo getStoreFileInfo(Configuration conf, FileSystem fs, HRegionInfo regionInfo, HRegionInfo regionInfoForFs, String familyName, FileStatus status) throws IOException { // if this is a primary region, just return the StoreFileInfo constructed from path if (regionInfo.equals(regionInfoForFs)) { return new StoreFileInfo(conf, fs, status); } // else create a store file link. The link file does not exists on filesystem though. HFileLink link = new HFileLink(conf, HFileLink.createPath(regionInfoForFs.getTable(), regionInfoForFs.getEncodedName() , familyName, status.getPath().getName())); return new StoreFileInfo(conf, fs, status, link); }
/** * Return if the specified file is a valid store file or not. * @param fileStatus The {@link FileStatus} of the file * @return <tt>true</tt> if the file is valid */ public static boolean isValid(final FileStatus fileStatus) throws IOException { final Path p = fileStatus.getPath(); if (fileStatus.isDirectory()) return false; // Check for empty hfile. Should never be the case but can happen // after data loss in hdfs for whatever reason (upgrade, etc.): HBASE-646 // NOTE: that the HFileLink is just a name, so it's an empty file. if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0) { LOG.warn("Skipping " + p + " because it is empty. HBASE-646 DATA LOSS?"); return false; } return validateStoreFileName(p.getName()); }
private void verifyRestore(final Path rootDir, final HTableDescriptor sourceHtd, final HTableDescriptor htdClone) throws IOException { String[] files = SnapshotTestingUtils.listHFileNames(fs, FSUtils.getTableDir(rootDir, htdClone.getTableName())); assertEquals(12, files.length); for (int i = 0; i < files.length; i += 2) { String linkFile = files[i]; String refFile = files[i+1]; assertTrue(linkFile + " should be a HFileLink", HFileLink.isHFileLink(linkFile)); assertTrue(refFile + " should be a Referene", StoreFileInfo.isReference(refFile)); assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(linkFile)); Path refPath = getReferredToFile(refFile); LOG.debug("get reference name for file " + refFile + " = " + refPath); assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName())); assertEquals(linkFile, refPath.getName()); } }
/** * Verify the validity of the snapshot * * @param conf The current {@link Configuration} instance. * @param fs {@link FileSystem} * @param snapshotDir {@link Path} to the Snapshot directory of the snapshot to verify * @param snapshotDesc the {@link SnapshotDescription} of the snapshot to verify * @throws CorruptedSnapshotException if the snapshot is corrupted * @throws IOException if an error occurred while scanning the directory */ public static void verifySnapshot(final Configuration conf, final FileSystem fs, final Path snapshotDir, final SnapshotDescription snapshotDesc) throws IOException { final TableName table = TableName.valueOf(snapshotDesc.getTable()); visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() { public void storeFile (final String region, final String family, final String hfile) throws IOException { HFileLink link = HFileLink.create(conf, table, region, family, hfile); try { link.getFileStatus(fs); } catch (FileNotFoundException e) { throw new CorruptedSnapshotException("Corrupted snapshot '" + snapshotDesc + "'", e); } } }); }
/** * Returns the location where the inputPath will be copied. * - hfiles are encoded as hfile links hfile-region-table * - logs are encoded as serverName/logName */ private Path getOutputPath(final Path inputPath) throws IOException { Path path; if (HFileLink.isHFileLink(inputPath) || StoreFileInfo.isReference(inputPath)) { String family = inputPath.getParent().getName(); TableName table = HFileLink.getReferencedTableName(inputPath.getName()); String region = HFileLink.getReferencedRegionName(inputPath.getName()); String hfile = HFileLink.getReferencedHFileName(inputPath.getName()); path = new Path(FSUtils.getTableDir(new Path("./"), table), new Path(region, new Path(family, hfile))); } else if (isHLogLinkPath(inputPath)) { String logName = inputPath.getName(); path = new Path(new Path(outputRoot, HConstants.HREGION_OLDLOGDIR_NAME), logName); } else { path = inputPath; } return new Path(outputArchive, path); }
/** * Try to open the "source" file. * Throws an IOException if the communication with the inputFs fail or * if the file is not found. */ private FSDataInputStream openSourceFile(Context context, final Path path) throws IOException { try { if (HFileLink.isHFileLink(path) || StoreFileInfo.isReference(path)) { return new HFileLink(inputRoot, inputArchive, path).open(inputFs); } else if (isHLogLinkPath(path)) { String serverName = path.getParent().getName(); String logName = path.getName(); return new HLogLink(inputRoot, serverName, logName).open(inputFs); } return inputFs.open(path); } catch (IOException e) { context.getCounter(Counter.MISSING_FILES).increment(1); LOG.error("Unable to open source file=" + path, e); throw e; } }
/** * Add the specified store file to the stats * @param region region encoded Name * @param family family name * @param hfile store file name * @return the store file information */ FileInfo addStoreFile(final String region, final String family, final String hfile) throws IOException { TableName table = snapshotTable; HFileLink link = HFileLink.create(conf, table, region, family, hfile); boolean inArchive = false; long size = -1; try { if ((inArchive = fs.exists(link.getArchivePath()))) { size = fs.getFileStatus(link.getArchivePath()).getLen(); hfileArchiveSize += size; hfileArchiveCount++; } else { size = link.getFileStatus(fs).getLen(); hfileSize += size; hfilesCount++; } } catch (FileNotFoundException e) { hfilesMissing++; } return new FileInfo(inArchive, size); }
/** * Return if the specified file is a valid store file or not. * @param fileStatus The {@link FileStatus} of the file * @return <tt>true</tt> if the file is valid */ public static boolean isValid(final FileStatus fileStatus) throws IOException { final Path p = fileStatus.getPath(); if (fileStatus.isDir()) return false; // Check for empty hfile. Should never be the case but can happen // after data loss in hdfs for whatever reason (upgrade, etc.): HBASE-646 // NOTE: that the HFileLink is just a name, so it's an empty file. if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0) { LOG.warn("Skipping " + p + " because it is empty. HBASE-646 DATA LOSS?"); return false; } return validateStoreFileName(p.getName()); }
public Reader(FileSystem fs, Path path, HFileLink hfileLink, long size, CacheConfig cacheConf, DataBlockEncoding preferredEncodingInCache, boolean closeIStream) throws IOException { super(path); FSDataInputStream in = hfileLink.open(fs); FSDataInputStream inNoChecksum = in; if (fs instanceof HFileSystem) { FileSystem noChecksumFs = ((HFileSystem)fs).getNoChecksumFs(); inNoChecksum = hfileLink.open(noChecksumFs); } reader = HFile.createReaderWithEncoding(fs, path, in, inNoChecksum, size, cacheConf, preferredEncodingInCache, closeIStream); bloomFilterType = BloomType.NONE; }
private FileStatus getLinkedFileStatus(HFileLink link) throws IOException { Path[] locations = link.getLocations(); FileStatus file; for (Path location : locations) { if (location != null) { try { file = fs.getFileStatus(location); if (file != null) { return file; } } catch (FileNotFoundException e) { } } } LOG.warn("The file " + link + " links to can not be found"); return null; }
/** Check if for a give file in archive, if there are other snapshots/tables still * reference it. * @param filePath file path in archive * @param snapshotFilesMap a map for store files in snapshots about how many snapshots refer * to it. * @return true or false */ private boolean isArchivedFileStillReferenced(final Path filePath, final Map<Path, Integer> snapshotFilesMap) { Integer c = snapshotFilesMap.get(filePath); // Check if there are other snapshots or table from clone_snapshot() (via back-reference) // still reference to it. if ((c != null) && (c == 1)) { Path parentDir = filePath.getParent(); Path backRefDir = HFileLink.getBackReferencesDir(parentDir, filePath.getName()); try { if (FSUtils.listStatus(fs, backRefDir) == null) { return false; } } catch (IOException e) { // For the purpose of this function, IOException is ignored and treated as // the file is still being referenced. } } return true; }
/** * Returns a StoreFileInfo from the given FileStatus. Secondary replicas refer to the * files of the primary region, so an HFileLink is used to construct the StoreFileInfo. This * way ensures that the secondary will be able to continue reading the store files even if * they are moved to archive after compaction * @throws IOException */ public static StoreFileInfo getStoreFileInfo(Configuration conf, FileSystem fs, RegionInfo regionInfo, RegionInfo regionInfoForFs, String familyName, Path path) throws IOException { // if this is a primary region, just return the StoreFileInfo constructed from path if (RegionInfo.COMPARATOR.compare(regionInfo, regionInfoForFs) == 0) { return new StoreFileInfo(conf, fs, path); } // else create a store file link. The link file does not exists on filesystem though. HFileLink link = HFileLink.build(conf, regionInfoForFs.getTable(), regionInfoForFs.getEncodedName(), familyName, path.getName()); if (StoreFileInfo.isReference(path)) { Reference reference = Reference.read(fs, path); return new StoreFileInfo(conf, fs, link.getFileStatus(fs), reference); } return new StoreFileInfo(conf, fs, link.getFileStatus(fs), link); }
private void verifyRestore(final Path rootDir, final TableDescriptor sourceHtd, final TableDescriptor htdClone) throws IOException { List<String> files = SnapshotTestingUtils.listHFileNames(fs, FSUtils.getTableDir(rootDir, htdClone.getTableName())); assertEquals(12, files.size()); for (int i = 0; i < files.size(); i += 2) { String linkFile = files.get(i); String refFile = files.get(i+1); assertTrue(linkFile + " should be a HFileLink", HFileLink.isHFileLink(linkFile)); assertTrue(refFile + " should be a Referene", StoreFileInfo.isReference(refFile)); assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(linkFile)); Path refPath = getReferredToFile(refFile); LOG.debug("get reference name for file " + refFile + " = " + refPath); assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName())); assertEquals(linkFile, refPath.getName()); } }
@Test public void testEqualsWithLink() throws IOException { Path origin = new Path("/origin"); Path tmp = TEST_UTIL.getDataTestDir(); Path mob = new Path("/mob"); Path archive = new Path("/archive"); HFileLink link1 = new HFileLink(new Path(origin, "f1"), new Path(tmp, "f1"), new Path(mob, "f1"), new Path(archive, "f1")); HFileLink link2 = new HFileLink(new Path(origin, "f1"), new Path(tmp, "f1"), new Path(mob, "f1"), new Path(archive, "f1")); StoreFileInfo info1 = new StoreFileInfo(TEST_UTIL.getConfiguration(), TEST_UTIL.getTestFileSystem(), null, link1); StoreFileInfo info2 = new StoreFileInfo(TEST_UTIL.getConfiguration(), TEST_UTIL.getTestFileSystem(), null, link2); assertEquals(info1, info2); assertEquals(info1.hashCode(), info2.hashCode()); }
/** * Returns the location where the inputPath will be copied. */ private Path getOutputPath(final SnapshotFileInfo inputInfo) throws IOException { Path path = null; switch (inputInfo.getType()) { case HFILE: Path inputPath = new Path(inputInfo.getHfile()); String family = inputPath.getParent().getName(); TableName table =HFileLink.getReferencedTableName(inputPath.getName()); String region = HFileLink.getReferencedRegionName(inputPath.getName()); String hfile = HFileLink.getReferencedHFileName(inputPath.getName()); path = new Path(FSUtils.getTableDir(new Path("./"), table), new Path(region, new Path(family, hfile))); break; case WAL: LOG.warn("snapshot does not keeps WALs: " + inputInfo); break; default: throw new IOException("Invalid File Type: " + inputInfo.getType().toString()); } return new Path(outputArchive, path); }
/** * Add the specified store file to the stats * @param region region encoded Name * @param family family name * @param hfile store file name * @return the store file information */ FileInfo addStoreFile(final HRegionInfo region, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { HFileLink link = HFileLink.create(conf, snapshotTable, region.getEncodedName(), family, storeFile.getName()); boolean inArchive = false; long size = -1; try { if ((inArchive = fs.exists(link.getArchivePath()))) { size = fs.getFileStatus(link.getArchivePath()).getLen(); hfileArchiveSize.addAndGet(size); hfileArchiveCount.incrementAndGet(); } else { size = link.getFileStatus(fs).getLen(); hfileSize.addAndGet(size); hfilesCount.incrementAndGet(); } } catch (FileNotFoundException e) { hfilesMissing.incrementAndGet(); } return new FileInfo(inArchive, size); }
