/** * Runs through the HBase rootdir and creates a reverse lookup map for * table StoreFile names to the full Path. * <br> * Example...<br> * Key = 3944417774205889744 <br> * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744 * * @param fs The file system to use. * @param hbaseRootDir The root directory to scan. * @param errors ErrorReporter instance or null * @return Map keyed by StoreFile name with a value of the full Path. * @throws IOException When scanning the directory fails. */ public static Map<String, Path> getTableStoreFilePathMap( final FileSystem fs, final Path hbaseRootDir, ErrorReporter errors) throws IOException { Map<String, Path> map = new HashMap<String, Path>(); // if this method looks similar to 'getTableFragmentation' that is because // it was borrowed from it. // only include the directory paths to tables for (Path tableDir : FSUtils.getTableDirs(fs, hbaseRootDir)) { getTableStoreFilePathMap(map, fs, hbaseRootDir, FSUtils.getTableName(tableDir), errors); } return map; }
public void checkUnDeletedQueues() throws ReplicationException { undeletedQueueIds = getUnDeletedQueues(); undeletedQueueIds.forEach((replicator, queueIds) -> { queueIds.forEach(queueId -> { ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId); String msg = "Undeleted replication queue for removed peer found: " + String.format("[removedPeerId=%s, replicator=%s, queueId=%s]", queueInfo.getPeerId(), replicator, queueId); errorReporter.reportError(HBaseFsck.ErrorReporter.ERROR_CODE.UNDELETED_REPLICATION_QUEUE, msg); }); }); undeletedHFileRefsPeerIds = getUndeletedHFileRefsPeers(); undeletedHFileRefsPeerIds.stream() .map( peerId -> "Undeleted replication hfile-refs queue for removed peer " + peerId + " found") .forEach(msg -> errorReporter .reportError(HBaseFsck.ErrorReporter.ERROR_CODE.UNDELETED_REPLICATION_QUEUE, msg)); }
public TableLockChecker(ZooKeeperWatcher zkWatcher, ErrorReporter errorReporter) { this.zkWatcher = zkWatcher; this.errorReporter = errorReporter; expireTimeout = zkWatcher.getConfiguration().getLong( TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT, TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS); }
public void checkTableLocks() throws IOException { TableLockManager tableLockManager = TableLockManager.createTableLockManager(zkWatcher.getConfiguration(), zkWatcher, null); final long expireDate = EnvironmentEdgeManager.currentTime() - expireTimeout; MetadataHandler handler = new MetadataHandler() { @Override public void handleMetadata(byte[] ownerMetadata) { ZooKeeperProtos.TableLock data = TableLockManager.fromBytes(ownerMetadata); String msg = "Table lock acquire attempt found:"; if (data != null) { msg = msg + String.format("[tableName=%s:%s, lockOwner=%s, threadId=%s, " + "purpose=%s, isShared=%s, createTime=%s]", data.getTableName().getNamespace().toStringUtf8(), data.getTableName().getQualifier().toStringUtf8(), ProtobufUtil.toServerName(data.getLockOwner()), data.getThreadId(), data.getPurpose(), data.getIsShared(), data.getCreateTime()); } if (data != null && data.hasCreateTime() && data.getCreateTime() < expireDate) { errorReporter.reportError(HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK, msg); } else { errorReporter.print(msg); } } }; tableLockManager.visitAllLocks(handler); }
/** * Print a human readable summary of hfile quarantining operations. * @param out */ public void report(ErrorReporter out) { out.print("Checked " + hfilesChecked.get() + " hfile for corruption"); out.print(" HFiles corrupted: " + corrupted.size()); if (inQuarantineMode) { out.print(" HFiles successfully quarantined: " + quarantined.size()); for (Path sq : quarantined) { out.print(" " + sq); } out.print(" HFiles failed quarantine: " + failures.size()); for (Path fq : failures) { out.print(" " + fq); } } out.print(" HFiles moved while checking: " + missing.size()); for (Path mq : missing) { out.print(" " + mq); } String initialState = (corrupted.size() == 0) ? "OK" : "CORRUPTED"; String fixedState = (corrupted.size() == quarantined.size()) ? "OK" : "CORRUPTED"; if (inQuarantineMode) { out.print("Summary: " + initialState + " => " + fixedState); } else { out.print("Summary: " + initialState); } }
public void checkTableLocks() throws IOException { TableLockManager tableLockManager = TableLockManager.createTableLockManager(zkWatcher.getConfiguration(), zkWatcher, null); final long expireDate = EnvironmentEdgeManager.currentTime() - expireTimeout; MetadataHandler handler = new MetadataHandler() { @Override public void handleMetadata(byte[] ownerMetadata) { ZooKeeperProtos.TableLock data = TableLockManager.fromBytes(ownerMetadata); String msg = "Table lock acquire attempt found:"; if (data != null) { msg = msg + String.format("[tableName=%s, lockOwner=%s, threadId=%s, " + "purpose=%s, isShared=%s, createTime=%s]", Bytes.toString(data.getTableName().toByteArray()), ProtobufUtil.toServerName(data.getLockOwner()), data.getThreadId(), data.getPurpose(), data.getIsShared(), data.getCreateTime()); } if (data != null && data.hasCreateTime() && data.getCreateTime() < expireDate) { errorReporter.reportError(HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK, msg); } else { errorReporter.print(msg); } } }; tableLockManager.visitAllLocks(handler); }
public void checkTableLocks() throws IOException { TableLockManager tableLockManager = TableLockManager.createTableLockManager(zkWatcher.getConfiguration(), zkWatcher, null); final long expireDate = EnvironmentEdgeManager.currentTimeMillis() - expireTimeout; MetadataHandler handler = new MetadataHandler() { @Override public void handleMetadata(byte[] ownerMetadata) { ZooKeeperProtos.TableLock data = TableLockManager.fromBytes(ownerMetadata); String msg = "Table lock acquire attempt found:"; if (data != null) { msg = msg + String.format("[tableName=%s, lockOwner=%s, threadId=%s, " + "purpose=%s, isShared=%s, createTime=%s]", Bytes.toString(data.getTableName().toByteArray()), ProtobufUtil.toServerName(data.getLockOwner()), data.getThreadId(), data.getPurpose(), data.getIsShared(), data.getCreateTime()); } if (data != null && data.hasCreateTime() && data.getCreateTime() < expireDate) { errorReporter.reportError(HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK, msg); } else { errorReporter.print(msg); } } }; tableLockManager.visitAllLocks(handler); }
public void checkTableLocks() throws IOException { TableLockManager tableLockManager = TableLockManager.createTableLockManager(zkWatcher.getConfiguration(), zkWatcher, null); final long expireDate = EnvironmentEdgeManager.currentTimeMillis() - expireTimeout; MetadataHandler handler = new MetadataHandler() { @Override public void handleMetadata(byte[] ownerMetadata) { ZooKeeperProtos.TableLock data = TableLockManager.fromBytes(ownerMetadata); String msg = "Table lock acquire attempt found:"; if (data != null) { msg = msg + String.format("[tableName=%s, lockOwner=%s, threadId=%s, " + "purpose=%s, isShared=%s, createTime=%s]", Bytes.toString(data.getTableName().toByteArray()), ProtobufUtil.toServerName(data.getLockOwner()), data.getThreadId(), data.getPurpose(), data.getIsShared(), data.getCreateTime()); } if (data.hasCreateTime() && data.getCreateTime() < expireDate) { errorReporter.reportError(HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK, msg); } else { errorReporter.print(msg); } } }; tableLockManager.visitAllLocks(handler); }
/** * Runs through the HBase rootdir/tablename and creates a reverse lookup map for * table StoreFile names to the full Path. * <br> * Example...<br> * Key = 3944417774205889744 <br> * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744 * * @param map map to add values. If null, this method will create and populate one to return * @param fs The file system to use. * @param hbaseRootDir The root directory to scan. * @param tableName name of the table to scan. * @param errors ErrorReporter instance or null * @return Map keyed by StoreFile name with a value of the full Path. * @throws IOException When scanning the directory fails. */ public static Map<String, Path> getTableStoreFilePathMap(Map<String, Path> map, final FileSystem fs, final Path hbaseRootDir, TableName tableName, ErrorReporter errors) throws IOException { if (map == null) { map = new HashMap<String, Path>(); } // only include the directory paths to tables Path tableDir = FSUtils.getTableDir(hbaseRootDir, tableName); // Inside a table, there are compaction.dir directories to skip. Otherwise, all else // should be regions. PathFilter familyFilter = new FamilyDirFilter(fs); FileStatus[] regionDirs = fs.listStatus(tableDir, new RegionDirFilter(fs)); for (FileStatus regionDir : regionDirs) { if (null != errors) { errors.progress(); } Path dd = regionDir.getPath(); // else its a region name, now look in region for families FileStatus[] familyDirs = fs.listStatus(dd, familyFilter); for (FileStatus familyDir : familyDirs) { if (null != errors) { errors.progress(); } Path family = familyDir.getPath(); if (family.getName().equals(HConstants.RECOVERED_EDITS_DIR)) { continue; } // now in family, iterate over the StoreFiles and // put in map FileStatus[] familyStatus = fs.listStatus(family); for (FileStatus sfStatus : familyStatus) { if (null != errors) { errors.progress(); } Path sf = sfStatus.getPath(); map.put( sf.getName(), sf); } } } return map; }
/** * Runs through the HBase rootdir and creates a reverse lookup map for * table StoreFile names to the full Path. * <br> * Example...<br> * Key = 3944417774205889744 <br> * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744 * * @param fs The file system to use. * @param hbaseRootDir The root directory to scan. * @param sfFilter optional path filter to apply to store files * @param executor optional executor service to parallelize this operation * @param errors ErrorReporter instance or null * @return Map keyed by StoreFile name with a value of the full Path. * @throws IOException When scanning the directory fails. * @throws InterruptedException */ public static Map<String, Path> getTableStoreFilePathMap( final FileSystem fs, final Path hbaseRootDir, PathFilter sfFilter, ExecutorService executor, ErrorReporter errors) throws IOException, InterruptedException { ConcurrentHashMap<String, Path> map = new ConcurrentHashMap<>(1024, 0.75f, 32); // if this method looks similar to 'getTableFragmentation' that is because // it was borrowed from it. // only include the directory paths to tables for (Path tableDir : FSUtils.getTableDirs(fs, hbaseRootDir)) { getTableStoreFilePathMap(map, fs, hbaseRootDir, FSUtils.getTableName(tableDir), sfFilter, executor, errors); } return map; }
public ReplicationChecker(Configuration conf, ZKWatcher zkw, ErrorReporter errorReporter) { this.peerStorage = ReplicationStorageFactory.getReplicationPeerStorage(zkw, conf); this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, conf); this.errorReporter = errorReporter; }
public boolean hasUnDeletedQueues() { return errorReporter.getErrorList() .contains(HBaseFsck.ErrorReporter.ERROR_CODE.UNDELETED_REPLICATION_QUEUE); }