public MasterFileSystem(Server master, MasterServices services) throws IOException { this.conf = master.getConfiguration(); this.master = master; this.services = services; // Set filesystem to be that of this.rootdir else we get complaints about // mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is // default localfs. Presumption is that rootdir is fully-qualified before // we get to here with appropriate fs scheme. this.rootdir = FSUtils.getRootDir(conf); this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY); // Cover both bases, the old way of setting default fs and the new. // We're supposed to run on 0.20 and 0.21 anyways. this.fs = this.rootdir.getFileSystem(conf); FSUtils.setFsDefault(conf, new Path(this.fs.getUri())); // make sure the fs has the same conf fs.setConf(conf); // setup the filesystem variable // set up the archived logs path this.oldLogDir = createInitialFileSystemLayout(); HFileSystem.addLocationsOrderInterceptor(conf); this.splitLogManager = new SplitLogManager(master, master.getConfiguration(), master, services, master.getServerName()); this.distributedLogReplay = this.splitLogManager.isLogReplaying(); }
ServerManager(final Server master, final MasterServices services, final boolean connect) throws IOException { this.master = master; this.services = services; Configuration c = master.getConfiguration(); maxSkew = c.getLong("hbase.master.maxclockskew", 30000); warningSkew = c.getLong("hbase.master.warningclockskew", 10000); this.connection = connect ? (ClusterConnection)ConnectionFactory.createConnection(c) : null; int pingMaxAttempts = Math.max(1, master.getConfiguration().getInt( "hbase.master.maximum.ping.server.attempts", 10)); int pingSleepInterval = Math.max(1, master.getConfiguration().getInt( "hbase.master.ping.server.retry.sleep.interval", 100)); this.pingRetryCounterFactory = new RetryCounterFactory(pingMaxAttempts, pingSleepInterval); this.rpcControllerFactory = this.connection == null ? null : connection.getRpcControllerFactory(); }
public OpenedRegionHandler(Server server, AssignmentManager assignmentManager, HRegionInfo regionInfo, OpenRegionCoordination coordination, OpenRegionCoordination.OpenRegionDetails ord) { super(server, EventType.RS_ZK_REGION_OPENED); this.assignmentManager = assignmentManager; this.regionInfo = regionInfo; this.coordination = coordination; this.ord = ord; if(regionInfo.isMetaRegion()) { priority = OpenedPriority.META; } else if(regionInfo.getTable() .getNamespaceAsString().equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) { priority = OpenedPriority.SYSTEM; } else { priority = OpenedPriority.USER; } }
/** * Its OK to construct this object even when region-servers are not online. It does lookup the * orphan tasks in coordination engine but it doesn't block waiting for them to be done. * @param server the server instance * @param conf the HBase configuration * @param stopper the stoppable in case anything is wrong * @param master the master services * @param serverName the master server name * @throws IOException */ public SplitLogManager(Server server, Configuration conf, Stoppable stopper, MasterServices master, ServerName serverName) throws IOException { this.server = server; this.conf = conf; this.stopper = stopper; this.choreService = new ChoreService(serverName.toString() + "_splitLogManager_"); if (server.getCoordinatedStateManager() != null) { SplitLogManagerCoordination coordination = ((BaseCoordinatedStateManager) server.getCoordinatedStateManager()) .getSplitLogManagerCoordination(); Set<String> failedDeletions = Collections.synchronizedSet(new HashSet<String>()); SplitLogManagerDetails details = new SplitLogManagerDetails(tasks, master, failedDeletions, serverName); coordination.setDetails(details); coordination.init(); // Determine recovery mode } this.unassignedTimeout = conf.getInt("hbase.splitlog.manager.unassigned.timeout", DEFAULT_UNASSIGNED_TIMEOUT); this.timeoutMonitor = new TimeoutMonitor(conf.getInt("hbase.splitlog.manager.timeoutmonitor.period", 1000), stopper); choreService.scheduleChore(timeoutMonitor); }
private void OpenRegion(Server server, RegionServerServices rss, HTableDescriptor htd, HRegionInfo hri, OpenRegionCoordination coordination) throws IOException, NodeExistsException, KeeperException, DeserializationException { // Create it OFFLINE node, which is what Master set before sending OPEN RPC ZKAssign.createNodeOffline(server.getZooKeeper(), hri, server.getServerName()); OpenRegionCoordination.OpenRegionDetails ord = coordination.getDetailsForNonCoordinatedOpening(); OpenRegionHandler openHandler = new OpenRegionHandler(server, rss, hri, htd, -1, coordination, ord); rss.getRegionsInTransitionInRS().put(hri.getEncodedNameAsBytes(), Boolean.TRUE); openHandler.process(); // This parse is not used? RegionTransition.parseFrom(ZKAssign.getData(server.getZooKeeper(), hri.getEncodedName())); // delete the node, which is what Master do after the region is opened ZKAssign.deleteNode(server.getZooKeeper(), hri.getEncodedName(), EventType.RS_ZK_REGION_OPENED, server.getServerName()); }
@Override public boolean rollback(Server server, RegionServerServices services) throws IOException { if (this.currentRegion.getRegionInfo().getTable().getNameAsString() .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) { if(secondSplit){ super.rollback(server, services); latch.countDown(); return true; } } return super.rollback(server, services); }
@Override public void initialize(Server server) { this.server = server; this.watcher = server.getZooKeeper(); splitLogWorkerCoordination = new ZkSplitLogWorkerCoordination(this, watcher); splitLogManagerCoordination = new ZKSplitLogManagerCoordination(this, watcher); splitTransactionCoordination = new ZKSplitTransactionCoordination(this, watcher); closeRegionCoordination = new ZkCloseRegionCoordination(this, watcher); openRegionCoordination = new ZkOpenRegionCoordination(this, watcher); regionMergeCoordination = new ZkRegionMergeCoordination(this, watcher); }
RegionStates(final Server master, final TableStateManager tableStateManager, final ServerManager serverManager, final RegionStateStore regionStateStore) { this.tableStateManager = tableStateManager; this.regionStateStore = regionStateStore; this.serverManager = serverManager; this.server = master; }
CatalogJanitor(final Server server, final MasterServices services) { super("CatalogJanitor-" + server.getServerName().toShortString(), server, server .getConfiguration().getInt("hbase.catalogjanitor.interval", 300000)); this.server = server; this.services = services; this.connection = server.getConnection(); }
RegionStateStore(final Server server) { Configuration conf = server.getConfiguration(); // No need to persist if using ZK but not migrating noPersistence = ConfigUtil.useZKForAssignment(conf) && !conf.getBoolean("hbase.assignment.usezk.migrating", false); this.server = server; initialized = false; }
public CreateTableHandler(Server server, MasterFileSystem fileSystemManager, HTableDescriptor hTableDescriptor, Configuration conf, HRegionInfo [] newRegions, MasterServices masterServices) { super(server, EventType.C_M_CREATE_TABLE); this.fileSystemManager = fileSystemManager; this.hTableDescriptor = hTableDescriptor; this.conf = conf; this.newRegions = newRegions; this.assignmentManager = masterServices.getAssignmentManager(); this.tableLockManager = masterServices.getTableLockManager(); this.tableLock = this.tableLockManager.writeLock(this.hTableDescriptor.getTableName() , EventType.C_M_CREATE_TABLE.toString()); }
public DisableTableHandler(Server server, TableName tableName, AssignmentManager assignmentManager, TableLockManager tableLockManager, boolean skipTableStateCheck) { super(server, EventType.C_M_DISABLE_TABLE); this.tableName = tableName; this.assignmentManager = assignmentManager; this.tableLockManager = tableLockManager; this.skipTableStateCheck = skipTableStateCheck; }
public EnableTableHandler(Server server, TableName tableName, AssignmentManager assignmentManager, TableLockManager tableLockManager, boolean skipTableStateCheck) { super(server, EventType.C_M_ENABLE_TABLE); this.tableName = tableName; this.assignmentManager = assignmentManager; this.tableLockManager = tableLockManager; this.skipTableStateCheck = skipTableStateCheck; }
public EnableTableHandler(MasterServices services, TableName tableName, AssignmentManager assignmentManager, TableLockManager tableLockManager, boolean skipTableStateCheck) { this((Server)services, tableName, assignmentManager, tableLockManager, skipTableStateCheck); this.services = services; }
public ClosedRegionHandler(Server server, AssignmentManager assignmentManager, HRegionInfo regionInfo) { super(server, EventType.RS_ZK_REGION_CLOSED); this.assignmentManager = assignmentManager; this.regionInfo = regionInfo; if(regionInfo.isMetaRegion()) { priority = ClosedPriority.META; } else { priority = ClosedPriority.USER; } }
public GeneralBulkAssigner(final Server server, final Map<ServerName, List<HRegionInfo>> bulkPlan, final AssignmentManager am, final boolean waitTillAllAssigned) { super(server); this.bulkPlan = bulkPlan; this.assignmentManager = am; this.waitTillAllAssigned = waitTillAllAssigned; }
private int putUpJettyServer() throws IOException { if (!conf.getBoolean("hbase.master.infoserver.redirect", true)) { return -1; } int infoPort = conf.getInt("hbase.master.info.port.orig", HConstants.DEFAULT_MASTER_INFOPORT); // -1 is for disabling info server, so no redirecting if (infoPort < 0 || infoServer == null) { return -1; } String addr = conf.get("hbase.master.info.bindAddress", "0.0.0.0"); if (!Addressing.isLocalAddress(InetAddress.getByName(addr))) { String msg = "Failed to start redirecting jetty server. Address " + addr + " does not belong to this host. Correct configuration parameter: " + "hbase.master.info.bindAddress"; LOG.error(msg); throw new IOException(msg); } RedirectServlet.regionServerInfoPort = infoServer.getPort(); if(RedirectServlet.regionServerInfoPort == infoPort) { return infoPort; } masterJettyServer = new org.mortbay.jetty.Server(); Connector connector = new SelectChannelConnector(); connector.setHost(addr); connector.setPort(infoPort); masterJettyServer.addConnector(connector); masterJettyServer.setStopAtShutdown(true); Context context = new Context(masterJettyServer, "/", Context.NO_SESSIONS); context.addServlet(RedirectServlet.class, "/*"); try { masterJettyServer.start(); } catch (Exception e) { throw new IOException("Failed to start redirecting jetty server", e); } return connector.getLocalPort(); }
public BulkReOpen(final Server server, final Map<ServerName, List<HRegionInfo>> serverToRegions, final AssignmentManager am) { super(server); this.assignmentManager = am; this.rsToRegions = serverToRegions; }
/** * @param watcher * @param sn ServerName * @param master In an instance of a Master. */ ActiveMasterManager(ZooKeeperWatcher watcher, ServerName sn, Server master) { super(watcher); watcher.registerListener(this); this.sn = sn; this.master = master; }
public SplitLogWorker(Server hserver, Configuration conf, RegionServerServices server, TaskExecutor splitTaskExecutor) { this.server = server; this.conf = conf; this.coordination = ((BaseCoordinatedStateManager) hserver.getCoordinatedStateManager()) .getSplitLogWorkerCoordination(); this.server = server; coordination.init(server, conf, splitTaskExecutor, this); }
public PairOfSameType<Region> execute(final Server server, final RegionServerServices services) throws IOException { if (User.isHBaseSecurityEnabled(parent.getBaseConf())) { LOG.warn("Should use execute(Server, RegionServerServices, User)"); } return execute(server, services, null); }
/** * Run the transaction. * @param server Hosting server instance. Can be null when testing * @param services Used to online/offline regions. * @throws IOException If thrown, transaction failed. * Call {@link #rollback(Server, RegionServerServices)} * @return Regions created * @throws IOException * @see #rollback(Server, RegionServerServices) */ @Override public PairOfSameType<Region> execute(final Server server, final RegionServerServices services, User user) throws IOException { this.server = server; this.rsServices = services; useZKForAssignment = server == null ? true : ConfigUtil.useZKForAssignment(server.getConfiguration()); if (useCoordinatedStateManager(server)) { std = ((BaseCoordinatedStateManager) server.getCoordinatedStateManager()) .getSplitTransactionCoordination().getDefaultDetails(); } PairOfSameType<Region> regions = createDaughters(server, services, user); if (this.parent.getCoprocessorHost() != null) { if (user == null) { parent.getCoprocessorHost().preSplitAfterPONR(); } else { try { user.getUGI().doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { parent.getCoprocessorHost().preSplitAfterPONR(); return null; } }); } catch (InterruptedException ie) { InterruptedIOException iioe = new InterruptedIOException(); iioe.initCause(ie); throw iioe; } } } regions = stepsAfterPONR(server, services, regions, user); transition(SplitTransactionPhase.COMPLETED); return regions; }
public PairOfSameType<Region> stepsAfterPONR(final Server server, final RegionServerServices services, final PairOfSameType<Region> regions, User user) throws IOException { openDaughters(server, services, regions.getFirst(), regions.getSecond()); if (useCoordinatedStateManager(server)) { ((BaseCoordinatedStateManager) server.getCoordinatedStateManager()) .getSplitTransactionCoordination().completeSplitTransaction(services, regions.getFirst(), regions.getSecond(), std, parent); } transition(SplitTransactionPhase.BEFORE_POST_SPLIT_HOOK); // Coprocessor callback if (parent.getCoprocessorHost() != null) { if (user == null) { this.parent.getCoprocessorHost().postSplit(regions.getFirst(), regions.getSecond()); } else { try { user.getUGI().doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { parent.getCoprocessorHost().postSplit(regions.getFirst(), regions.getSecond()); return null; } }); } catch (InterruptedException ie) { InterruptedIOException iioe = new InterruptedIOException(); iioe.initCause(ie); throw iioe; } } } transition(SplitTransactionPhase.AFTER_POST_SPLIT_HOOK); return regions; }
DaughterOpener(final Server s, final HRegion r) { super((s == null? "null-services": s.getServerName()) + "-daughterOpener=" + r.getRegionInfo().getEncodedName()); setDaemon(true); this.server = s; this.r = r; }
/** * Open daughter regions, add them to online list and update meta. * @param server * @param daughter * @throws IOException * @throws KeeperException */ void openDaughterRegion(final Server server, final HRegion daughter) throws IOException, KeeperException { HRegionInfo hri = daughter.getRegionInfo(); LoggingProgressable reporter = server == null ? null : new LoggingProgressable(hri, server.getConfiguration().getLong( "hbase.regionserver.split.daughter.open.log.interval", 10000)); daughter.openHRegion(reporter); }
@Test public void testFailAfterPONR() throws IOException, KeeperException { final int rowcount = TEST_UTIL.loadRegion(this.parent, CF); assertTrue(rowcount > 0); int parentRowCount = countRows(this.parent); assertEquals(rowcount, parentRowCount); // Start transaction. SplitTransactionImpl st = prepareGOOD_SPLIT_ROW(); SplitTransactionImpl spiedUponSt = spy(st); Mockito .doThrow(new MockedFailedDaughterOpen()) .when(spiedUponSt) .openDaughterRegion((Server) Mockito.anyObject(), (HRegion) Mockito.anyObject()); // Run the execute. Look at what it returns. boolean expectedException = false; Server mockServer = Mockito.mock(Server.class); when(mockServer.getConfiguration()).thenReturn(TEST_UTIL.getConfiguration()); try { spiedUponSt.execute(mockServer, null); } catch (IOException e) { if (e.getCause() != null && e.getCause() instanceof MockedFailedDaughterOpen) { expectedException = true; } } assertTrue(expectedException); // Run rollback returns that we should restart. assertFalse(spiedUponSt.rollback(null, null)); // Make sure that region a and region b are still in the filesystem, that // they have not been removed; this is supposed to be the case if we go // past point of no return. Path tableDir = this.parent.getRegionFileSystem().getTableDir(); Path daughterADir = new Path(tableDir, spiedUponSt.getFirstDaughter().getEncodedName()); Path daughterBDir = new Path(tableDir, spiedUponSt.getSecondDaughter().getEncodedName()); assertTrue(TEST_UTIL.getTestFileSystem().exists(daughterADir)); assertTrue(TEST_UTIL.getTestFileSystem().exists(daughterBDir)); }
@Override public boolean rollback(final Server server, final RegionServerServices services) throws IOException { if (User.isHBaseSecurityEnabled(parent.getBaseConf())) { LOG.warn("Should use rollback(Server, RegionServerServices, User)"); } return rollback(server, services, null); }
public static HeapMemoryManager create(Configuration conf, FlushRequester memStoreFlusher, Server server, RegionServerAccounting regionServerAccounting) { BlockCache blockCache = CacheConfig.instantiateBlockCache(conf); if (blockCache instanceof ResizableBlockCache) { return new HeapMemoryManager((ResizableBlockCache) blockCache, memStoreFlusher, server, regionServerAccounting); } return null; }
@VisibleForTesting HeapMemoryManager(ResizableBlockCache blockCache, FlushRequester memStoreFlusher, Server server, RegionServerAccounting regionServerAccounting) { Configuration conf = server.getConfiguration(); this.blockCache = blockCache; this.memStoreFlusher = memStoreFlusher; this.server = server; this.regionServerAccounting = regionServerAccounting; this.tunerOn = doInit(conf); this.defaultChorePeriod = conf.getInt(HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, HBASE_RS_HEAP_MEMORY_TUNER_DEFAULT_PERIOD); this.heapOccupancyLowWatermark = conf.getFloat(HConstants.HEAP_OCCUPANCY_LOW_WATERMARK_KEY, HConstants.DEFAULT_HEAP_OCCUPANCY_LOW_WATERMARK); }
public OpenMetaHandler(final Server server, final RegionServerServices rsServices, HRegionInfo regionInfo, final HTableDescriptor htd, long masterSystemTime, OpenRegionCoordination coordination, OpenRegionCoordination.OpenRegionDetails ord) { super(server, rsServices, regionInfo, htd, EventType.M_RS_OPEN_META, masterSystemTime, coordination, ord); }
public CloseRegionHandler(final Server server, final RegionServerServices rsServices, final HRegionInfo regionInfo, final boolean abort, CloseRegionCoordination closeRegionCoordination, CloseRegionCoordination.CloseRegionDetails crd, ServerName destination) { this(server, rsServices, regionInfo, abort, closeRegionCoordination, crd, EventType.M_RS_CLOSE_REGION, destination); }
protected CloseRegionHandler(final Server server, final RegionServerServices rsServices, HRegionInfo regionInfo, boolean abort, CloseRegionCoordination closeRegionCoordination, CloseRegionCoordination.CloseRegionDetails crd, EventType eventType, ServerName destination) { super(server, eventType); this.server = server; this.rsServices = rsServices; this.regionInfo = regionInfo; this.abort = abort; this.destination = destination; this.closeRegionCoordination = closeRegionCoordination; this.closeRegionDetails = crd; useZKForAssignment = ConfigUtil.useZKForAssignment(server.getConfiguration()); }
public OpenRegionHandler(final Server server, final RegionServerServices rsServices, HRegionInfo regionInfo, HTableDescriptor htd, long masterSystemTime, OpenRegionCoordination coordination, OpenRegionCoordination.OpenRegionDetails ord) { this(server, rsServices, regionInfo, htd, EventType.M_RS_OPEN_REGION, masterSystemTime, coordination, ord); }
protected OpenRegionHandler(final Server server, final RegionServerServices rsServices, final HRegionInfo regionInfo, final HTableDescriptor htd, EventType eventType, long masterSystemTime, OpenRegionCoordination coordination, OpenRegionCoordination.OpenRegionDetails ord) { super(server, eventType); this.rsServices = rsServices; this.regionInfo = regionInfo; this.htd = htd; this.coordination = coordination; this.ord = ord; useZKForAssignment = ConfigUtil.useZKForAssignment(server.getConfiguration()); this.masterSystemTime = masterSystemTime; }
PostOpenDeployTasksThread(final HRegion region, final Server server, final RegionServerServices services, final AtomicBoolean signaller, long masterSystemTime) { super("PostOpenDeployTasks:" + region.getRegionInfo().getEncodedName()); this.setDaemon(true); this.server = server; this.services = services; this.region = region; this.signaller = signaller; this.masterSystemTime = masterSystemTime; }
public WALSplitterHandler(final Server server, SplitLogWorkerCoordination coordination, SplitLogWorkerCoordination.SplitTaskDetails splitDetails, CancelableProgressable reporter, AtomicInteger inProgressTasks, TaskExecutor splitTaskExecutor, RecoveryMode mode) { super(server, EventType.RS_LOG_REPLAY); this.splitTaskDetails = splitDetails; this.coordination = coordination; this.reporter = reporter; this.inProgressTasks = inProgressTasks; this.inProgressTasks.incrementAndGet(); this.serverName = server.getServerName(); this.splitTaskExecutor = splitTaskExecutor; this.mode = mode; }
public CloseMetaHandler(final Server server, final RegionServerServices rsServices, final HRegionInfo regionInfo, final boolean abort, CloseRegionCoordination closeRegionCoordination, CloseRegionCoordination.CloseRegionDetails crd) { super(server, rsServices, regionInfo, abort, closeRegionCoordination, crd, EventType.M_RS_CLOSE_META); }
public RegionReplicaFlushHandler(Server server, ClusterConnection connection, RpcRetryingCallerFactory rpcRetryingCallerFactory, RpcControllerFactory rpcControllerFactory, int operationTimeout, HRegion region) { super(server, EventType.RS_REGION_REPLICA_FLUSH); this.connection = connection; this.rpcRetryingCallerFactory = rpcRetryingCallerFactory; this.rpcControllerFactory = rpcControllerFactory; this.operationTimeout = operationTimeout; this.region = region; }
/** @param server */ public LogRoller(final Server server, final RegionServerServices services) { super(); this.server = server; this.services = services; this.rollperiod = this.server.getConfiguration(). getLong("hbase.regionserver.logroll.period", 3600000); this.threadWakeFrequency = this.server.getConfiguration(). getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000); }
public void stepsAfterPONR(final Server server, final RegionServerServices services, final HRegion mergedRegion, User user) throws IOException { openMergedRegion(server, services, mergedRegion); if (useCoordination(server)) { ((BaseCoordinatedStateManager) server.getCoordinatedStateManager()) .getRegionMergeCoordination().completeRegionMergeTransaction(services, mergedRegionInfo, region_a, region_b, rmd, mergedRegion); } if (rsCoprocessorHost != null) { if (user == null) { rsCoprocessorHost.postMerge(region_a, region_b, mergedRegion); } else { try { user.getUGI().doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { rsCoprocessorHost.postMerge(region_a, region_b, mergedRegion); return null; } }); } catch (InterruptedException ie) { InterruptedIOException iioe = new InterruptedIOException(); iioe.initCause(ie); throw iioe; } } } }