/** * Print the assignment plan to the system output stream * @param plan */ public static void printAssignmentPlan(FavoredNodesPlan plan) { if (plan == null) return; LOG.info("========== Start to print the assignment plan ================"); // sort the map based on region info Map<HRegionInfo, List<ServerName>> assignmentMap = new TreeMap<HRegionInfo, List<ServerName>>(plan.getAssignmentMap()); for (Map.Entry<HRegionInfo, List<ServerName>> entry : assignmentMap.entrySet()) { String serverList = FavoredNodeAssignmentHelper.getFavoredNodesAsString(entry.getValue()); String regionName = entry.getKey().getRegionNameAsString(); LOG.info("Region: " + regionName ); LOG.info("Its favored nodes: " + serverList); } LOG.info("========== Finish to print the assignment plan ================"); }
public void printDispersionScores(TableName table, SnapshotOfRegionAssignmentFromMeta snapshot, int numRegions, FavoredNodesPlan newPlan, boolean simplePrint) { if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) { return; } AssignmentVerificationReport report = new AssignmentVerificationReport(); report.fillUpDispersion(table, snapshot, newPlan); List<Float> dispersion = report.getDispersionInformation(); if (simplePrint) { DecimalFormat df = new java.text.DecimalFormat("#.##"); System.out.println("\tAvg dispersion score: " + df.format(dispersion.get(0)) + " hosts;\tMax dispersion score: " + df.format(dispersion.get(1)) + " hosts;\tMin dispersion score: " + df.format(dispersion.get(2)) + " hosts;"); } else { LOG.info("For Table: " + table + " ; #Total Regions: " + numRegions + " ; The average dispersion score is " + dispersion.get(0)); } }
/** * Shuffle the assignment plan by switching two favored node positions. * @param plan The assignment plan * @param p1 The first switch position * @param p2 The second switch position * @return */ private FavoredNodesPlan shuffleAssignmentPlan(FavoredNodesPlan plan, FavoredNodesPlan.Position p1, FavoredNodesPlan.Position p2) { FavoredNodesPlan shuffledPlan = new FavoredNodesPlan(); for (Map.Entry<HRegionInfo, List<ServerName>> entry : plan.getAssignmentMap().entrySet()) { HRegionInfo region = entry.getKey(); // copy the server list from the original plan List<ServerName> shuffledServerList = new ArrayList<ServerName>(); shuffledServerList.addAll(entry.getValue()); // start to shuffle shuffledServerList.set(p1.ordinal(), entry.getValue().get(p2.ordinal())); shuffledServerList.set(p2.ordinal(), entry.getValue().get(p1.ordinal())); // update the plan shuffledPlan.updateAssignmentPlan(region, shuffledServerList); } return shuffledPlan; }
/** * To verify the region assignment status. * It will check the assignment plan consistency between hbase:meta and * region servers. * Also it will verify weather the number of region movement and * the number regions on the primary region server are expected * * @param plan * @param regionMovementNum * @param numRegionsOnPrimaryRS * @throws InterruptedException * @throws IOException */ private void verifyRegionAssignment(FavoredNodesPlan plan, int regionMovementNum, int numRegionsOnPrimaryRS) throws InterruptedException, IOException { // Verify the assignment plan in hbase:meta is consistent with the expected plan. verifyMETAUpdated(plan); // Verify the number of region movement is expected verifyRegionMovementNum(regionMovementNum); // Verify the number of regions is assigned to the primary region server // based on the plan is expected verifyRegionOnPrimaryRS(numRegionsOnPrimaryRS); // Verify all the online region server are updated with the assignment plan verifyRegionServerUpdated(plan); }
public SnapshotOfRegionAssignmentFromMeta(Connection connection, Set<TableName> disabledTables, boolean excludeOfflinedSplitParents) { this.connection = connection; tableToRegionMap = new HashMap<TableName, List<HRegionInfo>>(); regionToRegionServerMap = new HashMap<HRegionInfo, ServerName>(); regionServerToRegionMap = new HashMap<ServerName, List<HRegionInfo>>(); regionNameToRegionInfoMap = new TreeMap<String, HRegionInfo>(); existingAssignmentPlan = new FavoredNodesPlan(); this.disabledTables = disabledTables; this.excludeOfflinedSplitParents = excludeOfflinedSplitParents; }
public FavoredNodesPlan getNewAssignmentPlan() throws IOException { // Get the current region assignment snapshot by scanning from the META SnapshotOfRegionAssignmentFromMeta assignmentSnapshot = this.getRegionAssignmentSnapshot(); // Get the region locality map Map<String, Map<String, Float>> regionLocalityMap = null; if (this.enforceLocality) { regionLocalityMap = FSUtils.getRegionDegreeLocalityMappingFromFS(conf); } // Initialize the assignment plan FavoredNodesPlan plan = new FavoredNodesPlan(); // Get the table to region mapping Map<TableName, List<HRegionInfo>> tableToRegionMap = assignmentSnapshot.getTableToRegionMap(); LOG.info("Start to generate the new assignment plan for the " + + tableToRegionMap.keySet().size() + " tables" ); for (TableName table : tableToRegionMap.keySet()) { try { if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) { continue; } // TODO: maybe run the placement in parallel for each table genAssignmentPlan(table, assignmentSnapshot, regionLocalityMap, plan, USE_MUNKRES_FOR_PLACING_SECONDARY_AND_TERTIARY); } catch (Exception e) { LOG.error("Get some exceptions for placing primary region server" + "for table " + table + " because " + e); } } LOG.info("Finish to generate the new assignment plan for the " + + tableToRegionMap.keySet().size() + " tables" ); return plan; }
/** * Update the assignment plan into hbase:meta * @param plan the assignments plan to be updated into hbase:meta * @throws IOException if cannot update assignment plan in hbase:meta */ public void updateAssignmentPlanToMeta(FavoredNodesPlan plan) throws IOException { try { LOG.info("Start to update the hbase:meta with the new assignment plan"); Map<HRegionInfo, List<ServerName>> assignmentMap = plan.getAssignmentMap(); FavoredNodeAssignmentHelper.updateMetaWithFavoredNodesInfo(assignmentMap, conf); LOG.info("Updated the hbase:meta with the new assignment plan"); } catch (Exception e) { LOG.error("Failed to update hbase:meta with the new assignment" + "plan because " + e.getMessage()); } }
public void updateAssignmentPlan(FavoredNodesPlan plan) throws IOException { LOG.info("Start to update the new assignment plan for the hbase:meta table and" + " the region servers"); // Update the new assignment plan to META updateAssignmentPlanToMeta(plan); // Update the new assignment plan to Region Servers updateAssignmentPlanToRegionServers(plan); LOG.info("Finish to update the new assignment plan for the hbase:meta table and" + " the region servers"); }
/** * Return how many regions will move per table since their primary RS will * change * * @param newPlan - new AssignmentPlan * @return how many primaries will move per table */ public Map<TableName, Integer> getRegionsMovement(FavoredNodesPlan newPlan) throws IOException { Map<TableName, Integer> movesPerTable = new HashMap<TableName, Integer>(); SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot(); Map<TableName, List<HRegionInfo>> tableToRegions = snapshot .getTableToRegionMap(); FavoredNodesPlan oldPlan = snapshot.getExistingAssignmentPlan(); Set<TableName> tables = snapshot.getTableSet(); for (TableName table : tables) { int movedPrimaries = 0; if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) { continue; } List<HRegionInfo> regions = tableToRegions.get(table); for (HRegionInfo region : regions) { List<ServerName> oldServers = oldPlan.getFavoredNodes(region); List<ServerName> newServers = newPlan.getFavoredNodes(region); if (oldServers != null && newServers != null) { ServerName oldPrimary = oldServers.get(0); ServerName newPrimary = newServers.get(0); if (oldPrimary.compareTo(newPrimary) != 0) { movedPrimaries++; } } } movesPerTable.put(table, movedPrimaries); } return movesPerTable; }
/** * Verify the meta has updated to the latest assignment plan * @param plan * @throws IOException */ private void verifyMETAUpdated(FavoredNodesPlan expectedPlan) throws IOException { FavoredNodesPlan planFromMETA = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan(); assertTrue("The assignment plan is NOT consistent with the expected plan ", planFromMETA.equals(expectedPlan)); }
public SnapshotOfRegionAssignmentFromMeta(CatalogTracker tracker, Set<TableName> disabledTables, boolean excludeOfflinedSplitParents) { this.tracker = tracker; tableToRegionMap = new HashMap<TableName, List<HRegionInfo>>(); regionToRegionServerMap = new HashMap<HRegionInfo, ServerName>(); regionServerToRegionMap = new HashMap<ServerName, List<HRegionInfo>>(); regionNameToRegionInfoMap = new TreeMap<String, HRegionInfo>(); existingAssignmentPlan = new FavoredNodesPlan(); this.disabledTables = disabledTables; this.excludeOfflinedSplitParents = excludeOfflinedSplitParents; }
