@Test public void testCachePreventsImplRequest() throws Exception { // Disable negative cache. conf.setLong( CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_NEGATIVE_CACHE_SECS, 0); Groups groups = new Groups(conf); groups.cacheGroupsAdd(Arrays.asList(myGroups)); groups.refresh(); FakeGroupMapping.clearBlackList(); assertEquals(0, FakeGroupMapping.getRequestCount()); // First call hits the wire assertTrue(groups.getGroups("me").size() == 2); assertEquals(1, FakeGroupMapping.getRequestCount()); // Second count hits cache assertTrue(groups.getGroups("me").size() == 2); assertEquals(1, FakeGroupMapping.getRequestCount()); }
@Test public void testCacheEntriesExpire() throws Exception { conf.setLong( CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_SECS, 1); FakeTimer timer = new FakeTimer(); final Groups groups = new Groups(conf, timer); groups.cacheGroupsAdd(Arrays.asList(myGroups)); groups.refresh(); FakeGroupMapping.clearBlackList(); // We make an entry groups.getGroups("me"); int startingRequestCount = FakeGroupMapping.getRequestCount(); timer.advance(20 * 1000); // Cache entry has expired so it results in a new fetch groups.getGroups("me"); assertEquals(startingRequestCount + 1, FakeGroupMapping.getRequestCount()); }
/** * Validate the netgroups, both group membership and ACL * functionality * * Note: assumes a specific acl setup done by testNetgroups * * @param groups group to user mapping service * @param acl ACL set up in a specific way, see testNetgroups */ private void validateNetgroups(Groups groups, AccessControlList acl) throws Exception { // check that the netgroups are working List<String> elvisGroups = groups.getGroups("elvis"); assertTrue(elvisGroups.contains("@lasVegas")); assertTrue(elvisGroups.contains("@memphis")); List<String> jerryLeeLewisGroups = groups.getGroups("jerryLeeLewis"); assertTrue(jerryLeeLewisGroups.contains("@memphis")); // allowed because his netgroup is in ACL UserGroupInformation elvis = UserGroupInformation.createRemoteUser("elvis"); assertUserAllowed(elvis, acl); // allowed because he's in ACL UserGroupInformation carlPerkins = UserGroupInformation.createRemoteUser("carlPerkins"); assertUserAllowed(carlPerkins, acl); // not allowed because he's not in ACL and has no netgroups UserGroupInformation littleRichard = UserGroupInformation.createRemoteUser("littleRichard"); assertUserNotAllowed(littleRichard, acl); }
public QueuePlacementPolicy(List<QueuePlacementRule> rules, Map<FSQueueType, Set<String>> configuredQueues, Configuration conf) throws AllocationConfigurationException { for (int i = 0; i < rules.size()-1; i++) { if (rules.get(i).isTerminal()) { throw new AllocationConfigurationException("Rules after rule " + i + " in queue placement policy can never be reached"); } } if (!rules.get(rules.size()-1).isTerminal()) { throw new AllocationConfigurationException( "Could get past last queue placement rule without assigning"); } this.rules = rules; this.configuredQueues = configuredQueues; groups = new Groups(conf); }
@Override protected String getQueueForApp(String requestedQueue, String user, Groups groups, Map<FSQueueType, Set<String>> configuredQueues) throws IOException { List<String> groupNames = groups.getGroups(user); for (int i = 1; i < groupNames.size(); i++) { String group = cleanName(groupNames.get(i)); if (configuredQueues.get(FSQueueType.LEAF).contains("root." + group) || configuredQueues.get(FSQueueType.PARENT).contains( "root." + group)) { return "root." + group; } } return ""; }
@Override protected String getQueueForApp(String requestedQueue, String user, Groups groups, Map<FSQueueType, Set<String>> configuredQueues) throws IOException { // Apply the nested rule String queueName = nestedRule.assignAppToQueue(requestedQueue, user, groups, configuredQueues); if (queueName != null && queueName.length() != 0) { if (!queueName.startsWith("root.")) { queueName = "root." + queueName; } // Verify if the queue returned by the nested rule is an configured leaf queue, // if yes then skip to next rule in the queue placement policy if (configuredQueues.get(FSQueueType.LEAF).contains(queueName)) { return ""; } return queueName + "." + cleanName(user); } return queueName; }
@Override public RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings( RefreshUserToGroupsMappingsRequest request) throws YarnException, IOException { String argName = "refreshUserToGroupsMappings"; UserGroupInformation user = checkAcls(argName); checkRMStatus(user.getShortUserName(), argName, "refresh user-groups."); Groups.getUserToGroupsMappingService( getConfiguration(new Configuration(false), YarnConfiguration.CORE_SITE_CONFIGURATION_FILE)).refresh(); RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService"); return recordFactory.newRecordInstance( RefreshUserToGroupsMappingsResponse.class); }
@Before public void setup() throws IOException { this.conf = new JobConf(); this.conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, NullGroupsProvider.class.getName()); this.conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true); Groups.getUserToGroupsMappingService(conf); this.ctx = buildHistoryContext(this.conf); WebApp webApp = mock(HsWebApp.class); when(webApp.name()).thenReturn("hsmockwebapp"); this.hsWebServices= new HsWebServices(ctx, conf, webApp); this.hsWebServices.setResponse(mock(HttpServletResponse.class)); Job job = ctx.getAllJobs().values().iterator().next(); this.jobIdStr = job.getID().toString(); Task task = job.getTasks().values().iterator().next(); this.taskIdStr = task.getID().toString(); this.taskAttemptIdStr = task.getAttempts().keySet().iterator().next().toString(); }
/** * Validate the netgroups, both group membership and ACL * functionality * * Note: assumes a specific acl setup done by testNetgroups * * @param groups group to user mapping service * @param acl ACL set up in a specific way, see testNetgroups */ private void validateNetgroups(Groups groups, AccessControlList acl) throws Exception { // check that the netgroups are working List<String> elvisGroups = groups.getGroups("elvis"); assertTrue(elvisGroups.contains("@lasVegas")); assertTrue(elvisGroups.contains("@memphis")); List<String> jerryLeeLewisGroups = groups.getGroups("jerryLeeLewis"); assertTrue(jerryLeeLewisGroups.contains("@memphis")); // allowed becuase his netgroup is in ACL UserGroupInformation elvis = UserGroupInformation.createRemoteUser("elvis"); assertUserAllowed(elvis, acl); // allowed because he's in ACL UserGroupInformation carlPerkins = UserGroupInformation.createRemoteUser("carlPerkins"); assertUserAllowed(carlPerkins, acl); // not allowed because he's not in ACL and has no netgroups UserGroupInformation littleRichard = UserGroupInformation.createRemoteUser("littleRichard"); assertUserNotAllowed(littleRichard, acl); }
@Override public RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings( RefreshUserToGroupsMappingsRequest request) throws YarnException, IOException { String argName = "refreshUserToGroupsMappings"; UserGroupInformation user = checkAcls(argName); if (!isRMActive()) { RMAuditLogger.logFailure(user.getShortUserName(), argName, adminAcl.toString(), "AdminService", "ResourceManager is not active. Can not refresh user-groups."); throwStandbyException(); } Groups.getUserToGroupsMappingService( getConfiguration(new Configuration(false), YarnConfiguration.CORE_SITE_CONFIGURATION_FILE)).refresh(); RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService"); return recordFactory.newRecordInstance( RefreshUserToGroupsMappingsResponse.class); }
static void setupGroupsProvider() throws IOException { Configuration conf = new Configuration(); conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, MyGroupsProvider.class.getName()); Groups.getUserToGroupsMappingService(conf); MyGroupsProvider.mapping.put(jobSubmitter, Arrays.asList("group1")); MyGroupsProvider.mapping.put(viewColleague, Arrays.asList("group2")); MyGroupsProvider.mapping.put(modifyColleague, Arrays.asList("group1")); MyGroupsProvider.mapping.put(unauthorizedUser, Arrays.asList("evilSociety")); MyGroupsProvider.mapping.put(mrAdminGroupMember, Arrays.asList(mrAdminGroup)); MyGroupsProvider.mapping.put(viewAndModifyColleague, Arrays.asList("group3")); MyGroupsProvider.mapping.put(qAdmin, Arrays.asList("group4")); mrOwner = UserGroupInformation.getCurrentUser().getShortUserName(); MyGroupsProvider.mapping.put(mrOwner, Arrays.asList( new String[] { "group5", "group6" })); MyGroupsProvider.mapping.put(jobSubmitter1, Arrays.asList("group7")); MyGroupsProvider.mapping.put(jobSubmitter2, Arrays.asList("group7")); MyGroupsProvider.mapping.put(jobSubmitter3, Arrays.asList("group7")); MyGroupsProvider.mapping.put(mrAdminUser, Arrays.asList("group8")); }
public PoolPlacementPolicy(List<PoolPlacementRule> rules, Set<String> configuredPools, Configuration conf) throws AllocationConfigurationException { for (int i = 0; i < rules.size()-1; i++) { if (rules.get(i).isTerminal()) { throw new AllocationConfigurationException("Rules after rule " + i + " in pool placement policy can never be reached"); } } if (!rules.get(rules.size()-1).isTerminal()) { throw new AllocationConfigurationException( "Could get past last pool placement rule without assigning"); } this.rules = rules; this.configuredPools = configuredPools; groups = new Groups(conf); }
@Before public void init() throws HadoopIllegalArgumentException, IOException { conf = new JobConf(); conf.set(JHAdminConfig.JHS_ADMIN_ADDRESS, "0.0.0.0:0"); conf.setClass("hadoop.security.group.mapping", MockUnixGroupsMapping.class, GroupMappingServiceProvider.class); conf.setLong("hadoop.security.groups.cache.secs", groupRefreshTimeoutSec); Groups.getUserToGroupsMappingService(conf); jobHistoryService = mock(JobHistory.class); alds = mock(AggregatedLogDeletionService.class); hsAdminServer = new HSAdminServer(alds, jobHistoryService) { @Override protected Configuration createConf() { return conf; } }; hsAdminServer.init(conf); hsAdminServer.start(); conf.setSocketAddr(JHAdminConfig.JHS_ADMIN_ADDRESS, hsAdminServer.clientRpcServer.getListenerAddress()); hsAdminClient = new HSAdmin(conf); }
