Java 类org.apache.hadoop.yarn.api.records.Token 实例源码

@Override
public StartContainersResponse startContainers(
    StartContainersRequest requests) throws YarnException {
  StartContainersResponse response =
      recordFactory.newRecordInstance(StartContainersResponse.class);
  for (StartContainerRequest request : requests.getStartContainerRequests()) {
    Token containerToken = request.getContainerToken();
    ContainerTokenIdentifier tokenId = null;

    try {
      tokenId = newContainerTokenIdentifier(containerToken);
    } catch (IOException e) {
      throw RPCUtil.getRemoteException(e);
    }
    ContainerStatus status =
        recordFactory.newRecordInstance(ContainerStatus.class);
    status.setState(ContainerState.RUNNING);
    status.setContainerId(tokenId.getContainerID());
    status.setExitStatus(0);
    statuses.add(status);

  }
  return response;
}

@SuppressWarnings("deprecation")
private Container newContainerInstance(int id, Priority priority,
    Resource capability, String hostName) throws IOException {
  HPCCommandExecutor.setJobState(id, "assigned", conf);
  NodeId nodeId = NodeId.newInstance(hostName, 0);
  Container container = Records.newRecord(Container.class);
  container.setNodeId(nodeId);
  container.setPriority(priority);
  container.setResource(capability);
  container.setId(ContainerId.newInstance(appAttemptId, ++containerId));
  Token token = Token.newInstance(nodeId.toString().getBytes(),
      nodeId.toString(), nodeId.toString().getBytes(), nodeId.toString());
  byte[] bytes = container.getId().toString().getBytes();
  ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
  buffer.put(bytes);
  token.setIdentifier(buffer);
  container.setContainerToken(token);
  container.setNodeHttpAddress(hostName + ":0");
  LOG.info("Allocated container " + container.getId() + " for Job id:" + id);
  return container;
}

private String testStopContainer(YarnRPC rpc,
    ApplicationAttemptId appAttemptId, NodeId nodeId,
    ContainerId containerId, Token nmToken, boolean isExceptionExpected) {
  try {
    stopContainer(rpc, nmToken,
        Arrays.asList(new ContainerId[] { containerId }), appAttemptId,
        nodeId);
    if (isExceptionExpected) {
      fail("Exception was expected!!");
    }
    return "";
  } catch (Exception e) {
    e.printStackTrace();
    return e.getMessage();
  }
}

private String testStartContainer(YarnRPC rpc,
    ApplicationAttemptId appAttemptId, NodeId nodeId,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    boolean isExceptionExpected) {
  try {
    startContainer(rpc, nmToken, containerToken, nodeId,
        appAttemptId.toString());
    if (isExceptionExpected){
      fail("Exception was expected!!");        
    }
    return "";
  } catch (Exception e) {
    e.printStackTrace();
    return e.getMessage();
  }
}

private void stopContainer(YarnRPC rpc, Token nmToken,
    List<ContainerId> containerId, ApplicationAttemptId appAttemptId,
    NodeId nodeId) throws Exception {
  StopContainersRequest request =
      StopContainersRequest.newInstance(containerId);
  ContainerManagementProtocol proxy = null;
  try {
    proxy =
        getContainerManagementProtocolProxy(rpc, nmToken, nodeId,
            appAttemptId.toString());
    StopContainersResponse response = proxy.stopContainers(request);
    if (response.getFailedRequests() != null &&
        response.getFailedRequests().containsKey(containerId)) {
      parseAndThrowException(response.getFailedRequests().get(containerId)
          .deSerialize());
    }
  } catch (Exception e) {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}

private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}

private Token getDelegationToken(
    final UserGroupInformation loggedInUser,
    final MRClientProtocol hsService, final String renewerString)
    throws IOException, InterruptedException {
  // Get the delegation token directly as it is a little difficult to setup
  // the kerberos based rpc.
  Token token = loggedInUser
      .doAs(new PrivilegedExceptionAction<Token>() {
        @Override
        public Token run() throws IOException {
          GetDelegationTokenRequest request = Records
              .newRecord(GetDelegationTokenRequest.class);
          request.setRenewer(renewerString);
          return hsService.getDelegationToken(request).getDelegationToken();
        }

      });
  return token;
}

public NMToken createAndGetNMToken(String applicationSubmitter,
    ApplicationAttemptId appAttemptId, Container container) {
  try {
    this.readLock.lock();
    HashSet<NodeId> nodeSet = this.appAttemptToNodeKeyMap.get(appAttemptId);
    NMToken nmToken = null;
    if (nodeSet != null) {
      if (!nodeSet.contains(container.getNodeId())) {
        LOG.info("Sending NMToken for nodeId : " + container.getNodeId()
            + " for container : " + container.getId());
        Token token =
            createNMToken(container.getId().getApplicationAttemptId(),
              container.getNodeId(), applicationSubmitter);
        nmToken = NMToken.newInstance(container.getNodeId(), token);
        nodeSet.add(container.getNodeId());
      }
    }
    return nmToken;
  } finally {
    this.readLock.unlock();
  }
}

private Container newContainerInstance(int id, Priority priority,
    Resource capability, String hostName) throws IOException {
  NodeId nodeId = NodeId.newInstance(hostName, 0);
  Container container = Records.newRecord(Container.class);
  container.setNodeId(nodeId);
  container.setPriority(priority);
  container.setResource(capability);
  container.setId(ContainerId.newContainerId(appAttemptId, ++containerId));
  Token token = Token.newInstance(nodeId.toString().getBytes(),
      nodeId.toString(), nodeId.toString().getBytes(), nodeId.toString());
  byte[] bytes = container.getId().toString().getBytes();
  ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
  buffer.put(bytes);
  token.setIdentifier(buffer);
  container.setContainerToken(token);
  container.setNodeHttpAddress(hostName + ":0");
  return container;
}

/**
* Test CancelDelegationTokenRequestPBImpl.
* Test a transformation to prototype and back
*/
@Test
public void testCancelDelegationTokenRequestPBImpl() {

  Token token = getDelegationToken();

  CancelDelegationTokenRequestPBImpl original = new CancelDelegationTokenRequestPBImpl();
  original.setDelegationToken(token);
  CancelDelegationTokenRequestProto protoType = original.getProto();

  CancelDelegationTokenRequestPBImpl copy = new CancelDelegationTokenRequestPBImpl(protoType);
  assertNotNull(copy.getDelegationToken());
  //compare source and converted
  assertEquals(token, copy.getDelegationToken());

}

/**
* Test RenewDelegationTokenRequestPBImpl.
* Test a transformation to prototype and back
*/

@Test
public void testRenewDelegationTokenRequestPBImpl() {

  Token token = getDelegationToken();

  RenewDelegationTokenRequestPBImpl original = new RenewDelegationTokenRequestPBImpl();
  original.setDelegationToken(token);
  RenewDelegationTokenRequestProto protoType = original.getProto();

  RenewDelegationTokenRequestPBImpl copy = new RenewDelegationTokenRequestPBImpl(protoType);
  assertNotNull(copy.getDelegationToken());
  //compare source and converted
  assertEquals(token, copy.getDelegationToken());

}

@Private
@Unstable
public static AllocateResponse newInstance(int responseId,
    List<ContainerStatus> completedContainers,
    List<Container> allocatedContainers, List<NodeReport> updatedNodes,
    Resource availResources, AMCommand command, int numClusterNodes,
    PreemptionMessage preempt, List<NMToken> nmTokens, Token amRMToken,
    List<ContainerResourceIncrease> increasedContainers,
    List<ContainerResourceDecrease> decreasedContainers) {
  AllocateResponse response =
      newInstance(responseId, completedContainers, allocatedContainers,
        updatedNodes, availResources, command, numClusterNodes, preempt,
        nmTokens, increasedContainers, decreasedContainers);
  response.setAMRMToken(amRMToken);
  return response;
}

@Override
public Token getDelegationToken() {
  CancelDelegationTokenRequestProtoOrBuilder p = viaProto ? proto : builder;
  if (this.token != null) {
    return this.token;
  }
  this.token = convertFromProtoFormat(p.getToken());
  return this.token;
}

@Override
public void setDelegationToken(Token token) {
  maybeInitBuilder();
  if (token == null)
    builder.clearToken();
  this.token = token;
}

@Override
public Token getRMDelegationToken() {
  GetDelegationTokenResponseProtoOrBuilder p = viaProto ? proto : builder;
  if (this.appToken != null) {
    return this.appToken;
  }
  if (!p.hasToken()) {
    return null;
  }
  this.appToken = convertFromProtoFormat(p.getToken());
  return this.appToken;  
}

private long renewDelegationToken(final UserGroupInformation loggedInUser,
    final MRClientProtocol hsService, final Token dToken)
    throws IOException, InterruptedException {
  long nextExpTime = loggedInUser.doAs(new PrivilegedExceptionAction<Long>() {

    @Override
    public Long run() throws IOException {
      RenewDelegationTokenRequest request = Records
          .newRecord(RenewDelegationTokenRequest.class);
      request.setDelegationToken(dToken);
      return hsService.renewDelegationToken(request).getNextExpirationTime();
    }
  });
  return nextExpTime;
}

@Override
public void setDelegationToken(Token token) {
  maybeInitBuilder();
  if (token == null) 
    builder.clearToken();
  this.token = token;
}

@Override
public Token getContainerToken() {
  StartContainerRequestProtoOrBuilder p = viaProto ? proto : builder;
  if (this.containerToken != null) {
    return this.containerToken;
  }
  if (!p.hasContainerToken()) {
    return null;
  }
  this.containerToken = convertFromProtoFormat(p.getContainerToken());
  return this.containerToken;
}

@Override
public synchronized Token getAMRMToken() {
  AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
  if (amrmToken != null) {
    return amrmToken;
  }
  if (!p.hasAmRmToken()) {
    return null;
  }
  this.amrmToken = convertFromProtoFormat(p.getAmRmToken());
  return amrmToken;
}

@Override
public synchronized void setAMRMToken(Token amRMToken) {
  maybeInitBuilder();
  if (amRMToken == null) {
    builder.clearAmRmToken();
  }
  this.amrmToken = amRMToken;
}

@Override
public void setContainerToken(Token containerToken) {
  maybeInitBuilder();
  if (containerToken == null) 
    builder.clearContainerToken();
  this.containerToken = containerToken;
}

@Override
public synchronized Token getToken() {
  NMTokenProtoOrBuilder p = viaProto ? proto : builder;
  if (this.token != null) {
    return this.token;
  }
  if (!p.hasToken()) {
    return null;
  }
  this.token = convertFromProtoFormat(p.getToken()); 
  return token;
}

@Override
public synchronized void setToken(Token token) {
  maybeInitBuilder();
  if (token == null) {
    builder.clearToken();
  }
  this.token = token;
}

@Override
public void setDelegationToken(Token token) {
  maybeInitBuilder();
  if (token == null)
    builder.clearToken();
  this.token = token;
}

@Override
public Token getAMRMToken() {
  ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
  if (amRmToken != null) {
    return amRmToken;
  }
  if (!p.hasAmRmToken()) {
    return null;
  }
  amRmToken = convertFromProtoFormat(p.getAmRmToken());
  return amRmToken;
}

@Override
public void setClientToAMToken(Token clientToAMToken) {
  maybeInitBuilder();
  if (clientToAMToken == null) 
    builder.clearClientToAmToken();
  this.clientToAMToken = clientToAMToken;
}

@Override
public void setAMRMToken(Token amRmToken) {
  maybeInitBuilder();
  if (amRmToken == null) {
    builder.clearAmRmToken();
  }
  this.amRmToken = amRmToken;
}

@Override
public Token getContainerToken() {
  ContainerResourceIncreaseProtoOrBuilder p = viaProto ? proto : builder;
  if (this.token != null) {
    return this.token;
  }
  if (p.hasContainerToken()) {
    this.token = convertFromProtoFormat(p.getContainerToken());
  }
  return this.token;
}

public static ContainerTokenIdentifier newContainerTokenIdentifier(
    Token containerToken) throws IOException {
  org.apache.hadoop.security.token.Token<ContainerTokenIdentifier> token =
      new org.apache.hadoop.security.token.Token<ContainerTokenIdentifier>(
          containerToken.getIdentifier()
              .array(), containerToken.getPassword().array(), new Text(
              containerToken.getKind()),
          new Text(containerToken.getService()));
  return token.decodeIdentifier();
}

public static Token newContainerToken(NodeId nodeId, byte[] password,
    ContainerTokenIdentifier tokenIdentifier) {
  // RPC layer client expects ip:port as service for tokens
  InetSocketAddress addr =
      NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort());
  // NOTE: use SecurityUtil.setTokenService if this becomes a "real" token
  Token containerToken =
      Token.newInstance(tokenIdentifier.getBytes(),
        ContainerTokenIdentifier.KIND.toString(), password, SecurityUtil
          .buildTokenService(addr).toString());
  return containerToken;
}

@Override
public Token getDelegationToken() {
  RenewDelegationTokenRequestProtoOrBuilder p = viaProto ? proto : builder;
  if (this.token != null) {
    return this.token;
  }
  this.token = convertFromProtoFormat(p.getToken());
  return this.token;
}

public static Token newInstance(byte[] password,
    NMTokenIdentifier identifier) {
  NodeId nodeId = identifier.getNodeId();
  // RPC layer client expects ip:port as service for tokens
  InetSocketAddress addr =
      NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort());
  Token nmToken =
      Token.newInstance(identifier.getBytes(),
        NMTokenIdentifier.KIND.toString(), password, SecurityUtil
          .buildTokenService(addr).toString());
  return nmToken;
}

public static Token newContainerToken(ContainerId cId, String host,
    int port, String user, Resource r, long expiryTime, int masterKeyId,
    byte[] password, long rmIdentifier) throws IOException {
  ContainerTokenIdentifier identifier =
      new ContainerTokenIdentifier(cId, host + ":" + port, user, r,
        expiryTime, masterKeyId, rmIdentifier, Priority.newInstance(0), 0);
  return newContainerToken(BuilderUtils.newNodeId(host, port), password,
      identifier);
}

public static Container newContainer(ContainerId containerId, NodeId nodeId,
    String nodeHttpAddress, Resource resource, Priority priority,
    Token containerToken) {
  Container container = recordFactory.newRecordInstance(Container.class);
  container.setId(containerId);
  container.setNodeId(nodeId);
  container.setNodeHttpAddress(nodeHttpAddress);
  container.setResource(resource);
  container.setPriority(priority);
  container.setContainerToken(containerToken);
  return container;
}

public static Token newContainerToken(NodeId nodeId,
    byte[] password, ContainerTokenIdentifier tokenIdentifier) {
  // RPC layer client expects ip:port as service for tokens
  InetSocketAddress addr =
      NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort());
  // NOTE: use SecurityUtil.setTokenService if this becomes a "real" token
  Token containerToken =
      Token.newInstance(tokenIdentifier.getBytes(),
        ContainerTokenIdentifier.KIND.toString(), password, SecurityUtil
          .buildTokenService(addr).toString());
  return containerToken;
}

private void updateAMRMToken(Token token) throws IOException {
  org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken =
      new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(token
        .getIdentifier().array(), token.getPassword().array(), new Text(
        token.getKind()), new Text(token.getService()));
  UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser();
  currentUGI.addToken(amrmToken);
  amrmToken.setService(ClientRMProxy.getAMRMTokenService(getConfig()));
}

@VisibleForTesting
public static Token newContainerToken(NodeId nodeId,
    byte[] password, ContainerTokenIdentifier tokenIdentifier) {
  // RPC layer client expects ip:port as service for tokens
  InetSocketAddress addr =
      NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort());
  // NOTE: use SecurityUtil.setTokenService if this becomes a "real" token
  Token containerToken =
      newToken(Token.class, tokenIdentifier.getBytes(),
        ContainerTokenIdentifier.KIND.toString(), password, SecurityUtil
          .buildTokenService(addr).toString());
  return containerToken;
}

public static ContainerTokenIdentifier newContainerTokenIdentifier(
    Token containerToken) throws IOException {
  org.apache.hadoop.security.token.Token<ContainerTokenIdentifier> token =
      new org.apache.hadoop.security.token.Token<ContainerTokenIdentifier>(
          containerToken.getIdentifier()
              .array(), containerToken.getPassword().array(), new Text(
              containerToken.getKind()),
          new Text(containerToken.getService()));
  return token.decodeIdentifier();
}

public static ApplicationReport newApplicationReport(
    ApplicationId applicationId, ApplicationAttemptId applicationAttemptId,
    String user, String queue, String name, String host, int rpcPort,
    Token clientToAMToken, YarnApplicationState state, String diagnostics,
    String url, long startTime, long finishTime,
    FinalApplicationStatus finalStatus,
    ApplicationResourceUsageReport appResources, String origTrackingUrl,
    float progress, String appType, Token amRmToken, Set<String> tags) {
  ApplicationReport report = recordFactory
      .newRecordInstance(ApplicationReport.class);
  report.setApplicationId(applicationId);
  report.setCurrentApplicationAttemptId(applicationAttemptId);
  report.setUser(user);
  report.setQueue(queue);
  report.setName(name);
  report.setHost(host);
  report.setRpcPort(rpcPort);
  report.setClientToAMToken(clientToAMToken);
  report.setYarnApplicationState(state);
  report.setDiagnostics(diagnostics);
  report.setTrackingUrl(url);
  report.setStartTime(startTime);
  report.setFinishTime(finishTime);
  report.setFinalApplicationStatus(finalStatus);
  report.setApplicationResourceUsageReport(appResources);
  report.setOriginalTrackingUrl(origTrackingUrl);
  report.setProgress(progress);
  report.setApplicationType(appType);
  report.setAMRMToken(amRmToken);
  report.setApplicationTags(tags);
  return report;
}

private void
    getContainerStatus(YarnRPC rpc,
        org.apache.hadoop.yarn.api.records.Token nmToken,
        ContainerId containerId,
        ApplicationAttemptId appAttemptId, NodeId nodeId,
        boolean isExceptionExpected) throws Exception {
  List<ContainerId> containerIds = new ArrayList<ContainerId>();
  containerIds.add(containerId);
  GetContainerStatusesRequest request =
      GetContainerStatusesRequest.newInstance(containerIds);
  ContainerManagementProtocol proxy = null;
  try {
    proxy =
        getContainerManagementProtocolProxy(rpc, nmToken, nodeId,
            appAttemptId.toString());
    GetContainerStatusesResponse statuses = proxy.getContainerStatuses(request);
    if (statuses.getFailedRequests() != null
        && statuses.getFailedRequests().containsKey(containerId)) {
      parseAndThrowException(statuses.getFailedRequests().get(containerId)
        .deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}