private void init(final UserGroupInformation ugi, final DelegationParam delegation, final UserParam username, final DoAsParam doAsUser, final UriFsPathParam path, final HttpOpParam<?> op, final Param<?, ?>... parameters) { if (LOG.isTraceEnabled()) { LOG.trace("HTTP " + op.getValue().getType() + ": " + op + ", " + path + ", ugi=" + ugi + ", " + username + ", " + doAsUser + Param.toSortedString(", ", parameters)); } //clear content type response.setContentType(null); // set the remote address, if coming in via a trust proxy server then // the address with be that of the proxied client REMOTE_ADDRESS.set(JspHelper.getRemoteAddr(request)); }
void start() throws IOException { final InetSocketAddress httpAddr = getAddress(conf); final String httpsAddrString = conf.get( DFSConfigKeys.DFS_JOURNALNODE_HTTPS_ADDRESS_KEY, DFSConfigKeys.DFS_JOURNALNODE_HTTPS_ADDRESS_DEFAULT); InetSocketAddress httpsAddr = NetUtils.createSocketAddr(httpsAddrString); HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf, httpAddr, httpsAddr, "journal", DFSConfigKeys.DFS_JOURNALNODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY, DFSConfigKeys.DFS_JOURNALNODE_KEYTAB_FILE_KEY); httpServer = builder.build(); httpServer.setAttribute(JN_ATTRIBUTE_KEY, localJournalNode); httpServer.setAttribute(JspHelper.CURRENT_CONF, conf); httpServer.addInternalServlet("getJournal", "/getJournal", GetJournalEditServlet.class, true); httpServer.start(); }
private void generateNodeDataHeader(JspWriter out, DatanodeDescriptor d, String suffix, boolean alive, int nnInfoPort, String nnaddr, String scheme) throws IOException { // from nn_browsedfscontent.jsp: String url = "///" + JspHelper.Url.authority(scheme, d) + "/browseDirectory.jsp?namenodeInfoPort=" + nnInfoPort + "&dir=" + URLEncoder.encode("/", "UTF-8") + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnaddr); String name = d.getXferAddrWithHostname(); if (!name.matches("\\d+\\.\\d+.\\d+\\.\\d+.*")) name = name.replaceAll("\\.[^.:]*", ""); int idx = (suffix != null && name.endsWith(suffix)) ? name .indexOf(suffix) : -1; out.print(rowTxt() + "<td class=\"name\"> <a title=\"" + url + "\" href=\"" + url + "\">" + ((idx > 0) ? name.substring(0, idx) : name) + "</a>" + ((alive) ? "" : "\n") + "<td class=\"address\">" + d.getXferAddr()); }
@Test(timeout = 15000) public void testNamenodeJspHelperRedirectToRandomDataNode() throws IOException, InterruptedException { final String urlPart = "browseDirectory.jsp?namenodeInfoPort="; ServletContext context = mock(ServletContext.class); HttpServletRequest request = mock(HttpServletRequest.class); HttpServletResponse resp = mock(HttpServletResponse.class); when(request.getScheme()).thenReturn("http"); when(request.getParameter(UserParam.NAME)).thenReturn("localuser"); when(context.getAttribute(NAMENODE_ATTRIBUTE_KEY)).thenReturn( cluster.getNameNode()); when(context.getAttribute(JspHelper.CURRENT_CONF)).thenReturn(conf); ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class); doAnswer(new Answer<String>() { @Override public String answer(InvocationOnMock invocation) throws Throwable { return null; } }).when(resp).sendRedirect(captor.capture()); NamenodeJspHelper.redirectToRandomDataNode(context, request, resp); assertTrue(captor.getValue().contains(urlPart)); }
/** Create a redirection URL */ private URL createRedirectURL(UserGroupInformation ugi, DatanodeID host, HttpServletRequest request, NameNode nn) throws IOException { final String hostname = host instanceof DatanodeInfo ? ((DatanodeInfo)host).getHostName() : host.getIpAddr(); final String scheme = request.getScheme(); final int port = "https".equals(scheme) ? (Integer)getServletContext().getAttribute(DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY) : host.getInfoPort(); final String encodedPath = ServletUtil.getRawPath(request, "/fileChecksum"); String dtParam = ""; if (UserGroupInformation.isSecurityEnabled()) { String tokenString = ugi.getTokens().iterator().next().encodeToUrlString(); dtParam = JspHelper.getDelegationTokenUrlParam(tokenString); } String addr = nn.getNameNodeAddressHostPortString(); String addrParam = JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, addr); return new URL(scheme, hostname, port, "/getFileChecksum" + encodedPath + '?' + "ugi=" + ServletUtil.encodeQueryValue(ugi.getShortUserName()) + dtParam + addrParam); }
private void generateNodeDataHeader(JspWriter out, DatanodeDescriptor d, String suffix, boolean alive, int nnHttpPort, String nnaddr) throws IOException { // from nn_browsedfscontent.jsp: String url = HttpConfig.getSchemePrefix() + d.getHostName() + ":" + d.getInfoPort() + "/browseDirectory.jsp?namenodeInfoPort=" + nnHttpPort + "&dir=" + URLEncoder.encode("/", "UTF-8") + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnaddr); String name = d.getXferAddrWithHostname(); if (!name.matches("\\d+\\.\\d+.\\d+\\.\\d+.*")) name = name.replaceAll("\\.[^.:]*", ""); int idx = (suffix != null && name.endsWith(suffix)) ? name .indexOf(suffix) : -1; out.print(rowTxt() + "<td class=\"name\"><a title=\"" + d.getXferAddr() + "\" href=\"" + url + "\">" + ((idx > 0) ? name.substring(0, idx) : name) + "</a>" + ((alive) ? "" : "\n")); }
private void generateNodeDataHeader(JspWriter out, DatanodeDescriptor d, String suffix, boolean alive, int nnHttpPort, String nnaddr) throws IOException { // from nn_browsedfscontent.jsp: String url = HttpConfig2.getSchemePrefix() + d.getHostName() + ":" + d.getInfoPort() + "/browseDirectory.jsp?namenodeInfoPort=" + nnHttpPort + "&dir=" + URLEncoder.encode("/", "UTF-8") + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnaddr); String name = d.getXferAddrWithHostname(); if (!name.matches("\\d+\\.\\d+.\\d+\\.\\d+.*")) { name = name.replaceAll("\\.[^.:]*", ""); } int idx = (suffix != null && name.endsWith(suffix)) ? name.indexOf(suffix) : -1; out.print(rowTxt() + "<td class=\"name\"><a title=\"" + d.getXferAddr() + "\" href=\"" + url + "\">" + ((idx > 0) ? name.substring(0, idx) : name) + "</a>" + ((alive) ? "" : "\n")); }
private void generateNodeDataHeader(JspWriter out, DatanodeDescriptor d, String suffix, boolean alive, int nnHttpPort, String nnaddr) throws IOException { // from nn_browsedfscontent.jsp: String url = "///" + d.getHostName() + ":" + d.getInfoPort() + "/browseDirectory.jsp?namenodeInfoPort=" + nnHttpPort + "&dir=" + URLEncoder.encode("/", "UTF-8") + JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, nnaddr); String name = d.getXferAddrWithHostname(); if (!name.matches("\\d+\\.\\d+.\\d+\\.\\d+.*")) name = name.replaceAll("\\.[^.:]*", ""); int idx = (suffix != null && name.endsWith(suffix)) ? name .indexOf(suffix) : -1; out.print(rowTxt() + "<td class=\"name\"> <a title=\"" + url + "\" href=\"" + url + "\">" + ((idx > 0) ? name.substring(0, idx) : name) + "</a>" + ((alive) ? "" : "\n") + "<td class=\"address\">" + d.getXferAddr()); }
void start() throws IOException { final InetSocketAddress httpAddr = getAddress(conf); final String httpsAddrString = conf.get( DFSConfigKeys.DFS_JOURNALNODE_HTTPS_ADDRESS_KEY, DFSConfigKeys.DFS_JOURNALNODE_HTTPS_ADDRESS_DEFAULT); InetSocketAddress httpsAddr = NetUtils.createSocketAddr(httpsAddrString); HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf, httpAddr, httpsAddr, "journal", DFSConfigKeys.DFS_JOURNALNODE_INTERNAL_SPNEGO_USER_NAME_KEY, DFSConfigKeys.DFS_JOURNALNODE_KEYTAB_FILE_KEY); httpServer = builder.build(); httpServer.setAttribute(JN_ATTRIBUTE_KEY, localJournalNode); httpServer.setAttribute(JspHelper.CURRENT_CONF, conf); httpServer.addInternalServlet("getJournal", "/getJournal", GetJournalEditServlet.class, true); httpServer.start(); }
/** * Service a GET request as described below. * Request: * GET http://<host>:<port>/monitor?class=...&key=...&... HTTP/1.1 * */ public void doGet(final HttpServletRequest request, final HttpServletResponse response) throws IOException { final ServletContext context = getServletContext(); final Configuration conf = (Configuration) context.getAttribute(JspHelper.CURRENT_CONF); final UserGroupInformation ugi = getUGI(request, conf); System.out.println("[" + new Date().toString() + "]" + request.getRequestURL() + (request.getQueryString() == null ? "" : ("?"+request.getQueryString()))); try{ ugi.doAs(new PrivilegedExceptionAction<Void>(){ public Void run() throws Exception { doAll(request, response); return null; } }); }catch(InterruptedException e){ throw new IOException(e); } }
/** getting a client for connecting to dfs */ protected DFSClient getDFSClient(HttpServletRequest request) throws IOException, InterruptedException { final Configuration conf = (Configuration) getServletContext().getAttribute(JspHelper.CURRENT_CONF); UserGroupInformation ugi = getUGI(request, conf); DFSClient client = ugi.doAs(new PrivilegedExceptionAction<DFSClient>() { @Override public DFSClient run() throws IOException { return new DFSClient(nameNodeAddr, conf); } }); return client; }
/** Create a redirection URI */ protected URI createUri(String parent, HdfsFileStatus i, UserGroupInformation ugi, ClientProtocol nnproxy, HttpServletRequest request, String dt) throws IOException, URISyntaxException { String scheme = request.getScheme(); final DatanodeID host = pickSrcDatanode(parent, i, nnproxy); final String hostname; if (host instanceof DatanodeInfo) { hostname = ((DatanodeInfo)host).getHostName(); } else { hostname = host.getHost(); } String dtParam=""; if (dt != null) { dtParam=JspHelper.getDelegationTokenUrlParam(dt); } return new URI(scheme, null, hostname, "https".equals(scheme) ? (Integer)getServletContext().getAttribute("datanode.https.port") : host.getInfoPort(), "/streamFile" + i.getFullName(parent), "ugi=" + ugi.getShortUserName() + dtParam, null); }
private void initialize(Configuration conf) throws IOException { sslAddr = getSslAddr(conf); String nn = conf.get("hdfsproxy.dfs.namenode.address"); if (nn == null) throw new IOException("HDFS NameNode address is not specified"); InetSocketAddress nnAddr = NetUtils.createSocketAddr(nn); LOG.info("HDFS NameNode is at: " + nnAddr.getHostName() + ":" + nnAddr.getPort()); Configuration sslConf = new HdfsConfiguration(false); sslConf.addResource(conf.get("hdfsproxy.https.server.keystore.resource", "ssl-server.xml")); // unit testing sslConf.set("proxy.http.test.listener.addr", conf.get("proxy.http.test.listener.addr")); this.server = new ProxyHttpServer(sslAddr, sslConf); this.server.setAttribute("proxy.https.port", server.getPort()); this.server.setAttribute("name.node.address", nnAddr); this.server.setAttribute(JspHelper.CURRENT_CONF, new HdfsConfiguration()); this.server.addGlobalFilter("ProxyFilter", ProxyFilter.class.getName(), null); this.server.addServlet("listPaths", "/listPaths/*", ProxyListPathsServlet.class); this.server.addServlet("data", "/data/*", ProxyFileDataServlet.class); this.server.addServlet("streamFile", "/streamFile/*", ProxyStreamFile.class); }
@Override public UserGroupInformation getValue(final HttpContext context) { final Configuration conf = (Configuration) servletcontext .getAttribute(JspHelper.CURRENT_CONF); try { return JspHelper.getUGI(servletcontext, request, conf, AuthenticationMethod.KERBEROS, false); } catch (IOException e) { throw new SecurityException( SecurityUtil.FAILED_TO_GET_UGI_MSG_HEADER + " " + e, e); } }
protected String addDelegationTokenParam(String query) throws IOException { String tokenString = null; if (UserGroupInformation.isSecurityEnabled()) { synchronized (this) { tokenAspect.ensureTokenInitialized(); if (delegationToken != null) { tokenString = delegationToken.encodeToUrlString(); return (query + JspHelper.getDelegationTokenUrlParam(tokenString)); } } } return query; }
/** Create a redirection URL */ private URL createRedirectURL(UserGroupInformation ugi, DatanodeID host, HttpServletRequest request, NameNode nn) throws IOException { final String hostname = host instanceof DatanodeInfo ? host.getHostName() : host.getIpAddr(); final String scheme = request.getScheme(); int port = host.getInfoPort(); if ("https".equals(scheme)) { final Integer portObject = (Integer) getServletContext().getAttribute( DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY); if (portObject != null) { port = portObject; } } final String encodedPath = ServletUtil.getRawPath(request, "/fileChecksum"); String dtParam = ""; if (UserGroupInformation.isSecurityEnabled()) { String tokenString = ugi.getTokens().iterator().next().encodeToUrlString(); dtParam = JspHelper.getDelegationTokenUrlParam(tokenString); } String addr = nn.getNameNodeAddressHostPortString(); String addrParam = JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, addr); return new URL(scheme, hostname, port, "/getFileChecksum" + encodedPath + '?' + "ugi=" + ServletUtil.encodeQueryValue(ugi.getShortUserName()) + dtParam + addrParam); }
protected DFSClient getDFSClient(HttpServletRequest request) throws IOException, InterruptedException { final Configuration conf = (Configuration) getServletContext().getAttribute(JspHelper.CURRENT_CONF); UserGroupInformation ugi = getUGI(request, conf); final ServletContext context = getServletContext(); final DataNode datanode = (DataNode) context.getAttribute("datanode"); return DatanodeJspHelper.getDFSClient(request, datanode, conf, ugi); }
/** Create a redirection URL */ private URL createRedirectURL(String path, String encodedPath, HdfsFileStatus status, UserGroupInformation ugi, ClientProtocol nnproxy, HttpServletRequest request, String dt) throws IOException { String scheme = request.getScheme(); final LocatedBlocks blks = nnproxy.getBlockLocations( status.getFullPath(new Path(path)).toUri().getPath(), 0, 1); final Configuration conf = NameNodeHttpServer.getConfFromContext( getServletContext()); final DatanodeID host = pickSrcDatanode(blks, status, conf); final String hostname; if (host instanceof DatanodeInfo) { hostname = host.getHostName(); } else { hostname = host.getIpAddr(); } int port = "https".equals(scheme) ? host.getInfoSecurePort() : host .getInfoPort(); String dtParam = ""; if (dt != null) { dtParam = JspHelper.getDelegationTokenUrlParam(dt); } // Add namenode address to the url params NameNode nn = NameNodeHttpServer.getNameNodeFromContext( getServletContext()); String addr = nn.getNameNodeAddressHostPortString(); String addrParam = JspHelper.getUrlParam(JspHelper.NAMENODE_ADDRESS, addr); return new URL(scheme, hostname, port, "/streamFile" + encodedPath + '?' + "ugi=" + ServletUtil.encodeQueryValue(ugi.getShortUserName()) + dtParam + addrParam); }
/** Select a datanode to service this request. * Currently, this looks at no more than the first five blocks of a file, * selecting a datanode randomly from the most represented. * @param conf */ private DatanodeID pickSrcDatanode(LocatedBlocks blks, HdfsFileStatus i, Configuration conf) throws IOException { if (i.getLen() == 0 || blks.getLocatedBlocks().size() <= 0) { // pick a random datanode NameNode nn = NameNodeHttpServer.getNameNodeFromContext( getServletContext()); return NamenodeJspHelper.getRandomDatanode(nn); } return JspHelper.bestNode(blks, conf); }
/** Get DFSClient for a namenode corresponding to the BPID from a datanode */ public static DFSClient getDFSClient(final HttpServletRequest request, final DataNode datanode, final Configuration conf, final UserGroupInformation ugi) throws IOException, InterruptedException { final String nnAddr = request.getParameter(JspHelper.NAMENODE_ADDRESS); return getDFSClient(ugi, nnAddr, conf); }
private void setUpForDoGetTest(MiniDFSCluster cluster, Path testFile) { Mockito.doReturn(CONF).when(mockServletContext).getAttribute( JspHelper.CURRENT_CONF); Mockito.doReturn(NetUtils.getHostPortString(NameNode.getAddress(CONF))) .when(mockHttpServletRequest).getParameter("nnaddr"); Mockito.doReturn(testFile.toString()).when(mockHttpServletRequest) .getPathInfo(); Mockito.doReturn("/streamFile"+testFile.toString()).when(mockHttpServletRequest) .getRequestURI(); }
/** * Start the web server. */ @VisibleForTesting public void startInfoServer() throws IOException { final InetSocketAddress httpAddr = getHttpAddress(conf); final String httpsAddrString = conf.getTrimmed( DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_KEY, DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_DEFAULT); InetSocketAddress httpsAddr = NetUtils.createSocketAddr(httpsAddrString); HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf, httpAddr, httpsAddr, "secondary", DFSConfigKeys. DFS_SECONDARY_NAMENODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY, DFSConfigKeys.DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY); infoServer = builder.build(); infoServer.setAttribute("secondary.name.node", this); infoServer.setAttribute("name.system.image", checkpointImage); infoServer.setAttribute(JspHelper.CURRENT_CONF, conf); infoServer.addInternalServlet("imagetransfer", ImageServlet.PATH_SPEC, ImageServlet.class, true); infoServer.start(); LOG.info("Web server init done"); HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf); int connIdx = 0; if (policy.isHttpEnabled()) { InetSocketAddress httpAddress = infoServer.getConnectorAddress(connIdx++); conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, NetUtils.getHostPortString(httpAddress)); } if (policy.isHttpsEnabled()) { InetSocketAddress httpsAddress = infoServer.getConnectorAddress(connIdx); conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_KEY, NetUtils.getHostPortString(httpsAddress)); } }
private UserGroupInformation nonTokenUGI(String usernameFromQuery, String doAsUserFromQuery, String remoteUser) throws IOException { UserGroupInformation ugi = UserGroupInformation .createRemoteUser(remoteUser); JspHelper.checkUsername(ugi.getShortUserName(), usernameFromQuery); if (doAsUserFromQuery != null) { // create and attempt to authorize a proxy user ugi = UserGroupInformation.createProxyUser(doAsUserFromQuery, ugi); } return ugi; }
