private void initWebHdfs(Configuration conf) throws IOException { if (WebHdfsFileSystem.isEnabled(conf, HttpServer2.LOG)) { // set user pattern based on configuration file UserParam.setUserPattern(conf.get( DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY, DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT)); // add authentication filter for webhdfs final String className = conf.get( DFSConfigKeys.DFS_WEBHDFS_AUTHENTICATION_FILTER_KEY, DFSConfigKeys.DFS_WEBHDFS_AUTHENTICATION_FILTER_DEFAULT); final String name = className; final String pathSpec = WebHdfsFileSystem.PATH_PREFIX + "/*"; Map<String, String> params = getAuthFilterParams(conf); HttpServer2.defineFilter(httpServer.getWebAppContext(), name, className, params, new String[] { pathSpec }); HttpServer2.LOG.info("Added filter '" + name + "' (class=" + className + ")"); // add webhdfs packages httpServer.addJerseyResourcePackage(NamenodeWebHdfsMethods.class .getPackage().getName() + ";" + Param.class.getPackage().getName(), pathSpec); } }
/** test that access via webhdfs puts proper entry in audit log */ @Test public void testAuditWebHdfs() throws Exception { final Path file = new Path(fnames[0]); fs.setPermission(file, new FsPermission((short)0644)); fs.setOwner(file, "root", null); setupAuditLogs(); WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo, conf, WebHdfsFileSystem.SCHEME); InputStream istream = webfs.open(file); int val = istream.read(); istream.close(); verifyAuditLogsRepeat(true, 3); assertTrue("failed to read from file", val >= 0); }
/** test that stat via webhdfs puts proper entry in audit log */ @Test public void testAuditWebHdfsStat() throws Exception { final Path file = new Path(fnames[0]); fs.setPermission(file, new FsPermission((short)0644)); fs.setOwner(file, "root", null); setupAuditLogs(); WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo, conf, WebHdfsFileSystem.SCHEME); FileStatus st = webfs.getFileStatus(file); verifyAuditLogs(true); assertTrue("failed to stat file", st != null && st.isFile()); }
/** test that denied access via webhdfs puts proper entry in audit log */ @Test public void testAuditWebHdfsDenied() throws Exception { final Path file = new Path(fnames[0]); fs.setPermission(file, new FsPermission((short)0600)); fs.setOwner(file, "root", null); setupAuditLogs(); try { WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo, conf, WebHdfsFileSystem.SCHEME); InputStream istream = webfs.open(file); int val = istream.read(); fail("open+read must not succeed, got " + val); } catch(AccessControlException E) { System.out.println("got access denied, as expected."); } verifyAuditLogsRepeat(false, 2); }
private void initWebHdfs(Configuration conf) throws IOException { // set user pattern based on configuration file UserParam.setUserPattern(conf.get( HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY, HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT)); // add authentication filter for webhdfs final String className = conf.get( DFSConfigKeys.DFS_WEBHDFS_AUTHENTICATION_FILTER_KEY, DFSConfigKeys.DFS_WEBHDFS_AUTHENTICATION_FILTER_DEFAULT); final String name = className; final String pathSpec = WebHdfsFileSystem.PATH_PREFIX + "/*"; Map<String, String> params = getAuthFilterParams(conf); HttpServer2.defineFilter(httpServer.getWebAppContext(), name, className, params, new String[] { pathSpec }); HttpServer2.LOG.info("Added filter '" + name + "' (class=" + className + ")"); // add webhdfs packages httpServer.addJerseyResourcePackage(NamenodeWebHdfsMethods.class .getPackage().getName() + ";" + Param.class.getPackage().getName(), pathSpec); }
/** test that access via webhdfs puts proper entry in audit log */ @Test public void testAuditWebHdfs() throws Exception { final Path file = new Path(fnames[0]); fs.setPermission(file, new FsPermission((short)0644)); fs.setOwner(file, "root", null); setupAuditLogs(); WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo, conf, WebHdfsConstants.WEBHDFS_SCHEME); InputStream istream = webfs.open(file); int val = istream.read(); istream.close(); verifyAuditLogsRepeat(true, 3); assertTrue("failed to read from file", val >= 0); }
/** test that stat via webhdfs puts proper entry in audit log */ @Test public void testAuditWebHdfsStat() throws Exception { final Path file = new Path(fnames[0]); fs.setPermission(file, new FsPermission((short)0644)); fs.setOwner(file, "root", null); setupAuditLogs(); WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo, conf, WebHdfsConstants.WEBHDFS_SCHEME); FileStatus st = webfs.getFileStatus(file); verifyAuditLogs(true); assertTrue("failed to stat file", st != null && st.isFile()); }
/** test that denied access via webhdfs puts proper entry in audit log */ @Test public void testAuditWebHdfsDenied() throws Exception { final Path file = new Path(fnames[0]); fs.setPermission(file, new FsPermission((short)0600)); fs.setOwner(file, "root", null); setupAuditLogs(); try { WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo, conf, WebHdfsConstants.WEBHDFS_SCHEME); InputStream istream = webfs.open(file); int val = istream.read(); fail("open+read must not succeed, got " + val); } catch(AccessControlException E) { System.out.println("got access denied, as expected."); } verifyAuditLogsRepeat(false, 2); }
@Test public void testGetContentSummaryForEmptyDirectory() throws Exception { try (WebImageViewer viewer = new WebImageViewer( NetUtils.createSocketAddr("localhost:0"))) { viewer.initServer(originalFsimage.getAbsolutePath()); int port = viewer.getPort(); URL url = new URL("http://localhost:" + port + "/webhdfs/v1/parentDir/childDir2?op=GETCONTENTSUMMARY"); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); connection.setRequestMethod("GET"); connection.connect(); assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode()); // create a WebHdfsFileSystem instance URI uri = new URI("webhdfs://localhost:" + String.valueOf(port)); Configuration conf = new Configuration(); WebHdfsFileSystem webfs = (WebHdfsFileSystem) FileSystem.get(uri, conf); ContentSummary summary = webfs.getContentSummary(new Path( "/parentDir/childDir2")); verifyContentSummary(emptyDirSummaryFromDFS, summary); } }
@Test public void testGetContentSummaryForDirectory() throws Exception { try (WebImageViewer viewer = new WebImageViewer( NetUtils.createSocketAddr("localhost:0"))) { viewer.initServer(originalFsimage.getAbsolutePath()); int port = viewer.getPort(); URL url = new URL("http://localhost:" + port + "/webhdfs/v1/parentDir/?op=GETCONTENTSUMMARY"); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); connection.setRequestMethod("GET"); connection.connect(); assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode()); // create a WebHdfsFileSystem instance URI uri = new URI("webhdfs://localhost:" + String.valueOf(port)); Configuration conf = new Configuration(); WebHdfsFileSystem webfs = (WebHdfsFileSystem) FileSystem.get(uri, conf); ContentSummary summary = webfs.getContentSummary(new Path("/parentDir/")); verifyContentSummary(summaryFromDFS, summary); } }
@Test public void testGetContentSummaryForFile() throws Exception { try (WebImageViewer viewer = new WebImageViewer( NetUtils.createSocketAddr("localhost:0"))) { viewer.initServer(originalFsimage.getAbsolutePath()); int port = viewer.getPort(); URL url = new URL("http://localhost:" + port + "/webhdfs/v1/parentDir/file1?op=GETCONTENTSUMMARY"); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); connection.setRequestMethod("GET"); connection.connect(); assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode()); // create a WebHdfsFileSystem instance URI uri = new URI("webhdfs://localhost:" + String.valueOf(port)); Configuration conf = new Configuration(); WebHdfsFileSystem webfs = (WebHdfsFileSystem) FileSystem.get(uri, conf); ContentSummary summary = webfs. getContentSummary(new Path("/parentDir/file1")); verifyContentSummary(fileSummaryFromDFS, summary); } }
/** test that access via webhdfs puts proper entry in audit log */ @Test public void testAuditWebHdfs() throws Exception { final Path file = new Path(fnames[0]); fs.setPermission(file, new FsPermission((short)0644)); fs.setOwner(file, "root", null); setupAuditLogs(); WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo, conf); InputStream istream = webfs.open(file); int val = istream.read(); istream.close(); verifyAuditLogsRepeat(true, 3); assertTrue("failed to read from file", val >= 0); }
/** test that stat via webhdfs puts proper entry in audit log */ @Test public void testAuditWebHdfsStat() throws Exception { final Path file = new Path(fnames[0]); fs.setPermission(file, new FsPermission((short)0644)); fs.setOwner(file, "root", null); setupAuditLogs(); WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo, conf); FileStatus st = webfs.getFileStatus(file); verifyAuditLogs(true); assertTrue("failed to stat file", st != null && st.isFile()); }
/** test that denied access via webhdfs puts proper entry in audit log */ @Test public void testAuditWebHdfsDenied() throws Exception { final Path file = new Path(fnames[0]); fs.setPermission(file, new FsPermission((short)0600)); fs.setOwner(file, "root", null); setupAuditLogs(); try { WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo, conf); InputStream istream = webfs.open(file); int val = istream.read(); fail("open+read must not succeed, got " + val); } catch(AccessControlException E) { System.out.println("got access denied, as expected."); } verifyAuditLogsRepeat(false, 2); }
/** * test that access via webhdfs puts proper entry in audit log */ @Test public void testAuditWebHdfs() throws Exception { final Path file = new Path(fnames[0]); fs.setPermission(file, new FsPermission((short) 0644)); fs.setOwner(file, "root", null); setupAuditLogs(); WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo, conf); InputStream istream = webfs.open(file); int val = istream.read(); istream.close(); verifyAuditLogsRepeat(true, 3); assertTrue("failed to read from file", val >= 0); }
/** * test that stat via webhdfs puts proper entry in audit log */ @Test public void testAuditWebHdfsStat() throws Exception { final Path file = new Path(fnames[0]); fs.setPermission(file, new FsPermission((short) 0644)); fs.setOwner(file, "root", null); setupAuditLogs(); WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo, conf); FileStatus st = webfs.getFileStatus(file); verifyAuditLogs(true); assertTrue("failed to stat file", st != null && st.isFile()); }
/** * test that denied access via webhdfs puts proper entry in audit log */ @Test public void testAuditWebHdfsDenied() throws Exception { final Path file = new Path(fnames[0]); fs.setPermission(file, new FsPermission((short) 0600)); fs.setOwner(file, "root", null); setupAuditLogs(); try { WebHdfsFileSystem webfs = WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo, conf); InputStream istream = webfs.open(file); int val = istream.read(); fail("open+read must not succeed, got " + val); } catch (AccessControlException E) { System.out.println("got access denied, as expected."); } verifyAuditLogsRepeat(false, 2); }
/** * Initializes the service. * * @throws ServiceException thrown if the service could not be initialized. */ @Override protected void init() throws ServiceException { long updateInterval = getServiceConfig().getLong(UPDATE_INTERVAL, DAY); long maxLifetime = getServiceConfig().getLong(MAX_LIFETIME, 7 * DAY); long renewInterval = getServiceConfig().getLong(RENEW_INTERVAL, DAY); tokenKind = (HttpFSServerWebApp.get().isSslEnabled()) ? SWebHdfsFileSystem.TOKEN_KIND : WebHdfsFileSystem.TOKEN_KIND; secretManager = new DelegationTokenSecretManager(tokenKind, updateInterval, maxLifetime, renewInterval, HOUR); try { secretManager.startThreads(); } catch (IOException ex) { throw new ServiceException(ServiceException.ERROR.S12, DelegationTokenManager.class.getSimpleName(), ex.toString(), ex); } }