private Configuration createSecureConfig(String dataTransferProtection) throws Exception { HdfsConfiguration conf = new HdfsConfiguration(); SecurityUtil.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS, conf); conf.set(DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, hdfsPrincipal); conf.set(DFS_NAMENODE_KEYTAB_FILE_KEY, keytab); conf.set(DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, hdfsPrincipal); conf.set(DFS_DATANODE_KEYTAB_FILE_KEY, keytab); conf.set(DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, spnegoPrincipal); conf.setBoolean(DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true); conf.set(DFS_DATA_TRANSFER_PROTECTION_KEY, dataTransferProtection); conf.set(DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name()); conf.set(DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0"); conf.set(DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0"); conf.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY, 10); conf.set(DFS_ENCRYPT_DATA_TRANSFER_KEY, "true");//https://issues.apache.org/jira/browse/HDFS-7431 String keystoresDir = baseDir.getAbsolutePath(); String sslConfDir = KeyStoreTestUtil.getClasspathDir(this.getClass()); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); return conf; }
@AfterClass public static void tearDown() throws Exception { if (testMiniKDC != null) { testMiniKDC.stop(); } if (testTimelineServer != null) { testTimelineServer.stop(); } if (withSsl) { KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir); File base = new File(BASEDIR); FileUtil.fullyDelete(base); } }
@BeforeClass public static void setupServer() throws Exception { conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE, MemoryTimelineStore.class, TimelineStore.class); conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, "HTTPS_ONLY"); File base = new File(BASEDIR); FileUtil.fullyDelete(base); base.mkdirs(); keystoresDir = new File(BASEDIR).getAbsolutePath(); sslConfDir = KeyStoreTestUtil.getClasspathDir(TestTimelineWebServicesWithSSL.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); conf.addResource("ssl-server.xml"); conf.addResource("ssl-client.xml"); timelineServer = new ApplicationHistoryServer(); timelineServer.init(conf); timelineServer.start(); store = timelineServer.getTimelineStore(); }
@BeforeClass public static void setUp() throws Exception { conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTP_AND_HTTPS.name()); conf.set(NfsConfigKeys.NFS_HTTP_ADDRESS_KEY, "localhost:0"); conf.set(NfsConfigKeys.NFS_HTTPS_ADDRESS_KEY, "localhost:0"); // Use emphral port in case tests are running in parallel conf.setInt(NfsConfigKeys.DFS_NFS_SERVER_PORT_KEY, 0); conf.setInt(NfsConfigKeys.DFS_NFS_MOUNTD_PORT_KEY, 0); File base = new File(BASEDIR); FileUtil.fullyDelete(base); base.mkdirs(); keystoresDir = new File(BASEDIR).getAbsolutePath(); sslConfDir = KeyStoreTestUtil.getClasspathDir(TestNfs3HttpServer.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); cluster.waitActive(); }
@BeforeClass public static void setUp() throws Exception { conf = new Configuration(); conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true); conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name()); conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0"); conf.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0"); File base = new File(BASEDIR); FileUtil.fullyDelete(base); base.mkdirs(); keystoresDir = new File(BASEDIR).getAbsolutePath(); sslConfDir = KeyStoreTestUtil.getClasspathDir(TestHttpsFileSystem.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); cluster.waitActive(); OutputStream os = cluster.getFileSystem().create(new Path("/test")); os.write(23); os.close(); InetSocketAddress addr = cluster.getNameNode().getHttpsAddress(); nnAddr = NetUtils.getHostPortString(addr); conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, nnAddr); }
/** * Creates configuration for starting a secure cluster. * * @param dataTransferProtection supported QOPs * @return configuration for starting a secure cluster * @throws Exception if there is any failure */ protected HdfsConfiguration createSecureConfig( String dataTransferProtection) throws Exception { HdfsConfiguration conf = new HdfsConfiguration(); SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf); conf.set(DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, hdfsPrincipal); conf.set(DFS_NAMENODE_KEYTAB_FILE_KEY, keytab); conf.set(DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, hdfsPrincipal); conf.set(DFS_DATANODE_KEYTAB_FILE_KEY, keytab); conf.set(DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, spnegoPrincipal); conf.setBoolean(DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true); conf.set(DFS_DATA_TRANSFER_PROTECTION_KEY, dataTransferProtection); conf.set(DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name()); conf.set(DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0"); conf.set(DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0"); conf.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY, 10); String keystoresDir = baseDir.getAbsolutePath(); String sslConfDir = KeyStoreTestUtil.getClasspathDir(this.getClass()); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); return conf; }
@BeforeClass public static void setupServer() throws Exception { conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE, MemoryTimelineStore.class, TimelineStore.class); conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, "HTTPS_ONLY"); conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.0f); File base = new File(BASEDIR); FileUtil.fullyDelete(base); base.mkdirs(); keystoresDir = new File(BASEDIR).getAbsolutePath(); sslConfDir = KeyStoreTestUtil.getClasspathDir(TestTimelineWebServicesWithSSL.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); conf.addResource("ssl-server.xml"); conf.addResource("ssl-client.xml"); timelineServer = new ApplicationHistoryServer(); timelineServer.init(conf); timelineServer.start(); store = timelineServer.getTimelineStore(); }
/** * Creates configuration for starting a secure cluster. * * @param dataTransferProtection supported QOPs * @return configuration for starting a secure cluster * @throws Exception if there is any failure */ protected HdfsConfiguration createSecureConfig( String dataTransferProtection) throws Exception { HdfsConfiguration conf = new HdfsConfiguration(); SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf); conf.set(DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, hdfsPrincipal); conf.set(DFS_NAMENODE_KEYTAB_FILE_KEY, hdfsKeytab); conf.set(DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, hdfsPrincipal); conf.set(DFS_DATANODE_KEYTAB_FILE_KEY, hdfsKeytab); conf.set(DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, spnegoPrincipal); conf.setBoolean(DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true); conf.set(DFS_DATA_TRANSFER_PROTECTION_KEY, dataTransferProtection); conf.set(DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name()); conf.set(DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0"); conf.set(DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0"); conf.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY, 10); String keystoresDir = baseDir.getAbsolutePath(); String sslConfDir = KeyStoreTestUtil.getClasspathDir(this.getClass()); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); conf.set(DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, KeyStoreTestUtil.getClientSSLConfigFileName()); conf.set(DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, KeyStoreTestUtil.getServerSSLConfigFileName()); return conf; }
@BeforeClass public static void setUp() throws Exception { File base = new File(BASEDIR); FileUtil.fullyDelete(base); base.mkdirs(); conf = new Configuration(); keystoresDir = new File(BASEDIR).getAbsolutePath(); sslConfDir = KeyStoreTestUtil.getClasspathDir(TestNameNodeHttpServer.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); connectionFactory = URLConnectionFactory .newDefaultURLConnectionFactory(conf); conf.set(DFSConfigKeys.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, KeyStoreTestUtil.getClientSSLConfigFileName()); conf.set(DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, KeyStoreTestUtil.getServerSSLConfigFileName()); }
@After public void tearDown() throws Exception { if (testMiniKDC != null) { testMiniKDC.stop(); } if (testTimelineServer != null) { testTimelineServer.stop(); } if (client != null) { client.stop(); } if (withSsl) { KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir); File base = new File(BASEDIR); FileUtil.fullyDelete(base); } }
@Before public void setUp() throws Exception { LOG.debug("Error mode: " + error_mode.name()); conf = new YarnConfiguration(); filesToPurge = prepareCryptoMaterial(conf, KeyStoreTestUtil .getClasspathDir(TestYarnSSLServer.class)); setCryptoConfig(conf); conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true); cluster = new MiniYARNCluster(TestYarnSSLServer.class.getName(), 1, 3, 1, 1, false, true); cluster.init(conf); cluster.start(); LOG.info("Started cluster"); acClient = ClientRMProxy.createRMProxy(conf, ApplicationClientProtocol.class, true); }
@Before public void setUp() throws Exception { conf = new HdfsConfiguration(); filesToPurge = prepareCryptoMaterial(conf, KeyStoreTestUtil.getClasspathDir(TestDFSSSLServer.class)); setCryptoConfig(conf); String testDataPath = System .getProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA, "build/test/data"); File testDataCluster1 = new File(testDataPath, "dfs_cluster"); String c1Path = testDataCluster1.getAbsolutePath(); conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, c1Path); // Force DatanNode non-RPC communication to use plaintext socket // Until we test starting DN in secure-mode conf.set("hadoop.rpc.socket.factory.class.ClientProtocol", "org.apache.hadoop.net.StandardSocketFactory"); cluster = new MiniDFSCluster.Builder(conf).build(); LOG.info("DFS cluster started"); }
public void startServer(boolean isTestSessionCookie) throws Exception { Configuration conf = new Configuration(); if (isTestSessionCookie) { conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY, DummyFilterInitializer.class.getName()); } else { conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY, Dummy2FilterInitializer.class.getName()); } File base = new File(BASEDIR); FileUtil.fullyDelete(base); base.mkdirs(); keystoresDir = new File(BASEDIR).getAbsolutePath(); sslConfDir = KeyStoreTestUtil.getClasspathDir(TestSSLHttpServer.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); Configuration sslConf = KeyStoreTestUtil.getSslConfig(); server = new HttpServer2.Builder() .setName("test") .addEndpoint(new URI("http://localhost")) .addEndpoint(new URI("https://localhost")) .setConf(conf) .keyPassword(sslConf.get("ssl.server.keystore.keypassword")) .keyStore(sslConf.get("ssl.server.keystore.location"), sslConf.get("ssl.server.keystore.password"), sslConf.get("ssl.server.keystore.type", "jks")) .trustStore(sslConf.get("ssl.server.truststore.location"), sslConf.get("ssl.server.truststore.password"), sslConf.get("ssl.server.truststore.type", "jks")).build(); server.addServlet("echo", "/echo", TestHttpServer.EchoServlet.class); server.start(); }
@BeforeClass public static void setup() throws Exception { conf = new Configuration(); conf.setInt(HttpServer2.HTTP_MAX_THREADS, 10); File base = new File(BASEDIR); FileUtil.fullyDelete(base); base.mkdirs(); keystoresDir = new File(BASEDIR).getAbsolutePath(); sslConfDir = KeyStoreTestUtil.getClasspathDir(TestSSLHttpServer.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false, true, excludeCiphers); Configuration sslConf = KeyStoreTestUtil.getSslConfig(); clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, sslConf); clientSslFactory.init(); server = new HttpServer2.Builder() .setName("test") .addEndpoint(new URI("https://localhost")) .setConf(conf) .keyPassword(sslConf.get("ssl.server.keystore.keypassword")) .keyStore(sslConf.get("ssl.server.keystore.location"), sslConf.get("ssl.server.keystore.password"), sslConf.get("ssl.server.keystore.type", "jks")) .trustStore(sslConf.get("ssl.server.truststore.location"), sslConf.get("ssl.server.truststore.password"), sslConf.get("ssl.server.truststore.type", "jks")) .excludeCiphers( sslConf.get("ssl.server.exclude.cipher.list")).build(); server.addServlet("echo", "/echo", TestHttpServer.EchoServlet.class); server.addServlet("longheader", "/longheader", LongHeaderServlet.class); server.start(); baseUrl = new URL("https://" + NetUtils.getHostPortString(server.getConnectorAddress(0))); LOG.info("HTTP server started: " + baseUrl); }
@AfterClass public static void cleanup() throws Exception { server.stop(); FileUtil.fullyDelete(new File(BASEDIR)); KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir); clientSslFactory.destroy(); }
@BeforeClass public static void setUp() throws Exception { Configuration conf = new Configuration(); conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY, DummyFilterInitializer.class.getName()); File base = new File(BASEDIR); FileUtil.fullyDelete(base); base.mkdirs(); keystoresDir = new File(BASEDIR).getAbsolutePath(); sslConfDir = KeyStoreTestUtil.getClasspathDir(TestSSLHttpServer.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); Configuration sslConf = KeyStoreTestUtil.getSslConfig(); clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, sslConf); clientSslFactory.init(); server = new HttpServer2.Builder() .setName("test") .addEndpoint(new URI("http://localhost")) .addEndpoint(new URI("https://localhost")) .setConf(conf) .keyPassword(sslConf.get("ssl.server.keystore.keypassword")) .keyStore(sslConf.get("ssl.server.keystore.location"), sslConf.get("ssl.server.keystore.password"), sslConf.get("ssl.server.keystore.type", "jks")) .trustStore(sslConf.get("ssl.server.truststore.location"), sslConf.get("ssl.server.truststore.password"), sslConf.get("ssl.server.truststore.type", "jks")) .excludeCiphers( sslConf.get("ssl.server.exclude.cipher.list")) .build(); server.addServlet("echo", "/echo", TestHttpServer.EchoServlet.class); server.start(); }
private void encryptedShuffleWithCerts(boolean useClientCerts) throws Exception { try { Configuration conf = new Configuration(); String keystoresDir = new File(BASEDIR).getAbsolutePath(); String sslConfsDir = KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfsDir, conf, useClientCerts); conf.setBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY, true); startCluster(conf); FileSystem fs = FileSystem.get(getJobConf()); Path inputDir = new Path("input"); fs.mkdirs(inputDir); Writer writer = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt"))); writer.write("hello"); writer.close(); Path outputDir = new Path("output", "output"); JobConf jobConf = new JobConf(getJobConf()); jobConf.setInt("mapred.map.tasks", 1); jobConf.setInt("mapred.map.max.attempts", 1); jobConf.setInt("mapred.reduce.max.attempts", 1); jobConf.set("mapred.input.dir", inputDir.toString()); jobConf.set("mapred.output.dir", outputDir.toString()); JobClient jobClient = new JobClient(jobConf); RunningJob runJob = jobClient.submitJob(jobConf); runJob.waitForCompletion(); Assert.assertTrue(runJob.isComplete()); Assert.assertTrue(runJob.isSuccessful()); } finally { stopCluster(); } }
@AfterClass public static void tearDown() throws Exception { FileUtil.fullyDelete(new File(BASEDIR)); if (cluster != null) { cluster.shutdown(); } KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir); }
@BeforeClass public static void setUp() throws Exception { config = new Configuration(); cluster = new MiniDFSCluster.Builder(config).numDataNodes(2).build(); blockPoolId = cluster.getNamesystem().getBlockPoolId(); hftpUri = "hftp://" + config.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY); File base = new File(BASEDIR); FileUtil.fullyDelete(base); base.mkdirs(); keystoresDir = new File(BASEDIR).getAbsolutePath(); sslConfDir = KeyStoreTestUtil.getClasspathDir(TestHftpFileSystem.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, config, false); }
@AfterClass public static void tearDown() throws Exception { if (cluster != null) { cluster.shutdown(); } FileUtil.fullyDelete(new File(BASEDIR)); KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir); }
@BeforeClass public static void setUp() throws Exception { File base = new File(BASEDIR); FileUtil.fullyDelete(base); base.mkdirs(); conf = new Configuration(); keystoresDir = new File(BASEDIR).getAbsolutePath(); sslConfDir = KeyStoreTestUtil.getClasspathDir(TestNameNodeHttpServer.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); connectionFactory = URLConnectionFactory .newDefaultURLConnectionFactory(conf); }
private static void setupSsl() throws Exception { Configuration conf = new Configuration(); conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true); conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name()); conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0"); conf.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0"); File base = new File(BASEDIR); FileUtil.fullyDelete(base); assertTrue(base.mkdirs()); final String keystoresDir = new File(BASEDIR).getAbsolutePath(); final String sslConfDir = KeyStoreTestUtil.getClasspathDir(TestNameNodeRespectsBindHostKeys.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); }