@Override protected void setUp() throws Exception { // create runner instance runner = new ElasticsearchClusterRunner(); // create ES nodes runner.onBuild(new ElasticsearchClusterRunner.Builder() { @Override public void build(final int number, final Builder settingsBuilder) { } }).build( newConfigs().ramIndexStore().numOfNode(1) .clusterName(UUID.randomUUID().toString())); // wait for yellow status runner.ensureYellow(); }
private Client getClientForTransportProtocol() { final Client client; final Builder settingsBuilder = ImmutableSettings.builder(); settingsBuilder.put("name", "DataCleaner"); settingsBuilder.put("cluster.name", _clusterName); if (!StringUtils.isNullOrEmpty(_username) && !StringUtils.isNullOrEmpty(_password)) { settingsBuilder.put("shield.user", _username + ":" + _password); if (_ssl) { if (!Strings.isNullOrEmpty(_keystorePath)) { settingsBuilder.put("shield.ssl.keystore.path", _keystorePath); settingsBuilder.put("shield.ssl.keystore.password", _keystorePassword); } settingsBuilder.put("shield.transport.ssl", "true"); } } final Settings settings = settingsBuilder.build(); client = new TransportClient(settings); ((TransportClient) client).addTransportAddress(new InetSocketTransportAddress(_hostname, _port)); return client; }
@Override protected UsageAwareDatastoreConnection<ElasticSearchDataContext> createDatastoreConnection() { final Builder settingsBuilder = ImmutableSettings.builder(); settingsBuilder.put("name", "AnalyzerBeans"); settingsBuilder.put("cluster.name", _clusterName); final Settings settings = settingsBuilder.build(); final TransportClient client = new TransportClient(settings); client.addTransportAddress(new InetSocketTransportAddress(_hostname, _port)); final ElasticSearchDataContext dataContext; if (_tableDefs == null || _tableDefs.length == 0) { dataContext = new ElasticSearchDataContext(client, _indexName); } else { dataContext = new ElasticSearchDataContext(client, _indexName, _tableDefs); } return new DatastoreConnectionImpl<ElasticSearchDataContext>(dataContext, this); }
@Override protected void setUp() throws Exception { // create ES instance runner = new ElasticsearchClusterRunner(); // create ES nodes runner.onBuild(new ElasticsearchClusterRunner.Builder() { @Override public void build(final int number, final Builder settingBuilder) { } }).build( newConfigs() .clusterName("es-auth" + System.currentTimeMillis()) .ramIndexStore().numOfNode(1)); // wait for yellow status runner.ensureYellow(); }
@SuppressWarnings("resource") public static Client createClient(ElasticSearchConfig config) { if (client != null) { return client; } String elasticSearchHost = config.getHost(); int elasticSearchPort = config.getPort(); String clusterName = config.getClusterName(); Builder settingsBuilder = ImmutableSettings.settingsBuilder(); if (clusterName != null) { settingsBuilder.put("cluster.name", clusterName); } Settings settings = settingsBuilder.build(); if (elasticSearchHost != null) { logger.info("Creating a ElasticSearch TransportClient for address: {}:{}", elasticSearchHost, elasticSearchPort); int port = elasticSearchPort != 0 ? elasticSearchPort : 9300; InetSocketTransportAddress socketAddress = new InetSocketTransportAddress( elasticSearchHost, port); client = new TransportClient(settings).addTransportAddress(socketAddress); return client; } else { logger.info("Creating a ElasticSearch Node Client for address: %s:%s", elasticSearchHost, elasticSearchPort); clientNode = NodeBuilder.nodeBuilder().client(true).settings(settings).node(); client = clientNode.client(); return client; } }
private JestClient getClientForRestProtocol() { final JestClientFactory factory = new JestClientFactory(); HttpClientConfig.Builder builder = new HttpClientConfig.Builder("http://" + _hostname + ":" + _port).multiThreaded(true); if (!Strings.isNullOrEmpty(_username)) { builder = builder.defaultCredentials(_username, _password); } factory.setHttpClientConfig(builder.build()); return factory.getObject(); }
private Client getClientForJoiningClusterAsNode() { final Client client; final Builder settingsBuilder = ImmutableSettings.builder(); settingsBuilder.put("name", "DataCleaner"); settingsBuilder.put("shield.enabled", false); final Settings settings = settingsBuilder.build(); // .client(true) means no shards are stored on this node final Node node = nodeBuilder().clusterName(_clusterName).client(true).settings(settings).node(); client = node.client(); return client; }
@Override protected void setUp() throws Exception { // create runner instance runner = new ElasticsearchClusterRunner(); // create ES nodes runner.onBuild(new ElasticsearchClusterRunner.Builder() { @Override public void build(final int number, final Builder settingsBuilder) { } }).build(new String[] { "-numOfNode", "5", "-indexStoreType", "ram" }); // wait for yellow status runner.ensureYellow(); }
private void connect() { Builder builder = ImmutableSettings.settingsBuilder(); builder.put("cluster.name", cluster); builder.put("client.transport.ping_timeout", "30s"); TransportClient aClient = new TransportClient(builder.build()); aClient.addTransportAddress(new InetSocketTransportAddress(host, port)); Iterator<InetSocketTransportAddress> iter = otherHosts.iterator(); while (iter.hasNext()) { aClient.addTransportAddress(iter.next()); } _client = aClient; }
@Before public void setUp() throws Exception { runner = new ElasticsearchClusterRunner(); runner.onBuild(new ElasticsearchClusterRunner.Builder() { @Override public void build(final int index, final Builder settingsBuilder) { settingsBuilder.put("engine.filter.refresh", true); settingsBuilder.put("http.cors.enabled", true); settingsBuilder.put(QueryResultCache.INDICES_CACHE_QUERY_SIZE, "10%"); } }).build(newConfigs().numOfNode(1).ramIndexStore() .clusterName(UUID.randomUUID().toString())); runner.ensureGreen(); }
@Before public void setUp() throws Exception { runner = new ElasticsearchClusterRunner(); runner.onBuild(new ElasticsearchClusterRunner.Builder() { @Override public void build(final int index, final Builder settingsBuilder) { settingsBuilder.put("http.cors.enabled", true); } }).build( newConfigs().numOfNode(1).ramIndexStore() .clusterName(UUID.randomUUID().toString())); runner.ensureGreen(); }
/** * Stop the server. * @throws Exception */ public void stop() throws Exception { server.stop(); if (ds != null) { ds.close(); InitialContext ctx = TestUtil.initialContext(); ctx.unbind("java:comp/env/jdbc/ApiManagerDS"); } if (node != null) { if ("true".equals(System.getProperty("apiman.test.es-delete-index", "true"))) { client.execute(new DeleteIndex.Builder("apiman_manager").build()); } } }
/** * @see javax.servlet.ServletContextListener#contextInitialized(javax.servlet.ServletContextEvent) */ @Override public void contextInitialized(ServletContextEvent sce) { DistroESConfig config = new DistroESConfig(); File esHome = getDataDir(); if (esHome == null) { System.out.println("--------------------------------------------------------------"); System.err.println("No apiman-es data directory found. Embedded ES *not* started."); System.out.println("--------------------------------------------------------------"); return; } if (!esHome.exists()) { esHome.mkdirs(); } System.out.println("------------------------------------------------------------"); System.out.println("Starting apiman-es."); System.out.println(" HTTP Ports: " + config.getHttpPortRange()); System.out.println(" Transport Ports: " + config.getTransportPortRange()); System.out.println(" Bind Host: " + config.getBindHost()); System.out.println(" ES Data Dir: " + esHome); System.out.println("------------------------------------------------------------"); Builder settings = NodeBuilder.nodeBuilder().settings(); settings.put("path.home", esHome.getAbsolutePath()); settings.put("http.port", config.getHttpPortRange()); settings.put("transport.tcp.port", config.getTransportPortRange()); settings.put("discovery.zen.ping.multicast.enabled", false); if (config.getBindHost() != null) { settings.put("network.bind_host", config.getBindHost()); } String clusterName = "apiman"; node = NodeBuilder.nodeBuilder().client(false).clusterName(clusterName).data(true).local(false).settings(settings).build(); node.start(); System.out.println("-----------------------------"); System.out.println("apiman-es started!"); System.out.println("-----------------------------"); }
/** * Initialize any state for this DB. Called once per DB instance; there is * one DB instance per client thread. */ @Override public void init() throws DBException { // initialize OrientDB driver Properties props = getProperties(); this.indexKey = props.getProperty("es.index.key", DEFAULT_INDEX_KEY); String clusterName = props.getProperty("cluster.name", DEFAULT_CLUSTER_NAME); Boolean newdb = Boolean.parseBoolean(props.getProperty("elasticsearch.newdb", "false")); Builder settings = settingsBuilder() .put("node.local", "true") .put("path.data", System.getProperty("java.io.tmpdir") + "/esdata") .put("discovery.zen.ping.multicast.enabled", "false") .put("index.mapping._id.indexed", "true") .put("index.gateway.type", "none") .put("gateway.type", "none") .put("index.number_of_shards", "1") .put("index.number_of_replicas", "0"); //if properties file contains elasticsearch user defined properties //add it to the settings file (will overwrite the defaults). settings.put(props); System.out.println("ElasticSearch starting node = " + settings.get("cluster.name")); System.out.println("ElasticSearch node data path = " + settings.get("path.data")); node = nodeBuilder().clusterName(clusterName).settings(settings).node(); node.start(); client = node.client(); if (newdb) { client.admin().indices().prepareDelete(indexKey).execute().actionGet(); client.admin().indices().prepareCreate(indexKey).execute().actionGet(); } else { boolean exists = client.admin().indices().exists(Requests.indicesExistsRequest(indexKey)).actionGet().isExists(); if (!exists) { client.admin().indices().prepareCreate(indexKey).execute().actionGet(); } } }
@Override public void open(JobConf job, String name) throws IOException { clusterName = job.get(ElasticConstants.CLUSTER); host = job.get(ElasticConstants.HOST); port = job.getInt(ElasticConstants.PORT, 9300); Builder settingsBuilder = ImmutableSettings.settingsBuilder().classLoader( Settings.class.getClassLoader()); BufferedReader reader = new BufferedReader( job.getConfResourceAsReader("elasticsearch.conf")); String line; String parts[]; while ((line = reader.readLine()) != null) { if (StringUtils.isNotBlank(line) && !line.startsWith("#")) { line.trim(); parts = line.split("="); if (parts.length == 2) { settingsBuilder.put(parts[0].trim(), parts[1].trim()); } } } if (StringUtils.isNotBlank(clusterName)) settingsBuilder.put("cluster.name", clusterName); // Set the cluster name and build the settings Settings settings = settingsBuilder.build(); // Prefer TransportClient if (host != null && port > 1) { client = new TransportClient(settings) .addTransportAddress(new InetSocketTransportAddress(host, port)); } else if (clusterName != null) { node = nodeBuilder().settings(settings).client(true).node(); client = node.client(); } bulk = client.prepareBulk(); defaultIndex = job.get(ElasticConstants.INDEX, "nutch"); maxBulkDocs = job.getInt(ElasticConstants.MAX_BULK_DOCS, DEFAULT_MAX_BULK_DOCS); maxBulkLength = job.getInt(ElasticConstants.MAX_BULK_LENGTH, DEFAULT_MAX_BULK_LENGTH); }
/** * Stuff to do before the server is started. */ protected void preStart() throws Exception { if (ManagerTestUtils.getTestType() == TestType.jpa) { TestUtil.setProperty("apiman.hibernate.hbm2ddl.auto", "create-drop"); TestUtil.setProperty("apiman.hibernate.connection.datasource", "java:/comp/env/jdbc/ApiManagerDS"); try { InitialContext ctx = TestUtil.initialContext(); TestUtil.ensureCtx(ctx, "java:/comp/env"); TestUtil.ensureCtx(ctx, "java:/comp/env/jdbc"); String dbOutputPath = System.getProperty("apiman.test.h2-output-dir", null); if (dbOutputPath != null) { ds = createFileDatasource(new File(dbOutputPath)); } else { ds = createInMemoryDatasource(); } ctx.bind("java:/comp/env/jdbc/ApiManagerDS", ds); } catch (Exception e) { throw new RuntimeException(e); } } if (ManagerTestUtils.getTestType() == TestType.es && node == null) { System.out.println("Creating the ES node."); File esHome = new File("target/es"); String esHomeSP = System.getProperty("apiman.test.es-home", null); if (esHomeSP != null) { esHome = new File(esHomeSP); } if (esHome.isDirectory()) { FileUtils.deleteDirectory(esHome); } Builder settings = NodeBuilder.nodeBuilder().settings(); settings.put("path.home", esHome.getAbsolutePath()); settings.put("http.port", "6500-6600"); settings.put("transport.tcp.port", "6600-6700"); settings.put("discovery.zen.ping.multicast.enabled", false); String clusterName = System.getProperty("apiman.test.es-cluster-name", ES_CLUSTER_NAME); boolean isPersistent = "true".equals(System.getProperty("apiman.test.es-persistence", "false")); if (!isPersistent) { System.out.println("Creating non-persistent ES"); settings.put("index.store.type", "memory").put("gateway.type", "none") .put("index.number_of_shards", 1).put("index.number_of_replicas", 1); node = NodeBuilder.nodeBuilder().client(false).clusterName(clusterName).data(true).local(true) .settings(settings).build(); } else { System.out.println("Creating *persistent* ES here: " + esHome); node = NodeBuilder.nodeBuilder().client(false).clusterName(clusterName).data(true).local(false) .settings(settings).build(); } System.out.println("Starting the ES node."); node.start(); System.out.println("ES node was successfully started."); // TODO parameterize this String connectionUrl = "http://localhost:6500"; JestClientFactory factory = new JestClientFactory(); factory.setHttpClientConfig(new HttpClientConfig.Builder(connectionUrl).multiThreaded(true) .connTimeout(JEST_TIMEOUT ).readTimeout(JEST_TIMEOUT).build()); client = factory.getObject(); ES_CLIENT = client; } }