@Test public void testCheckException() throws Exception { TestCassandraHealthCheck testCassandraHealthCheck = new TestCassandraHealthCheck(TEST_SERVER); testCassandraHealthCheck.cluster = testCluster; FieldUtils.writeField(testCassandraHealthCheck, "logger", loggerMock, true); Mockito.when(session.execute(Matchers.anyString())).thenThrow(new RuntimeException("crap")); Result result = testCassandraHealthCheck.check(); Assert.assertFalse(result.isHealthy()); Mockito.verify(session).close(); Mockito.verify(session).execute(Matchers.anyString()); Assert.assertEquals(1, testCluster.nbrTimesCloseCalled); ArgumentCaptor<ContextedRuntimeException> exCaptor = ArgumentCaptor.forClass(ContextedRuntimeException.class); Mockito.verify(loggerMock).error(Matchers.anyString(), exCaptor.capture()); Assert.assertEquals(3, exCaptor.getValue().getContextLabels().size()); Assert.assertEquals(result.getError(), exCaptor.getValue()); }
@Test public void testCheckInvalidDatabase() throws Exception { TestMongoDbHealthCheck healthCheck = setTpcheckMocks(); Mockito.when(commandResult.get(Matchers.anyString())).thenReturn(Integer.valueOf(0)); Result result = healthCheck.check(); Mockito.verify(loggerMock).debug("connectionUrl={} databaseList={} stats={}", TEST_CONNECT_URL, "", "commandResult"); Mockito.verify(mongoClientMock).close(); Assert.assertFalse(result.isHealthy()); ArgumentCaptor<ContextedRuntimeException> exCaptor = ArgumentCaptor.forClass(ContextedRuntimeException.class); Mockito.verify(loggerMock).error(Matchers.anyString(), exCaptor.capture()); Assert.assertEquals(4, exCaptor.getValue().getContextLabels().size()); Assert.assertEquals("Database has nothing in it.", exCaptor.getValue().getCause().getMessage()); }
@Override public void filter(ContainerRequestContext requestContext) throws IOException { RequestLog.startTiming(this); String path = requestContext.getUriInfo().getAbsolutePath().getPath(); if (path.startsWith("/v1/data") || path.startsWith("/data")) { // See if we have any unhealthy checks Map<String, Result> unhealthyChecks = getUnhealthy(); if (!unhealthyChecks.keySet().isEmpty()) { unhealthyChecks.entrySet() .forEach(entry -> LOG.error("Healthcheck '{}' failed: {}", entry.getKey(), entry.getValue())); RequestLog.stopTiming(this); requestContext.abortWith( Response.status(Status.SERVICE_UNAVAILABLE) .entity("Service is unhealthy. At least 1 healthcheck is failing") .build() ); return; } } RequestLog.stopTiming(this); }
@Test public void testBasic() throws Exception { Assert.assertTrue(healthCheck.execute().equals(Result.healthy())); Assert.assertTrue(connectionPool.getNumActive()==0); poolableConnectionFactory.setValidationQuery("crap"); Result testResult = healthCheck.execute(); Assert.assertTrue(!testResult.isHealthy()); Assert.assertTrue(testResult.getMessage() != null); Assert.assertTrue(testResult.getMessage().contains("validation error")); poolableConnectionFactory.setValidationQuery(VALIDATION_SQL); healthCheck.setMaxWaitingConnections(-1); testResult = healthCheck.execute(); Assert.assertTrue(!testResult.isHealthy()); Assert.assertTrue(testResult.getMessage() != null); Assert.assertTrue(testResult.getMessage().contains("Overloaded connection pool")); Assert.assertTrue(healthCheck.getMaxWaitingConnections() == -1); }
@Override public void run() { StormClusterState clusterState = workerData.getZkCluster(); String topologyId = workerData.getTopologyId(); Map<Integer, HealthCheckRegistry> taskHealthCheckMap = JStormHealthCheck.getTaskhealthcheckmap(); int cnt = 0; for (Map.Entry<Integer, HealthCheckRegistry> entry : taskHealthCheckMap.entrySet()) { Integer taskId = entry.getKey(); Map<String, Result> results = entry.getValue().runHealthChecks(); for (Map.Entry<String, Result> result : results.entrySet()) { if (!result.getValue().isHealthy()) { try { clusterState.report_task_error(topologyId, taskId, result.getValue().getMessage(), null); cnt++; } catch (Exception e) { LOG.error("Failed to update health data in ZK for topo-{} task-{}.", topologyId, taskId, e); } } } } LOG.info("Successfully updated {} health data to ZK for topology:{}", cnt, topologyId); }
@Override public void handle(final Request req, final Response rsp) throws Throwable { HealthCheckRegistry registry = req.require(HealthCheckRegistry.class); SortedMap<String, Result> checks = req.param("name").toOptional().map(name -> { SortedMap<String, Result> set = ImmutableSortedMap.of(name, registry.runHealthCheck(name)); return set; }).orElseGet(() -> registry.runHealthChecks()); final Status status; if (checks.isEmpty()) { status = Status.NOT_IMPLEMENTED; } else { status = checks.values().stream() .filter(it -> !it.isHealthy()) .findFirst() .map(it -> Status.SERVER_ERROR) .orElse(Status.OK); } rsp.status(status) .header("Cache-Control", "must-revalidate,no-cache,no-store") .send(checks); }
@Override protected Result check() throws Exception { LOG.warn("This is NOT an integration test. The current Healthcheck only checks if all the endpoints are reachable"); if(!portIsOpen(deployer.getHost(), deployer.getPort())){ return Result.unhealthy("The Deployer endpoint is not ready"); } if(!portIsOpen(grafana.getHost(), grafana.getPort())){ return Result.unhealthy("The Monitor endpoint is not ready"); } if(!portIsOpen(sla.getHost(), sla.getPort())) { return Result.unhealthy("The SLA endpoint is not ready"); } if(!portIsOpen(planner.getHost(), planner.getPort())){ return Result.unhealthy("The Planner endpoint is not ready"); } return Result.healthy(); }
@Override public ModelAndView doGet(IHTTPSession session) { ModelAndView result = new ModelAndView(); JsonObject entity = Json.object(); for (Entry<String, Result> cur : Metrics.HEALTH_REGISTRY.runHealthChecks().entrySet()) { JsonObject value = Json.object().add("status", cur.getValue().getDetails().get("status").toString()); if (!cur.getValue().isHealthy()) { value.add("message", cur.getValue().getMessage()); } entity.add(cur.getKey(), value); } result.setData(entity.toString()); return result; }
public static void health(HttpServerExchange exchange) { SortedMap<String, Result> results = HealthChecks.getHealthCheckRegistry().runHealthChecks(); boolean unhealthy = results.values().stream().anyMatch(result -> !result.isHealthy()); if (unhealthy) { /* * Set a 500 status code also. A lot of systems / dev ops tools can * easily test status codes but are not set up to parse JSON. * Let's keep it simple for everyone. */ exchange.setStatusCode(500); } Exchange.body().sendJson(exchange, results); }
@Test public void shouldReturnNotHealthyBecauseOfErrorOnError() throws Exception { when(h.execute("select 1")).thenThrow(new MappingException("bad error here")); Result check = healthCheck.check(); assertThat(check).isNotNull() .extracting(Result::getMessage) .containsOnly("bad error here"); }
@Test public void testCheck() throws Exception { TestCassandraHealthCheck testCassandraHealthCheck = new TestCassandraHealthCheck(TEST_SERVER); testCassandraHealthCheck.cluster = testCluster; Result result = testCassandraHealthCheck.check(); Assert.assertTrue(result.isHealthy()); Mockito.verify(session).close(); Mockito.verify(session).execute(Matchers.anyString()); Assert.assertEquals(1, testCluster.nbrTimesCloseCalled); }
@Test public void checkHappyPath() throws Exception { mockSetup(); Result result = healthCheck.check(); Assert.assertTrue(result.isHealthy()); Mockito.verify(connectionMock).close(); Mockito.verify(statementMock).close(); Mockito.verify(resultSetMock).close(); }
@Test public void checkQueryExecutionFailure() throws Exception { Mockito.when(dataSourceMock.getConnection()).thenReturn(connectionMock); Mockito.when(connectionMock.createStatement()).thenReturn(statementMock); Mockito.when(statementMock.executeQuery(TEST_QUERY)).thenThrow(TEST_EXCEPTION); Result result = healthCheck.check(); Assert.assertTrue(!result.isHealthy()); Assert.assertEquals(TEST_EXCEPTION, result.getError().getCause()); Mockito.verify(loggerMock).error(Matchers.anyString(), Matchers.any(ContextedRuntimeException.class)); Mockito.verify(connectionMock).close(); Mockito.verify(statementMock).close(); }
@Test public void checkCloseFailure() throws Exception { mockSetup(); Mockito.doThrow(TEST_EXCEPTION).when(connectionMock).close(); Result result = healthCheck.check(); Assert.assertTrue(result.isHealthy()); Mockito.verify(loggerMock).warn(Matchers.anyString(), Matchers.any(ContextedRuntimeException.class)); }
@Test public void testCheck() throws Exception { TestMongoDbHealthCheck healthCheck = setTpcheckMocks(); Mockito.when(commandResult.get(Matchers.anyString())).thenReturn(Integer.valueOf(1)); Result result = healthCheck.check(); Mockito.verify(loggerMock).debug("connectionUrl={} databaseList={} stats={}", TEST_CONNECT_URL, "", "commandResult"); Mockito.verify(mongoClientMock).close(); Assert.assertTrue(result.isHealthy()); }
@Test public void checkHappyPath() throws Exception { mockSetup(); Result result = healthCheck.check(); Assert.assertTrue(result.isHealthy()); Mockito.verify(connectionMock).close(); Mockito.verify(channelMock).close(); Mockito.verify(channelMock).queueDeclarePassive(TEST_QUEUE); }
@Test public void checkNonexistentQueue() throws Exception { mockSetup(); Mockito.when(channelMock.queueDeclarePassive(TEST_QUEUE)).thenThrow(TEST_EXCEPTION); Result result = healthCheck.check(); Assert.assertFalse(result.isHealthy()); Mockito.verify(loggerMock).error(Matchers.anyString(), Matchers.any(ContextedRuntimeException.class)); }
@Test public void returnsUnhealthyWhenCheckerThrowsException() throws Exception { when(checker.isSpaceAvailable()).thenThrow(new DiskSpaceCheckerException("Error")); Result result = healthCheck.check(); assertThat(result.isHealthy()).isFalse(); }
@Test public void returnsUnhealthyWhenCheckerFails() throws Exception { when(checker.isSpaceAvailable()).thenReturn(false); Result result = healthCheck.check(); assertThat(result.isHealthy()).isFalse(); }
@Test public void returnsHealthyWhenCheckerPasses() throws Exception { when(checker.isSpaceAvailable()).thenReturn(true); Result result = healthCheck.check(); assertThat(result.isHealthy()).isTrue(); }
@Test public void testCheckUnhealthy() throws Exception { doThrow(new ConsulException("error")).when(agent).ping(); final Result actual = healthCheck.check(); verify(agent).ping(); assertThat(actual.isHealthy()).isFalse(); }
@Test public void testHealthChecks() throws Exception { MetricRegistry metricRegistry = new MetricRegistry(); HealthCheckRegistry healthRegistry = new HealthCheckRegistry(); HikariConfig config = new HikariConfig(); config.setMaximumPoolSize(10); config.setMetricRegistry(metricRegistry); config.setHealthCheckRegistry(healthRegistry); config.setPoolName("test"); config.setDataSourceClassName("com.zaxxer.hikari.mocks.StubDataSource"); config.addHealthCheckProperty("connectivityCheckTimeoutMs", "1000"); config.addHealthCheckProperty("expected99thPercentileMs", "10"); HikariDataSource ds = new HikariDataSource(config); try { UtilityElf.quietlySleep(TimeUnit.SECONDS.toMillis(2)); Connection connection = ds.getConnection(); connection.close(); connection = ds.getConnection(); connection.close(); SortedMap<String, Result> healthChecks = healthRegistry.runHealthChecks(); Result connectivityResult = healthChecks.get("test.pool.ConnectivityCheck"); Assert.assertTrue(connectivityResult.isHealthy()); Result slaResult = healthChecks.get("test.pool.Connection99Percent"); Assert.assertTrue(slaResult.isHealthy()); } finally { ds.close(); } }
@Override protected Result check() throws Exception { final String saying = String.format(template, "TEST"); if (!saying.contains("TEST")) { return Result.unhealthy("template doesn't include a name"); } return Result.healthy(); }
@Test public void testCheckHealthyResult() throws Exception { HealthCheckTestTuple tuple = getTestStack(); MyriadDriverManager manager = tuple.getManager(); MesosDriverHealthCheck checker = tuple.getChecker(); manager.startDriver(); assertEquals(Result.healthy(), checker.check()); manager.stopDriver(false); }
@Test public void testCheckStoppedDriverUnhealthyResult() throws Exception { HealthCheckTestTuple tuple = getTestStack(); MyriadDriverManager manager = tuple.getManager(); MesosDriverHealthCheck checker = tuple.getChecker(); manager.startDriver(); manager.stopDriver(false); assertEquals(Result.unhealthy("Driver status: DRIVER_STOPPED"), checker.check()); }
@Test public void testCheckAbortedDriverUnhealthyResult() throws Exception { HealthCheckTestTuple tuple = getTestStack(); MyriadDriverManager manager = tuple.getManager(); MesosDriverHealthCheck checker = tuple.getChecker(); manager.startDriver(); manager.abortDriver(); assertEquals(Result.unhealthy("Driver status: DRIVER_ABORTED"), checker.check()); }
public Result healthCheck() { if (runnable == null) { return Result.unhealthy("Producer task not yet started"); } if (runnable.isCancelled()) { return Result.unhealthy("Producer task cancelled"); } if (runnable.isDone()) { return Result.unhealthy("Producer task not running any more"); } return Result.healthy(); }
/*** * Application Health Checks * @return response */ @Path(value="/check") @GET @Timed public Response check(){ final SortedMap<String, HealthCheck.Result> results = this.health.runHealthChecks(); for(Entry<String, Result> entryResult : results.entrySet()){ if(!entryResult.getValue().isHealthy()){ return Response.status(Status.INTERNAL_SERVER_ERROR).entity(entryResult.getValue().getMessage()).build(); } } return Response.status(Status.OK).entity("Healthy!").build(); }
@Override public void run(DashboardTestConfiguration configuration, Environment environment) throws Exception { // Generating HTTP Clients Client jerseyClient = new JerseyClientBuilder(environment).using(configuration.getJerseyClientConfiguration()) .build(getName()); // Configuring HealthChecks environment.healthChecks().register(getName(), new HealthCheck() { @Override protected Result check() throws Exception { return Result.healthy(); } }); }
@Override public void run() { StormClusterState clusterState = workerData.getZkCluster(); String topologyId = workerData.getTopologyId(); Map<Integer, HealthCheckRegistry> taskHealthCheckMap = JStormHealthCheck.getTaskhealthcheckmap(); int cnt = 0; for (Map.Entry<Integer, HealthCheckRegistry> entry : taskHealthCheckMap.entrySet()) { Integer taskId = entry.getKey(); Map<String, Result> results = entry.getValue().runHealthChecks(); for (Map.Entry<String, Result> result : results.entrySet()) { if (!result.getValue().isHealthy()) { try { clusterState.report_task_error(topologyId, taskId, result.getValue().getMessage(), ErrorConstants.WARN, ErrorConstants.CODE_QUEUE_FULL, ErrorConstants.DURATION_SECS_QUEUE_FULL); cnt++; } catch (Exception e) { LOG.error("Failed to update health data in ZK for topo-{} task-{}.", topologyId, taskId, e); } } } } if (cnt > 0) { LOG.info("Successfully updated {} health data to ZK for topology:{}", cnt, topologyId); } }
public static Result unhealthy(String message) { ResultBuilder builder = Result.builder(); builder.unhealthy().withMessage(message).withDetail("status", Status.ERROR); return builder.build(); }
public static Result healthy() { ResultBuilder builder = Result.builder(); builder.healthy().withDetail("status", Status.SUCCESS); return builder.build(); }
public static Result unknown() { ResultBuilder builder = Result.builder(); builder.healthy().withDetail("status", Status.UNKNOWN); return builder.build(); }
@Test public void checkFailure() throws Exception { healthcheck = new RabbitMQHealthCheck(connectionFactory, "NonExistentQueue"); Result result = healthcheck.check(); Assert.assertFalse(result.isHealthy()); }
@Test public void checkSuccess() throws Exception { healthcheck = new RabbitMQHealthCheck(connectionFactory, TEST_QUEUE); Result result = healthcheck.check(); Assert.assertTrue(result.isHealthy()); }
@Test public void testHealthCheckCreated() throws InterruptedException { assertEquals(2, RULE.getEnvironment().healthChecks().getNames().size()); assertEquals(Result.healthy("testValue"), RULE.getEnvironment().healthChecks().runHealthCheck("TestHealthCheck")); }
@Test public void testCheckHealthy() throws Exception { final Result actual = healthCheck.check(); verify(agent).ping(); assertThat(actual.isHealthy()).isTrue(); }
/** * Get all the failing healthchecks. * * @return A map of all the health checks that are unhealthy */ public Map<String, Result> getUnhealthy() { return HealthCheckRegistryFactory.getRegistry().runHealthChecks().entrySet().stream() .filter(entry -> !entry.getValue().isHealthy()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); }
@GET @Produces("application/json") public Map<String, Result> health() { return healthChecks.runHealthChecks(); }