/** * Collects Surefire test results * @param suites * @param packageNames * @return */ private Map<String, Boolean> prepareTestResults(Collection<SuiteResult> suites, Set<String> packageNames) { Map<String,Boolean> zephyrTestCaseMap = new HashMap<String, Boolean>(); for (Iterator<SuiteResult> iterator = suites.iterator(); iterator .hasNext();) { SuiteResult suiteResult = iterator.next(); List<CaseResult> cases = suiteResult.getCases(); for (CaseResult caseResult : cases) { boolean isPassed = caseResult.isPassed(); String name = caseResult.getFullName(); if (!zephyrTestCaseMap.containsKey(name)) { zephyrTestCaseMap.put(name, isPassed); packageNames.add(caseResult.getPackageName()); } } } return zephyrTestCaseMap; }
/** * Convert JUnit's TestResult representation into a generic UnitResults * @param jUnitResults The result of the JUnit run * @return The converted results */ public UnitResults convertJUnit(TestResult jUnitResults) { UnitResults results = new UnitResults(); if (jUnitResults == null) { return results; } for (SuiteResult sr : jUnitResults.getSuites()) { for (CaseResult cr : sr.getCases()) { UnitResult result = new UnitResult( cr.getClassName(), cr.getDisplayName(), cr.getErrorStackTrace(), cr.getDuration(), cr.getFailCount(), cr.getSkipCount(), cr.getPassCount() ); results.add(result); } } return results; }
public List<CaseResult> getQuarantinedTests() { ArrayList<CaseResult> list = new ArrayList<CaseResult>(); for (TopLevelItem item : getOwner().getItems()) { if (item instanceof Job) { Job job = (Job) item; Run lb = job.getLastBuild(); while (lb != null && (lb.hasntStartedYet() || lb.isBuilding())) lb = lb.getPreviousBuild(); if (lb != null && lb.getAction(TestResultAction.class) != null) { for (SuiteResult suite : lb.getAction(TestResultAction.class).getResult().getSuites()) { for (CaseResult test : suite.getCases()) { QuarantineTestAction action = test.getTestAction(QuarantineTestAction.class); if (action != null && action.isQuarantined()) { list.add(test); } } } } } } return list; }
public void testNoTestsHaveQuarantineActionForStandardPublisher() throws Exception { project.getPublishersList().remove(QuarantinableJUnitResultArchiver.class); DescribableList<TestDataPublisher, Descriptor<TestDataPublisher>> publishers = new DescribableList<TestDataPublisher, Descriptor<TestDataPublisher>>( project); publishers.add(new QuarantineTestDataPublisher()); project.getPublishersList().add(new JUnitResultArchiver("*.xml", false, publishers)); TestResult tr = getResultsFromJUnitResult("junit-1-failure.xml"); for (SuiteResult suite : tr.getSuites()) { for (CaseResult result : suite.getCases()) { assertNull(result.getTestAction(QuarantineTestAction.class)); } } }
public int getFailedTestCount(SuiteResult suiteResult) {//work! int failedTestCount = 0; List<CaseResult> cases = suiteResult.getCases(); for (CaseResult caseResult : cases) { if (!caseResult.isPassed() && !caseResult.isSkipped()) { // Any error, invalidates the suite result failedTestCount++; } } return failedTestCount; }
@Override public TestResultAction.Data contributeTestData(Run<?, ?> run, @Nonnull FilePath workspace, Launcher launcher, TaskListener listener, TestResult testResult) throws IOException, InterruptedException { log(listener.getLogger(), "Publishing test results"); Map<String, String> testSessionMap = workspace.act(new BrowserStackReportFileCallable(REPORT_FILE_PATTERN, run.getTimeInMillis())); AutomateActionData automateActionData = new AutomateActionData(); Map<String, Long> testCaseIndices = new HashMap<String, Long>(); int testCount = 0; int sessionCount = 0; for (SuiteResult suiteResult : testResult.getSuites()) { List<CaseResult> cases = suiteResult.getCases(); testCount += cases.size(); logDebug(listener.getLogger(), suiteResult.getName() + ": " + cases.size() + " test cases found."); for (CaseResult caseResult : cases) { String testCaseName = getTestCaseName(caseResult); Long testIndex = testCaseIndices.containsKey(testCaseName) ? testCaseIndices.get(testCaseName) : -1L; testCaseIndices.put(testCaseName, ++testIndex); logDebug(listener.getLogger(), testCaseName + " / " + testCaseName + " <=> " + testIndex); String testId = String.format("%s{%d}", testCaseName, testIndex); if (testSessionMap.containsKey(testId)) { AutomateTestAction automateTestAction = new AutomateTestAction(run, caseResult, testSessionMap.get(testId)); automateActionData.registerTestAction(caseResult.getId(), automateTestAction); logDebug(listener.getLogger(), "registerTestAction: " + testId + " => " + automateTestAction); sessionCount++; } } } testCaseIndices.clear(); log(listener.getLogger(), testCount + " tests recorded"); log(listener.getLogger(), sessionCount + " sessions captured"); log(listener.getLogger(), "Publishing test results: SUCCESS"); return automateActionData; }
@Test public void testThatReportsArePresent() throws Exception { /* =================== Prepare ================= */ new MockAutomateClient(); addBuildStep(); project.getBuildersList().add(new TouchBuilder()); project.getBuildersList().add(new CopyResourceFileToWorkspaceTarget("browserstack-reports", "REPORT-com.browserstack.automate.application.tests.TestCaseWithFourUniqueSessions.xml")); project.getBuildersList().add(new CopyResourceFileToWorkspaceTarget("surefire-reports", "TEST-com.browserstack.automate.application.tests.TestCaseWithFourUniqueSessions.xml")); JUnitResultArchiver jUnitResultArchiver = new JUnitResultArchiver("**/surefire-reports/*.xml"); jUnitResultArchiver.setTestDataPublishers(Collections.singletonList(new AutomateTestDataPublisher())); project.getPublishersList().add(jUnitResultArchiver); /* =================== Execute ================= */ FreeStyleBuild build = project.scheduleBuild2(0).get(); TestResultAction action = build.getAction(TestResultAction.class); /* =================== Verify ================= */ List<TestResultAction.Data> testData = Deencapsulation.getField(action, "testData"); // Assumption here is that there is only one suite result based on the XML files // copied into the workspace above. SuiteResult suiteResult = action.getResult().getSuites().iterator().next(); jenkinsRule.assertBuildStatus(Result.SUCCESS, build); Assert.assertNotNull("Additional data on the test MUST be set.", testData); Assert.assertTrue("Additional Test data MUST have values.", testData.size() >= 1); TestResultAction.Data data = testData.get(0); Assert.assertTrue("Additional Test data MUST be an instance of AutomateActionData.", data instanceof AutomateActionData); for (CaseResult caseResult : suiteResult.getCases()) { AutomateTestAction automateTestAction = (AutomateTestAction) data.getTestAction(caseResult).get(0); Session automateSession = automateTestAction.getSession(); Assert.assertNotNull("Automate Session should not be null.", automateSession); Assert.assertTrue("Session Id should not be null or empty.", StringUtils.isNotEmpty(automateSession.getId())); } }
public void testAllTestsHaveQuarantineAction() throws Exception { TestResult tr = getResultsFromJUnitResult("junit-1-failure.xml"); for (SuiteResult suite : tr.getSuites()) { for (CaseResult result : suite.getCases()) { assertNotNull(result.getTestAction(QuarantineTestAction.class)); } } }
@Override public String evaluate(Run<?, ?> run, FilePath filePath, TaskListener listener, String testResultFile) throws MacroEvaluationException, IOException, InterruptedException { StringBuilder buffer = new StringBuilder(); TestResult testResult = new TestResult(); File junitFile = new File(testResultFile); testResult.parse(junitFile); SuiteResult suite = testResult.getSuites().iterator().next(); int failCount = getFailedTestCount(suite); if (null == testResult) { return "No tests ran."; } if (failCount == 0) { buffer.append("All tests passed"); } else { buffer.append(failCount).append(" tests failed.\n"); boolean showOldFailures = !onlyRegressions; // if(maxLength < Integer.MAX_VALUE) { // maxLength *= 1024; // } if (maxTests > 0) { int printedTests = 0; int printedLength = 0; List<CaseResult> cases = suite.getCases(); for (CaseResult caseResult : cases) { if (!caseResult.isPassed() && !caseResult.isSkipped() && caseResult.getErrorDetails() != null) { // Any error, invalidates the suite result if (showOldFailures || getTestAge(caseResult) == 1) { if (printedTests < maxTests && printedLength <= maxLength) { printedLength += outputTest(buffer, caseResult, showStack, showMessage, maxLength - printedLength); printedTests++; } } } } if (failCount > printedTests) { buffer.append("... and ").append(failCount - printedTests).append(" other failed tests.\n\n"); } if (printedLength >= maxLength) { buffer.append("\n\n... output truncated.\n\n"); } } } return buffer.toString(); }
/** * Collects the surefire results and prepares Zephyr Tests * @param build * @param zephyrConfig * @return */ private boolean prepareZephyrTests(final AbstractBuild build, ZephyrConfigModel zephyrConfig) { boolean status = true; TestResultAction testResultAction = build.getAction(TestResultAction.class); Collection<SuiteResult> suites = null; try { suites = testResultAction.getResult().getSuites(); } catch (Exception e) { logger.println(e.getMessage()); return false; } if (suites == null || suites.size() == 0) { return false; } Set<String> packageNames = new HashSet<String>(); Map<String, Boolean> zephyrTestCaseMap = prepareTestResults(suites, packageNames); logger.print("Total Test Cases : " + zephyrTestCaseMap.size()); List<TestCaseResultModel> testcases = new ArrayList<TestCaseResultModel>(); Set<String> keySet = zephyrTestCaseMap.keySet(); for (Iterator<String> iterator = keySet.iterator(); iterator.hasNext();) { String testCaseName = iterator.next(); Boolean isPassed = zephyrTestCaseMap.get(testCaseName); RemoteTestcase testcase = new RemoteTestcase(); testcase.setName(testCaseName); testcase.setComments(TEST_CASE_COMMENT); testcase.setAutomated(AUTOMATED); testcase.setExternalId(EXTERNAL_ID); testcase.setPriority(TEST_CASE_PRIORITY); testcase.setTag(TEST_CASE_TAG); TestCaseResultModel caseWithStatus = new TestCaseResultModel(); caseWithStatus.setPassed(isPassed); caseWithStatus.setRemoteTestcase(testcase); testcases.add(caseWithStatus); } zephyrConfig.setTestcases(testcases); zephyrConfig.setPackageNames(packageNames); zephyrConfig.setCreatePackage(createPackage); return status; }
@Override public Data getTestData(AbstractBuild<?, ?> build, Launcher launcher, BuildListener listener, TestResult testResult) { Data data = new Data(build); MailNotifier notifier = new MailNotifier(listener); for (SuiteResult suite : testResult.getSuites()) { for (CaseResult result : suite.getCases()) { QuarantineTestAction previousAction = null; CaseResult previous = result.getPreviousResult(); AbstractBuild<?, ?> previousBuild = build.getPreviousCompletedBuild(); if (previous != null) { previousAction = previous.getTestAction(QuarantineTestAction.class); } // no immediate predecessor (e.g. because job failed or did not // run), try and go back in build history while (previous == null && previousBuild != null) { if (previousBuild.getAction(AbstractTestResultAction.class) != null) { hudson.tasks.test.TestResult tr = null; try { tr = previousBuild.getAction(AbstractTestResultAction.class).findCorrespondingResult( result.getId()); } catch (Exception e){ listener.getLogger().println("could not find result for id " + result.getId() + " in build " + previousBuild + ": " + e.getMessage()); } if (tr != null) { listener.getLogger().println("found " + tr.getDisplayName() + " in build " + previousBuild); previousAction = tr.getTestAction(QuarantineTestAction.class); break; } } else { listener.getLogger().println("build " + previousBuild + " does not have test results"); } previousBuild = previousBuild.getPreviousCompletedBuild(); } if (previousAction != null && previousAction.isQuarantined()) { QuarantineTestAction action = new QuarantineTestAction(data, result.getId()); action.quarantine(previousAction); data.addQuarantine(result.getId(), action); // send email if failed if (!result.isPassed()) { notifier.addResult(result, action); } } } } notifier.sendEmails(); return data; }