@Test public void junitResultArchiver() throws Exception { WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); p.setDefinition(new CpsFlowDefinition( "node {\n" + " writeFile text: '''<testsuite name='a'><testcase name='a1'/><testcase name='a2'><error>a2 failed</error></testcase></testsuite>''', file: 'a.xml'\n" + " writeFile text: '''<testsuite name='b'><testcase name='b1'/><testcase name='b2'/></testsuite>''', file: 'b.xml'\n" + " junit '*.xml'\n" + "}", true)); WorkflowRun b = r.assertBuildStatus(Result.UNSTABLE, p.scheduleBuild2(0).get()); TestResultAction a = b.getAction(TestResultAction.class); assertNotNull(a); assertEquals(4, a.getTotalCount()); assertEquals(1, a.getFailCount()); List<FlowNode> coreStepNodes = new DepthFirstScanner().filteredNodes(b.getExecution(), new NodeStepTypePredicate("step")); assertThat(coreStepNodes, Matchers.hasSize(1)); assertEquals("*.xml", ArgumentsAction.getStepArgumentsAsString(coreStepNodes.get(0))); }
public static int reportTests(BuildContext context, String filePattern) throws IOException, InterruptedException { context.listener.getLogger().println("Recording test results matching: '" + filePattern + "'"); final Result resultBefore = context.build.getResult(); JUnitResultArchiver task = new JUnitResultArchiver(filePattern, true, null, 1.0); task.perform(context.build, context.launcher, context.listener); final Result resultAfter = context.build.getResult(); final int testCount; TestResultAction testResultAction = context.build.getAction(TestResultAction.class); if (testResultAction != null) { testCount = testResultAction.getTotalCount(); } else { testCount = -1; } context.listener.getLogger().println(" Test results reported " + testCount + " tests. Status: " + resultAfter); if (isResultWorse(resultBefore, resultAfter)) { context.listener.getLogger().println(" Test results changed build status to " + resultAfter); } return testCount; }
@Override public void onCompleted(AbstractBuild build, TaskListener listener) { // TODO consider the possibility that there is >1 such action TestResultAction testResultAction = build.getAction(TestResultAction.class); HistoryAggregatedFlakyTestResultAction historyAction = build.getProject() .getAction(HistoryAggregatedFlakyTestResultAction.class); // Aggregate test running results if (historyAction != null) { historyAction.aggregateOneBuild(build); } if (testResultAction != null && testResultAction.getFailCount() > 0) { // Only add deflake action if there are test failures build.addAction( new DeflakeAction(getFailingTestClassMethodMap(testResultAction.getFailedTests()))); } }
public static JobResults build(final Job job) { JobResults jobResults = new JobResults(); final RunList<?> jobs = job.getBuilds(); int jobCount = 0; for (Run<?,?> jobRun : jobs) { if (jobCount > 9) break; if (jobRun.isBuilding()) continue; TestResultAction testResults = jobRun.getAction(TestResultAction.class); if (testResults == null) continue; jobCount++; final Collection<PackageResult> packageResults = testResults.getResult().getChildren(); for (PackageResult packageResult : packageResults) { final Collection<ClassResult> classResults = packageResult.getChildren(); for (ClassResult classResult : classResults) { jobResults.addResult(jobRun, classResult); } } } return jobResults; }
public List<CaseResult> getQuarantinedTests() { ArrayList<CaseResult> list = new ArrayList<CaseResult>(); for (TopLevelItem item : getOwner().getItems()) { if (item instanceof Job) { Job job = (Job) item; Run lb = job.getLastBuild(); while (lb != null && (lb.hasntStartedYet() || lb.isBuilding())) lb = lb.getPreviousBuild(); if (lb != null && lb.getAction(TestResultAction.class) != null) { for (SuiteResult suite : lb.getAction(TestResultAction.class).getResult().getSuites()) { for (CaseResult test : suite.getCases()) { QuarantineTestAction action = test.getTestAction(QuarantineTestAction.class); if (action != null && action.isQuarantined()) { list.add(test); } } } } } } return list; }
public void testQuarantiningMakesFinalResultFailIfQuarantineReleased() throws Exception { FreeStyleBuild build = runBuildWithJUnitResult("junit-1-failure.xml"); assertTrue(build.getResult() != Result.SUCCESS); TestResult tr = build.getAction(TestResultAction.class).getResult(); QuarantineTestAction action = tr.getSuite("SuiteA").getCase("TestB").getTestAction(QuarantineTestAction.class); action.quarantine("user1", "reason"); build = runBuildWithJUnitResult("junit-1-failure.xml"); assertTrue(build.getResult() == Result.SUCCESS); tr = build.getAction(TestResultAction.class).getResult(); action = tr.getSuite("SuiteA").getCase("TestB").getTestAction(QuarantineTestAction.class); action.release(); build = runBuildWithJUnitResult("junit-1-failure.xml"); System.out.println("result is " + build.getResult()); assertTrue(build.getResult() != Result.SUCCESS); }
@Override public TestResultAction.Data contributeTestData(Run<?, ?> run, @Nonnull FilePath workspace, Launcher launcher, TaskListener listener, TestResult testResult) throws IOException, InterruptedException { log(listener.getLogger(), "Publishing test results"); Map<String, String> testSessionMap = workspace.act(new BrowserStackReportFileCallable(REPORT_FILE_PATTERN, run.getTimeInMillis())); AutomateActionData automateActionData = new AutomateActionData(); Map<String, Long> testCaseIndices = new HashMap<String, Long>(); int testCount = 0; int sessionCount = 0; for (SuiteResult suiteResult : testResult.getSuites()) { List<CaseResult> cases = suiteResult.getCases(); testCount += cases.size(); logDebug(listener.getLogger(), suiteResult.getName() + ": " + cases.size() + " test cases found."); for (CaseResult caseResult : cases) { String testCaseName = getTestCaseName(caseResult); Long testIndex = testCaseIndices.containsKey(testCaseName) ? testCaseIndices.get(testCaseName) : -1L; testCaseIndices.put(testCaseName, ++testIndex); logDebug(listener.getLogger(), testCaseName + " / " + testCaseName + " <=> " + testIndex); String testId = String.format("%s{%d}", testCaseName, testIndex); if (testSessionMap.containsKey(testId)) { AutomateTestAction automateTestAction = new AutomateTestAction(run, caseResult, testSessionMap.get(testId)); automateActionData.registerTestAction(caseResult.getId(), automateTestAction); logDebug(listener.getLogger(), "registerTestAction: " + testId + " => " + automateTestAction); sessionCount++; } } } testCaseIndices.clear(); log(listener.getLogger(), testCount + " tests recorded"); log(listener.getLogger(), sessionCount + " sessions captured"); log(listener.getLogger(), "Publishing test results: SUCCESS"); return automateActionData; }
@Test public void testThatReportsArePresent() throws Exception { /* =================== Prepare ================= */ new MockAutomateClient(); addBuildStep(); project.getBuildersList().add(new TouchBuilder()); project.getBuildersList().add(new CopyResourceFileToWorkspaceTarget("browserstack-reports", "REPORT-com.browserstack.automate.application.tests.TestCaseWithFourUniqueSessions.xml")); project.getBuildersList().add(new CopyResourceFileToWorkspaceTarget("surefire-reports", "TEST-com.browserstack.automate.application.tests.TestCaseWithFourUniqueSessions.xml")); JUnitResultArchiver jUnitResultArchiver = new JUnitResultArchiver("**/surefire-reports/*.xml"); jUnitResultArchiver.setTestDataPublishers(Collections.singletonList(new AutomateTestDataPublisher())); project.getPublishersList().add(jUnitResultArchiver); /* =================== Execute ================= */ FreeStyleBuild build = project.scheduleBuild2(0).get(); TestResultAction action = build.getAction(TestResultAction.class); /* =================== Verify ================= */ List<TestResultAction.Data> testData = Deencapsulation.getField(action, "testData"); // Assumption here is that there is only one suite result based on the XML files // copied into the workspace above. SuiteResult suiteResult = action.getResult().getSuites().iterator().next(); jenkinsRule.assertBuildStatus(Result.SUCCESS, build); Assert.assertNotNull("Additional data on the test MUST be set.", testData); Assert.assertTrue("Additional Test data MUST have values.", testData.size() >= 1); TestResultAction.Data data = testData.get(0); Assert.assertTrue("Additional Test data MUST be an instance of AutomateActionData.", data instanceof AutomateActionData); for (CaseResult caseResult : suiteResult.getCases()) { AutomateTestAction automateTestAction = (AutomateTestAction) data.getTestAction(caseResult).get(0); Session automateSession = automateTestAction.getSession(); Assert.assertNotNull("Automate Session should not be null.", automateSession); Assert.assertTrue("Session Id should not be null or empty.", StringUtils.isNotEmpty(automateSession.getId())); } }
private TestResult getJUnitResults() { AbstractBuild build = getBuild(); TestResultAction jUnitAction = build.getAction(TestResultAction.class); if (jUnitAction == null) { return null; } return jUnitAction.getResult(); }
public String getOneLineTestResults() { TestResultAction testResultAction = build.getAction(TestResultAction.class); if (testResultAction == null) { return "No test results found."; } return String.format("%d tests run, %d skipped, %d failed.", testResultAction.getTotalCount(), testResultAction.getSkipCount(), testResultAction.getFailCount()); }
@Override public String getTextResult(Run<?, ?> run) { List<TestResultAction> actions = run.getActions(TestResultAction.class); StringBuilder builder = new StringBuilder(); for (TestResultAction action : actions) { List<CaseResult> failedTests = action.getFailedTests(); for (CaseResult result : failedTests) { builder.append(result.getTitle() + "\n"); builder.append(result.getErrorDetails() + "\n"); } } return builder.toString(); }
@Override public TestResultAction.Data getTestData(AbstractBuild<?, ?> abstractBuild, Launcher launcher, BuildListener buildListener, TestResult testResult) throws IOException, InterruptedException { FlakyTestResult flakyTestResult = launcher.getChannel().call(new FlakyTestResultCollector(testResult)); // TODO consider the possibility that there is >1 such action flakyTestResult.freeze(abstractBuild.getAction(AbstractTestResultAction.class), abstractBuild); return new JUnitFlakyTestData(flakyTestResult); }
@Test public void testGetTestResultFreeStyleBuild() { FreeStyleBuild build = mock(FreeStyleBuild.class); TestResultAction action = mock(TestResultAction.class); when(build.getAction(AbstractTestResultAction.class)).thenReturn(action); List<TestResult> result = TestResult.getResults(build); assertNotNull(result); assertEquals(1, result.size()); }
public hudson.tasks.test.TestResult getResultForTestId(String testObjectId) { TestResultAction action = build.getAction(TestResultAction.class); if (action != null && action.getResult() != null) { return action.getResult().findCorrespondingResult(testObjectId); } return null; }
public void testTextSummaryForUnquarantinedTestAuthenticated() throws Exception { FreeStyleBuild build = runBuildWithJUnitResult("junit-1-failure.xml"); TestResult tr = build.getAction(TestResultAction.class).getResult(); HtmlPage page = whenNavigatingToTestCase(tr.getSuite("SuiteA").getCase("TestA"), true); assertTrue(pageShowsText(page, "This test was not quarantined. Quarantine it.")); }
public void testTextSummaryForUnquarantinedTestNotAuthenticated() throws Exception { FreeStyleBuild build = runBuildWithJUnitResult("junit-1-failure.xml"); TestResult tr = build.getAction(TestResultAction.class).getResult(); HtmlPage page = whenNavigatingToTestCase(tr.getSuite("SuiteA").getCase("TestA"), false); assertTrue(pageShowsText(page, "This test was not quarantined.")); assertFalse(pageShowsText(page, "Quarantine it.")); }
public void testWhenQuarantiningTestSaysQuarantinedBy() throws Exception { FreeStyleBuild build = runBuildWithJUnitResult("junit-1-failure.xml"); TestResult tr = build.getAction(TestResultAction.class).getResult(); HtmlPage page = whenNavigatingToTestCase(tr.getSuite("SuiteA").getCase("TestA"), true); whenQuarantiningTestOnPage(page); page = whenNavigatingToTestCase(tr.getSuite("SuiteA").getCase("TestA"), false); assertTrue(pageShowsText(page, "This test was quarantined by user1")); }
public void testResultIsOnlyMarkedAsLatestIfLatest() throws Exception { FreeStyleBuild build = runBuildWithJUnitResult("junit-1-failure.xml"); TestResult tr1 = build.getAction(TestResultAction.class).getResult(); QuarantineTestAction action1 = tr1.getSuite("SuiteA").getCase("TestB").getTestAction(QuarantineTestAction.class); assertTrue(action1.isLatestResult()); build = runBuildWithJUnitResult("junit-1-failure.xml"); TestResult tr2 = build.getAction(TestResultAction.class).getResult(); QuarantineTestAction action2 = tr2.getSuite("SuiteA").getCase("TestB").getTestAction(QuarantineTestAction.class); assertFalse(action1.isLatestResult()); assertTrue(action2.isLatestResult()); }
public void testQuarantiningMakesFinalResultPass() throws Exception { FreeStyleBuild build = runBuildWithJUnitResult("junit-1-failure.xml"); assertTrue(build.getResult() != Result.SUCCESS); TestResult tr = build.getAction(TestResultAction.class).getResult(); QuarantineTestAction action = tr.getSuite("SuiteA").getCase("TestB").getTestAction(QuarantineTestAction.class); action.quarantine("user1", "reason"); build = runBuildWithJUnitResult("junit-1-failure.xml"); assertTrue(build.getResult() == Result.SUCCESS); }
public void testQuarantiningMakesFinalResultFailIfAnotherTestFails() throws Exception { FreeStyleBuild build = runBuildWithJUnitResult("junit-1-failure.xml"); assertTrue(build.getResult() != Result.SUCCESS); TestResult tr = build.getAction(TestResultAction.class).getResult(); QuarantineTestAction action = tr.getSuite("SuiteA").getCase("TestB").getTestAction(QuarantineTestAction.class); action.quarantine("user1", "reason"); build = runBuildWithJUnitResult("junit-2-failures.xml"); assertTrue(build.getResult() != Result.SUCCESS); }
public void testQuarantineStatusNotLostIfTestNotRun() throws Exception { FreeStyleBuild build = runBuildWithJUnitResult("junit-1-failure.xml"); assertTrue(build.getResult() != Result.SUCCESS); TestResult tr = build.getAction(TestResultAction.class).getResult(); QuarantineTestAction action = tr.getSuite("SuiteA").getCase("TestB").getTestAction(QuarantineTestAction.class); action.quarantine("user1", "reason"); build = runBuildWithJUnitResult("junit-1-failure-missing.xml"); assertTrue(build.getResult() == Result.SUCCESS); build = runBuildWithJUnitResult("junit-1-failure.xml"); assertTrue(build.getResult() == Result.SUCCESS); }
@Override public TestResultAction.Data getTestData(AbstractBuild<?, ?> abstractBuild, Launcher launcher, BuildListener buildListener, TestResult testResult) throws IOException, InterruptedException { return contributeTestData(abstractBuild, abstractBuild.getWorkspace(), launcher, buildListener, testResult); }
@Test public void maven_build_jar_project_on_master_succeeds() throws Exception { loadMavenJarProjectInGitRepo(this.gitRepoRule); String pipelineScript = "node('master') {\n" + " git($/" + gitRepoRule.toString() + "/$)\n" + " withMaven() {\n" + " sh 'mvn package verify'\n" + " }\n" + "}"; WorkflowJob pipeline = jenkinsRule.createProject(WorkflowJob.class, "build-on-master"); pipeline.setDefinition(new CpsFlowDefinition(pipelineScript, true)); WorkflowRun build = jenkinsRule.assertBuildStatus(Result.SUCCESS, pipeline.scheduleBuild2(0)); // verify Maven installation provided by the build agent is used // can be either "by the build agent with executable..." or "by the build agent with the environment variable MAVEN_HOME=..." jenkinsRule.assertLogContains("[withMaven] use Maven installation provided by the build agent with", build); // verify .pom is archived and fingerprinted // "[withMaven] Archive ... under jenkins/mvn/test/mono-module-maven-app/0.1-SNAPSHOT/mono-module-maven-app-0.1-SNAPSHOT.pom" jenkinsRule.assertLogContains("under jenkins/mvn/test/mono-module-maven-app/0.1-SNAPSHOT/mono-module-maven-app-0.1-SNAPSHOT.pom", build); // verify .jar is archived and fingerprinted jenkinsRule.assertLogContains("under jenkins/mvn/test/mono-module-maven-app/0.1-SNAPSHOT/mono-module-maven-app-0.1-SNAPSHOT.jar", build); Collection<String> artifactsFileNames = TestUtils.artifactsToArtifactsFileNames(build.getArtifacts()); assertThat(artifactsFileNames, hasItems("mono-module-maven-app-0.1-SNAPSHOT.pom", "mono-module-maven-app-0.1-SNAPSHOT.jar")); verifyFileIsFingerPrinted(pipeline, build, "jenkins/mvn/test/mono-module-maven-app/0.1-SNAPSHOT/mono-module-maven-app-0.1-SNAPSHOT.jar"); verifyFileIsFingerPrinted(pipeline, build, "jenkins/mvn/test/mono-module-maven-app/0.1-SNAPSHOT/mono-module-maven-app-0.1-SNAPSHOT.pom"); // verify Junit Archiver is called for maven-surefire-plugin jenkinsRule.assertLogContains("[withMaven] junitPublisher - Archive test results for Maven artifact jenkins.mvn.test:mono-module-maven-app:0.1-SNAPSHOT " + "generated by maven-surefire-plugin:test", build); TestResultAction testResultAction = build.getAction(TestResultAction.class); assertThat(testResultAction.getTotalCount(), is(3)); assertThat(testResultAction.getFailCount(), is(0)); // verify Junit Archiver is called for maven-failsafe-plugin jenkinsRule.assertLogContains("[withMaven] junitPublisher - Archive test results for Maven artifact jenkins.mvn.test:mono-module-maven-app:0.1-SNAPSHOT " + "generated by maven-failsafe-plugin:integration-test", build); // verify Task Scanner is called for jenkins.mvn.test:mono-module-maven-app jenkinsRule.assertLogContains("[withMaven] openTasksPublisher - Scan Tasks for Maven artifact jenkins.mvn.test:mono-module-maven-app:0.1-SNAPSHOT", build); TasksResultAction tasksResultAction = build.getAction(TasksResultAction.class); assertThat(tasksResultAction.getProjectActions().size(), is(1)); }
/** * Collects the surefire results and prepares Zephyr Tests * @param build * @param zephyrConfig * @return */ private boolean prepareZephyrTests(final AbstractBuild build, ZephyrConfigModel zephyrConfig) { boolean status = true; TestResultAction testResultAction = build.getAction(TestResultAction.class); Collection<SuiteResult> suites = null; try { suites = testResultAction.getResult().getSuites(); } catch (Exception e) { logger.println(e.getMessage()); return false; } if (suites == null || suites.size() == 0) { return false; } Set<String> packageNames = new HashSet<String>(); Map<String, Boolean> zephyrTestCaseMap = prepareTestResults(suites, packageNames); logger.print("Total Test Cases : " + zephyrTestCaseMap.size()); List<TestCaseResultModel> testcases = new ArrayList<TestCaseResultModel>(); Set<String> keySet = zephyrTestCaseMap.keySet(); for (Iterator<String> iterator = keySet.iterator(); iterator.hasNext();) { String testCaseName = iterator.next(); Boolean isPassed = zephyrTestCaseMap.get(testCaseName); RemoteTestcase testcase = new RemoteTestcase(); testcase.setName(testCaseName); testcase.setComments(TEST_CASE_COMMENT); testcase.setAutomated(AUTOMATED); testcase.setExternalId(EXTERNAL_ID); testcase.setPriority(TEST_CASE_PRIORITY); testcase.setTag(TEST_CASE_TAG); TestCaseResultModel caseWithStatus = new TestCaseResultModel(); caseWithStatus.setPassed(isPassed); caseWithStatus.setRemoteTestcase(testcase); testcases.add(caseWithStatus); } zephyrConfig.setTestcases(testcases); zephyrConfig.setPackageNames(packageNames); zephyrConfig.setCreatePackage(createPackage); return status; }
protected TestResult getResultsFromJUnitResult(final String xmlFileName) throws Exception { return runBuildWithJUnitResult(xmlFileName).getAction(TestResultAction.class).getResult(); }