Stop attaching test reruns to final test report entries (#387)

* treat rerun entries as seperate test runs

* Add changelog entry

* fix a typo

* remove debug code

* fix flaky test on mac

Author: gnikonorov

CHANGES.rst

@@ -3,6 +3,10 @@ Release Notes
 
 **3.1.0 (unreleased)**
 
+* Stop attaching test reruns to final test report entries (`#374 <https://github.com/pytest-dev/pytest-html/issues/374>`_)
+
+  * Thanks to `@VladimirPodolyan <https://github.com/VladimirPodolyan>`_ for reporting and `@gnikonorov <https://github.com/gnikonorov>`_ for the fix
+
 * Allow for report duration formatting (`#376 <https://github.com/pytest-dev/pytest-html/issues/376>`_)
 
   * Thanks to `@brettnolan <https://github.com/brettnolan>`_ for reporting and `@gnikonorov <https://github.com/gnikonorov>`_ for the fix

src/pytest_html/plugin.py

@@ -157,7 +157,7 @@ def __init__(self, outcome, report, logfile, config):
             if getattr(report, "when", "call") != "call":
                 self.test_id = "::".join([report.nodeid, report.when])
             self.time = getattr(report, "duration", 0.0)
-            self.formatted_time = getattr(report, "formatted_duration", 0.0)
+            self.formatted_time = self._format_time(report)
             self.outcome = outcome
             self.additional_html = []
             self.links_html = []
@@ -283,6 +283,37 @@ def append_extra_html(self, extra, extra_index, test_index):
                 )
                 self.links_html.append(" ")
 
+        def _format_time(self, report):
+            # parse the report duration into its display version and return
+            # it to the caller
+            duration = getattr(report, "duration", None)
+            if duration is None:
+                return ""
+
+            duration_formatter = getattr(report, "duration_formatter", None)
+            string_duration = str(duration)
+            if duration_formatter is None:
+                if "." in string_duration:
+                    split_duration = string_duration.split(".")
+                    split_duration[1] = split_duration[1][0:2]
+
+                    string_duration = ".".join(split_duration)
+
+                return string_duration
+            else:
+                # support %f, since time.strftime doesn't support it out of the box
+                # keep a precision of 2 for legacy reasons
+                formatted_milliseconds = "00"
+                if "." in string_duration:
+                    milliseconds = string_duration.split(".")[1]
+                    formatted_milliseconds = milliseconds[0:2]
+
+                duration_formatter = duration_formatter.replace(
+                    "%f", formatted_milliseconds
+                )
+                duration_as_gmtime = time.gmtime(report.duration)
+                return time.strftime(duration_formatter, duration_as_gmtime)
+
         def _populate_html_log_div(self, log, report):
             if report.longrepr:
                 # longreprtext is only filled out on failure by pytest
@@ -425,6 +456,10 @@ def append_failed(self, report):
             self.errors += 1
             self._appendrow("Error", report)
 
+    def append_rerun(self, report):
+        self.rerun += 1
+        self._appendrow("Rerun", report)
+
     def append_skipped(self, report):
         if hasattr(report, "wasxfail"):
             self.xfailed += 1
@@ -433,11 +468,6 @@ def append_skipped(self, report):
             self.skipped += 1
             self._appendrow("Skipped", report)
 
-    def append_other(self, report):
-        # For now, the only "other" the plugin give support is rerun
-        self.rerun += 1
-        self._appendrow("Rerun", report)
-
     def _generate_report(self, session):
         suite_stop_time = time.time()
         suite_time_delta = suite_stop_time - self.suite_start_time
@@ -604,32 +634,6 @@ def generate_summary_item(self):
         unicode_doc = unicode_doc.encode("utf-8", errors="xmlcharrefreplace")
         return unicode_doc.decode("utf-8")
 
-    def _format_duration(self, report):
-        # parse the report duration into its display version and return it to the caller
-        duration_formatter = getattr(report, "duration_formatter", None)
-        string_duration = str(report.duration)
-        if duration_formatter is None:
-            if "." in string_duration:
-                split_duration = string_duration.split(".")
-                split_duration[1] = split_duration[1][0:2]
-
-                string_duration = ".".join(split_duration)
-
-            return string_duration
-        else:
-            # support %f, since time.strftime doesn't support it out of the box
-            # keep a precision of 2 for legacy reasons
-            formatted_milliseconds = "00"
-            if "." in string_duration:
-                milliseconds = string_duration.split(".")[1]
-                formatted_milliseconds = milliseconds[0:2]
-
-            duration_formatter = duration_formatter.replace(
-                "%f", formatted_milliseconds
-            )
-            duration_as_gmtime = time.gmtime(report.duration)
-            return time.strftime(duration_formatter, duration_as_gmtime)
-
     def _generate_environment(self, config):
         if not hasattr(config, "_metadata") or config._metadata is None:
             return []
@@ -685,22 +689,23 @@ def _post_process_reports(self):
             #  through them all to figure out the outcome, xfail, duration,
             #    extras, and when it swapped from pass
             for test_report in test_reports:
-                full_text += test_report.longreprtext
-                extras.extend(getattr(test_report, "extra", []))
-                duration += getattr(test_report, "duration", 0.0)
+                if test_report.outcome == "rerun":
+                    # reruns are separate test runs for all intensive purposes
+                    self.append_rerun(test_report)
+                else:
+                    full_text += test_report.longreprtext
+                    extras.extend(getattr(test_report, "extra", []))
+                    duration += getattr(test_report, "duration", 0.0)
 
-                if (
-                    test_report.outcome not in ("passed", "rerun")
-                    and outcome == "passed"
-                ):
-                    outcome = test_report.outcome
-                    failure_when = test_report.when
+                    if (
+                        test_report.outcome not in ("passed", "rerun")
+                        and outcome == "passed"
+                    ):
+                        outcome = test_report.outcome
+                        failure_when = test_report.when
 
-                if hasattr(test_report, "wasxfail"):
-                    wasxfail = True
-
-                if test_report.outcome == "rerun":
-                    self.append_other(test_report)
+                    if hasattr(test_report, "wasxfail"):
+                        wasxfail = True
 
             # the following test_report.<X> = settings come at the end of us
             #  looping through all test_reports that make up a single
@@ -715,7 +720,6 @@ def _post_process_reports(self):
             test_report.longrepr = full_text
             test_report.extra = extras
             test_report.duration = duration
-            test_report.formatted_duration = self._format_duration(test_report)
 
             if wasxfail:
                 test_report.wasxfail = True
@@ -728,9 +732,6 @@ def _post_process_reports(self):
                 test_report.when = failure_when
                 self.append_failed(test_report)
 
-            # we don't append other here since the only case supported
-            #  for append_other is rerun, which is handled in the loop above
-
     def pytest_runtest_logreport(self, report):
         self.reports[report.nodeid].append(report)
 

testing/test_pytest_html.py

@@ -187,17 +187,47 @@ def test_fail(self, testdir):
         assert "AssertionError" in html
 
     def test_rerun(self, testdir):
+        testdir.makeconftest(
+            """
+            import pytest
+
+            @pytest.hookimpl(hookwrapper=True)
+            def pytest_runtest_makereport(item, call):
+                pytest_html = item.config.pluginmanager.getplugin("html")
+                outcome = yield
+                report = outcome.get_result()
+
+                extra = getattr(report, "extra", [])
+                if report.when == "call":
+                    extra.append(pytest_html.extras.url("http://www.example.com/"))
+                report.extra = extra
+        """
+        )
+
         testdir.makepyfile(
             """
             import pytest
-            @pytest.mark.flaky(reruns=5)
+            import time
+
+            @pytest.mark.flaky(reruns=2)
             def test_example():
+                time.sleep(1)
                 assert False
         """
         )
+
         result, html = run(testdir)
         assert result.ret
-        assert_results(html, passed=0, failed=1, rerun=5)
+        assert_results(html, passed=0, failed=1, rerun=2)
+
+        expected_report_durations = r'<td class="col-duration">1.\d{2}</td>'
+        assert len(re.findall(expected_report_durations, html)) == 3
+
+        expected_report_extras = (
+            r'<td class="col-links"><a class="url" href="http://www.example.com/" '
+            'target="_blank">URL</a> </td>'
+        )
+        assert len(re.findall(expected_report_extras, html)) == 3
 
     def test_no_rerun(self, testdir):
         testdir.makepyfile("def test_pass(): pass")