From 39f78413fc651a1e8ec334ce5b61ff75fdde625b Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Thu, 9 May 2019 13:45:46 +0200 Subject: Tests: add tests/report.html that links to all HTML test reports Currently this is for Cycles, Eevee and workbench tests. --- tests/python/modules/global_report.py | 75 +++++++++++++++++++++++++++++++++++ tests/python/modules/render_report.py | 30 ++++++++------ 2 files changed, 92 insertions(+), 13 deletions(-) create mode 100755 tests/python/modules/global_report.py (limited to 'tests/python') diff --git a/tests/python/modules/global_report.py b/tests/python/modules/global_report.py new file mode 100755 index 00000000000..e45d159bb2d --- /dev/null +++ b/tests/python/modules/global_report.py @@ -0,0 +1,75 @@ +# Apache License, Version 2.0 +# +# Generate a HTML page that links to all test reports. + +import glob +import os +import pathlib + +def _write_html(output_dir): + combined_reports = "" + + # Gather intermediate data for all tests and combine into one HTML file. + categories = sorted(glob.glob(os.path.join(output_dir, "report", "*"))) + + for category in categories: + category_name = os.path.basename(category) + combined_reports += "

" + category_name + "

\n" + + reports = sorted(glob.glob(os.path.join(category, "*.data"))) + for filename in reports: + filepath = os.path.join(output_dir, filename) + combined_reports += pathlib.Path(filepath).read_text() + + combined_reports += "
\n"; + + html = """ + + + {title} + + + + +
+
+ {combined_reports} +
+
+ + + """ . format(title="Blender Test Reports", + combined_reports=combined_reports) + + filepath = os.path.join(output_dir, "report.html") + pathlib.Path(filepath).write_text(html) + + +def add(output_dir, category, name, filepath, failed=None): + # Write HTML for single test. + if failed is None: + status = "none" + elif failed: + status = "failed" + else: + status = "ok" + + html = """ + ⬤ + {name}
+ """ . format(status=status, + name=name, + filepath=filepath) + + dirpath = os.path.join(output_dir, "report", category); + os.makedirs(dirpath, exist_ok=True) + filepath = os.path.join(dirpath, name + ".data") + pathlib.Path(filepath).write_text(html) + + # Combined into HTML, each time so we can see intermediate results + # while tests are still running. + _write_html(output_dir) diff --git a/tests/python/modules/render_report.py b/tests/python/modules/render_report.py index 9cff85f3dfa..5a2baa354c7 100755 --- a/tests/python/modules/render_report.py +++ b/tests/python/modules/render_report.py @@ -11,6 +11,8 @@ import subprocess import sys import time +from . import global_report + class COLORS_ANSI: RED = '\033[00;31m' @@ -72,19 +74,16 @@ def test_get_images(output_dir, filepath, reference_dir): ref_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "ref") ref_img = os.path.join(ref_dirpath, testname + ".png") - if not os.path.exists(ref_dirpath): - os.makedirs(ref_dirpath) + os.makedirs(ref_dirpath, exist_ok=True) if os.path.exists(old_img): shutil.copy(old_img, ref_img) new_dirpath = os.path.join(output_dir, os.path.basename(dirpath)) - if not os.path.exists(new_dirpath): - os.makedirs(new_dirpath) + os.makedirs(new_dirpath, exist_ok=True) new_img = os.path.join(new_dirpath, testname + ".png") diff_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "diff") - if not os.path.exists(diff_dirpath): - os.makedirs(diff_dirpath) + os.makedirs(diff_dirpath, exist_ok=True) diff_img = os.path.join(diff_dirpath, testname + ".diff.png") return old_img, ref_img, new_img, diff_img @@ -124,8 +123,7 @@ class Report: self.passed_tests = "" self.compare_tests = "" - if not os.path.exists(output_dir): - os.makedirs(output_dir) + os.makedirs(output_dir, exist_ok=True) def set_pixelated(self, pixelated): self.pixelated = pixelated @@ -149,8 +147,7 @@ class Report: def _write_data(self, dirname): # Write intermediate data for single test. outdir = os.path.join(self.output_dir, dirname) - if not os.path.exists(outdir): - os.makedirs(outdir) + os.makedirs(outdir, exist_ok=True) filepath = os.path.join(outdir, "failed.data") pathlib.Path(filepath).write_text(self.failed_tests) @@ -189,7 +186,8 @@ class Report: else: image_rendering = 'auto' - if len(failed_tests) > 0: + failed = len(failed_tests) > 0 + if failed: message = "

Run BLENDER_TEST_UPDATE=1 ctest to create or update reference images for failed tests.

" else: message = "" @@ -258,6 +256,13 @@ class Report: print_message("Report saved to: " + pathlib.Path(filepath).as_uri()) + + # Update global report + link_name = "Renders" if not comparison else "Comparison" + global_output_dir = os.path.dirname(self.output_dir) + global_failed = failed if not comparison else None + global_report.add(global_output_dir, self.title, link_name, filepath, global_failed) + def _relative_url(self, filepath): relpath = os.path.relpath(filepath, self.output_dir) return pathlib.Path(relpath).as_posix() @@ -316,8 +321,7 @@ class Report: # Create reference render directory. old_dirpath = os.path.dirname(old_img) - if not os.path.exists(old_dirpath): - os.makedirs(old_dirpath) + os.makedirs(old_dirpath, exist_ok=True) # Copy temporary to new image. if os.path.exists(new_img): -- cgit v1.2.3