diff --git a/README.rst b/README.rst index 49a68d03..86b3e6b1 100644 --- a/README.rst +++ b/README.rst @@ -84,19 +84,19 @@ The hash library can be generated with ``--mpl-generate-hash-library=path_to_file.json``. The hash library to be used can either be specified via the ``--mpl-hash-library=`` command line argument, or via the ``hash_library=`` keyword argument to the -``@pytest.mark.mpl_image_comapre`` decorator. +``@pytest.mark.mpl_image_compare`` decorator. Hybrid Mode: Hashes and Images ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It is possible to configure both hashes and baseline images. In this scenario -the hashes will be compared first, and the baseline images used if the hash +the hashes will be compared first, with the baseline images only used if the hash comparison fails. This is especially useful if the baseline images are external to the repository -with the tests in, and can be accessed remotely. In this situation if the hashes -match the baseline images wont be retrieved, saving time and bandwidth. Also it +containing the tests, and are accessed via HTTP. In this situation, if the hashes +match, the baseline images won't be retrieved, saving time and bandwidth. Also, it allows the tests to be modified and the hashes updated to reflect the changes without having to modify the external images. @@ -111,16 +111,16 @@ against, the tests can be run with:: and the tests will pass if the images are the same. If you omit the ``--mpl`` option, the tests will run but will only check that the code -runs without checking the output images. +runs, without checking the output images. Generating a Failure Summary ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -By specifying the ``--mpl-generate-summary=html`` CLI argument a HTML summary +By specifying the ``--mpl-generate-summary=html`` CLI argument, a HTML summary page will be generated showing the baseline, diff and result image for each failing test. If no baseline images are configured, just the result images will -be displayed. +be displayed. (See also, the **Results always** section below.) Options ------- @@ -182,6 +182,24 @@ are run. In addition, if both this option and the ``baseline_dir`` option in the ``mpl_image_compare`` decorator are used, the one in the decorator takes precedence. +Results always +^^^^^^^^^^^^^^ + +By default, result images are only generated for tests that fail. +Passing ``--mpl-results-always`` to pytest will force result images +to be generated for all tests, even for tests that pass. +If a baseline image exists, a diff image will also be generated. +All of these images will be shown in the summary (if requested). + +This option is useful for always *comparing* the result images again +the baseline images, while only *assessing* the tests against the +hash library. +If you only update your baseline images after merging a PR, this +option means that the generated summary will always show how the +PR affects the baseline images, with the success status of each +test (based on the hash library) also shown in the generated +summary. + Base style ^^^^^^^^^^ diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 2736eae3..b2568ff5 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -155,6 +155,12 @@ def pytest_addoption(parser): results_path_help = "directory for test results, relative to location where py.test is run" group.addoption('--mpl-results-path', help=results_path_help, action='store') parser.addini('mpl-results-path', help=results_path_help) + + results_always_help = "Always generate result images, not just for failed tests." + group.addoption('--mpl-results-always', action='store_true', + help=results_always_help) + parser.addini('mpl-results-always', help=results_always_help) + parser.addini('mpl-use-full-test-name', help="use fully qualified test name as the filename.", type='bool') @@ -175,6 +181,8 @@ def pytest_configure(config): results_dir = config.getoption("--mpl-results-path") or config.getini("mpl-results-path") hash_library = config.getoption("--mpl-hash-library") generate_summary = config.getoption("--mpl-generate-summary") + results_always = (config.getoption("--mpl-results-always") or + config.getini("mpl-results-always")) if config.getoption("--mpl-baseline-relative"): baseline_relative_dir = config.getoption("--mpl-baseline-path") @@ -205,7 +213,8 @@ def pytest_configure(config): results_dir=results_dir, hash_library=hash_library, generate_hash_library=generate_hash_lib, - generate_summary=generate_summary)) + generate_summary=generate_summary, + results_always=results_always)) else: @@ -262,7 +271,8 @@ def __init__(self, results_dir=None, hash_library=None, generate_hash_library=None, - generate_summary=None + generate_summary=None, + results_always=False ): self.config = config self.baseline_dir = baseline_dir @@ -274,6 +284,7 @@ def __init__(self, if generate_summary and generate_summary.lower() not in ("html",): raise ValueError(f"The mpl summary type '{generate_summary}' is not supported.") self.generate_summary = generate_summary + self.results_always = results_always # Generate the containing dir for all test results if not self.results_dir: @@ -282,6 +293,7 @@ def __init__(self, # We need global state to store all the hashes generated over the run self._generated_hash_library = {} + self._test_results = {} def get_compare(self, item): """ @@ -389,7 +401,6 @@ def generate_baseline_image(self, item, fig): **savefig_kwargs) close_mpl_figure(fig) - pytest.skip("Skipping test, since generating image") def generate_image_hash(self, item, fig): """ @@ -455,6 +466,10 @@ def load_hash_library(self, library_path): return json.load(fp) def compare_image_to_hash_library(self, item, fig, result_dir): + new_test = False + hash_comparison_pass = False + baseline_image_path = None + compare = self.get_compare(item) savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) @@ -470,33 +485,51 @@ def compare_image_to_hash_library(self, item, fig, result_dir): test_hash = self.generate_image_hash(item, fig) if hash_name not in hash_library: - return (f"Hash for test '{hash_name}' not found in {hash_library_filename}. " - f"Generated hash is {test_hash}.") + new_test = True + error_message = (f"Hash for test '{hash_name}' not found in {hash_library_filename}. " + f"Generated hash is {test_hash}.") - if test_hash == hash_library[hash_name]: - return + # Save the figure for later summary (will be removed later if not needed) + test_image = (result_dir / "result.png").absolute() + fig.savefig(str(test_image), **savefig_kwargs) - error_message = (f"Hash {test_hash} doesn't match hash " - f"{hash_library[hash_name]} in library " - f"{hash_library_filename} for test {hash_name}.") + if not new_test: + if test_hash == hash_library[hash_name]: + hash_comparison_pass = True + else: + error_message = (f"Hash {test_hash} doesn't match hash " + f"{hash_library[hash_name]} in library " + f"{hash_library_filename} for test {hash_name}.") # If the compare has only been specified with hash and not baseline # dir, don't attempt to find a baseline image at the default path. - if not self.baseline_directory_specified(item): - # Save the figure for later summary - test_image = (result_dir / "result.png").absolute() - fig.savefig(str(test_image), **savefig_kwargs) + if not hash_comparison_pass and not self.baseline_directory_specified(item) or new_test: return error_message - try: - baseline_image_path = self.obtain_baseline_image(item, result_dir) - baseline_image = baseline_image_path - baseline_image = None if not baseline_image.exists() else baseline_image - except Exception: - baseline_image = None + # If this is not a new test try and get the baseline image. + if not new_test: + baseline_error = None + # Ignore Errors here as it's possible the reference image dosen't exist yet. + try: + baseline_image_path = self.obtain_baseline_image(item, result_dir) + baseline_image = baseline_image_path + if baseline_image and not baseline_image.exists(): + baseline_image = None + # Get the baseline and generate a diff image, always so that + # --mpl-results-always can be respected. + baseline_comparison = self.compare_image_to_baseline(item, fig, result_dir) + except Exception as e: + baseline_image = None + baseline_error = e + + # If the hash comparison passes then return + if hash_comparison_pass: + return if baseline_image is None: error_message += f"\nUnable to find baseline image for {item}." + if baseline_error: + error_message += f"\n{baseline_error}" return error_message # Override the tolerance (if not explicitly set) to 0 as the hashes are not forgiving @@ -504,7 +537,7 @@ def compare_image_to_hash_library(self, item, fig, result_dir): if not tolerance: compare.kwargs['tolerance'] = 0 - comparison_error = (self.compare_image_to_baseline(item, fig, result_dir) or + comparison_error = (baseline_comparison or "\nHowever, the comparison to the baseline image succeeded.") return f"{error_message}\n{comparison_error}" @@ -548,14 +581,17 @@ def item_function_wrapper(*args, **kwargs): if remove_text: remove_ticks_and_titles(fig) + test_name = self.generate_test_name(item) + # What we do now depends on whether we are generating the # reference images or simply running the test. if self.generate_dir is not None: self.generate_baseline_image(item, fig) + if self.generate_hash_library is None: + pytest.skip("Skipping test, since generating image.") if self.generate_hash_library is not None: - hash_name = self.generate_test_name(item) - self._generated_hash_library[hash_name] = self.generate_image_hash(item, fig) + self._generated_hash_library[test_name] = self.generate_image_hash(item, fig) # Only test figures if not generating images if self.generate_dir is None: @@ -571,8 +607,11 @@ def item_function_wrapper(*args, **kwargs): close_mpl_figure(fig) + self._test_results[str(pathify(test_name))] = msg or True + if msg is None: - shutil.rmtree(result_dir) + if not self.results_always: + shutil.rmtree(result_dir) else: pytest.fail(msg, pytrace=False) @@ -592,8 +631,10 @@ def generate_summary_html(self, dir_list): f.write(HTML_INTRO) for directory in dir_list: + test_name = directory.parts[-1] + test_result = 'passed' if self._test_results[test_name] is True else 'failed' f.write('' - f'{directory.parts[-1]}\n' + f'{test_name} ({test_result})\n' f'\n' f'\n' f'\n' diff --git a/tests/baseline/2.0.x/test_modified.png b/tests/baseline/2.0.x/test_modified.png new file mode 100644 index 00000000..402b29fc Binary files /dev/null and b/tests/baseline/2.0.x/test_modified.png differ diff --git a/tests/baseline/2.0.x/test_unmodified.png b/tests/baseline/2.0.x/test_unmodified.png new file mode 100644 index 00000000..beb80cd1 Binary files /dev/null and b/tests/baseline/2.0.x/test_unmodified.png differ diff --git a/tests/baseline/hashes/mpl20_ft2104.json b/tests/baseline/hashes/mpl20_ft261.json similarity index 70% rename from tests/baseline/hashes/mpl20_ft2104.json rename to tests/baseline/hashes/mpl20_ft261.json index b16aba5d..76322063 100644 --- a/tests/baseline/hashes/mpl20_ft2104.json +++ b/tests/baseline/hashes/mpl20_ft261.json @@ -8,5 +8,9 @@ "test_pytest_mpl.test_remove_text": "9c284d7bcbbb1d6c1362b417859e4ce842b573a2fe32c7ceaafcf328a1eb7057", "test_pytest_mpl.test_parametrized[5]": "04c998af2d7932ca4a851d610e8a020d94a2f623d1301dbe9b59fe6efd28a5f7", "test_pytest_mpl.test_parametrized[50]": "937d986ab6b209e7d48eb30cc30e9db62c93bbc4c86768e276a5b454e63bca93", - "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498" -} \ No newline at end of file + "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498", + "test_pytest_mpl.test_hash_succeeds": "480062c2239ed9d70e361d1a5b578dc2aa756971161ac6e7287b492ae6118c59", + "test.test_modified": "54f6cf83d5b06fa2ecb7fa23d6e87898679178ef5d0dfdd2551a139f1932127b", + "test.test_new": "54f6cf83d5b06fa2ecb7fa23d6e87898679178ef5d0dfdd2551a139f1932127b", + "test.test_unmodified": "54f6cf83d5b06fa2ecb7fa23d6e87898679178ef5d0dfdd2551a139f1932127b" +} diff --git a/tests/baseline/hashes/mpl21_ft2104.json b/tests/baseline/hashes/mpl21_ft261.json similarity index 70% rename from tests/baseline/hashes/mpl21_ft2104.json rename to tests/baseline/hashes/mpl21_ft261.json index b16aba5d..8b2beb5a 100644 --- a/tests/baseline/hashes/mpl21_ft2104.json +++ b/tests/baseline/hashes/mpl21_ft261.json @@ -8,5 +8,9 @@ "test_pytest_mpl.test_remove_text": "9c284d7bcbbb1d6c1362b417859e4ce842b573a2fe32c7ceaafcf328a1eb7057", "test_pytest_mpl.test_parametrized[5]": "04c998af2d7932ca4a851d610e8a020d94a2f623d1301dbe9b59fe6efd28a5f7", "test_pytest_mpl.test_parametrized[50]": "937d986ab6b209e7d48eb30cc30e9db62c93bbc4c86768e276a5b454e63bca93", - "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498" -} \ No newline at end of file + "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498", + "test_pytest_mpl.test_hash_succeeds": "17b65dd0247b0dfd8c1b4b079352414ae0fe03c0a3e79d63c8b8670d84d4098f", + "test.test_modified": "14d326881467bc613e6504b87bd7d556a5e58668ff16b896fa3c15745cfb6336", + "test.test_new": "14d326881467bc613e6504b87bd7d556a5e58668ff16b896fa3c15745cfb6336", + "test.test_unmodified": "14d326881467bc613e6504b87bd7d556a5e58668ff16b896fa3c15745cfb6336" +} diff --git a/tests/baseline/hashes/mpl22_ft2104.json b/tests/baseline/hashes/mpl22_ft261.json similarity index 70% rename from tests/baseline/hashes/mpl22_ft2104.json rename to tests/baseline/hashes/mpl22_ft261.json index b16aba5d..08c26f4d 100644 --- a/tests/baseline/hashes/mpl22_ft2104.json +++ b/tests/baseline/hashes/mpl22_ft261.json @@ -8,5 +8,9 @@ "test_pytest_mpl.test_remove_text": "9c284d7bcbbb1d6c1362b417859e4ce842b573a2fe32c7ceaafcf328a1eb7057", "test_pytest_mpl.test_parametrized[5]": "04c998af2d7932ca4a851d610e8a020d94a2f623d1301dbe9b59fe6efd28a5f7", "test_pytest_mpl.test_parametrized[50]": "937d986ab6b209e7d48eb30cc30e9db62c93bbc4c86768e276a5b454e63bca93", - "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498" -} \ No newline at end of file + "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498", + "test_pytest_mpl.test_hash_succeeds": "e80557c8784fb920fb79b03b26dc072649a98811f00a8c212df8761e4351acde", + "test.test_modified": "80e0ee6df7cf7d9d9407395a25af30beb8763e98820a7be972764899246d2cd7", + "test.test_new": "80e0ee6df7cf7d9d9407395a25af30beb8763e98820a7be972764899246d2cd7", + "test.test_unmodified": "80e0ee6df7cf7d9d9407395a25af30beb8763e98820a7be972764899246d2cd7" +} diff --git a/tests/baseline/hashes/mpl30_ft2104.json b/tests/baseline/hashes/mpl30_ft261.json similarity index 70% rename from tests/baseline/hashes/mpl30_ft2104.json rename to tests/baseline/hashes/mpl30_ft261.json index b16aba5d..313078ea 100644 --- a/tests/baseline/hashes/mpl30_ft2104.json +++ b/tests/baseline/hashes/mpl30_ft261.json @@ -8,5 +8,9 @@ "test_pytest_mpl.test_remove_text": "9c284d7bcbbb1d6c1362b417859e4ce842b573a2fe32c7ceaafcf328a1eb7057", "test_pytest_mpl.test_parametrized[5]": "04c998af2d7932ca4a851d610e8a020d94a2f623d1301dbe9b59fe6efd28a5f7", "test_pytest_mpl.test_parametrized[50]": "937d986ab6b209e7d48eb30cc30e9db62c93bbc4c86768e276a5b454e63bca93", - "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498" -} \ No newline at end of file + "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498", + "test_pytest_mpl.test_hash_succeeds": "4e1157a93733cdb327f1741afdb0525f4d0e3f12e60b54f72c93db9f9c9ae27f", + "test.test_modified": "6e2e4ba7b77caf62df24f6b92d6fc51ab1b837bf98039750334f65c0a6c5d898", + "test.test_new": "6e2e4ba7b77caf62df24f6b92d6fc51ab1b837bf98039750334f65c0a6c5d898", + "test.test_unmodified": "6e2e4ba7b77caf62df24f6b92d6fc51ab1b837bf98039750334f65c0a6c5d898" +} diff --git a/tests/baseline/hashes/mpl31_ft261.json b/tests/baseline/hashes/mpl31_ft261.json index 588466c9..dbde4d54 100644 --- a/tests/baseline/hashes/mpl31_ft261.json +++ b/tests/baseline/hashes/mpl31_ft261.json @@ -9,5 +9,8 @@ "test_pytest_mpl.test_parametrized[5]": "be7dc9de64a5d6fd458c1f930d4aa56cf8196ddb0e8b5b07ab79a1f0ea9eb820", "test_pytest_mpl.test_parametrized[50]": "a8ae2427337803dc864784d88c4428a6af5a3e47d2bfc84c98b68b25fde75704", "test_pytest_mpl.test_parametrized[500]": "590ef42388378173e293bd37e95ff22d8e753d53327d1fb5d6bdf2bac4f84d01", - "test_pytest_mpl.test_hash_succeeds": "2a4da3a36b384df539f3f47d476f67a918f5eee1df360dbab9469b96260df78f" -} \ No newline at end of file + "test_pytest_mpl.test_hash_succeeds": "2a4da3a36b384df539f3f47d476f67a918f5eee1df360dbab9469b96260df78f", + "test.test_modified": "3675e5a48388e8cc341580e9b41115d3cf63d2465cf11eeed3faa23e84030fc2", + "test.test_new": "3675e5a48388e8cc341580e9b41115d3cf63d2465cf11eeed3faa23e84030fc2", + "test.test_unmodified": "3675e5a48388e8cc341580e9b41115d3cf63d2465cf11eeed3faa23e84030fc2" +} diff --git a/tests/baseline/hashes/mpl32_ft261.json b/tests/baseline/hashes/mpl32_ft261.json index ac0bc8fb..c39964b0 100644 --- a/tests/baseline/hashes/mpl32_ft261.json +++ b/tests/baseline/hashes/mpl32_ft261.json @@ -9,5 +9,8 @@ "test_pytest_mpl.test_parametrized[5]": "9b2b5b1df784c8f9a5fc624840138fe7b4dbdd42cf592fe5733c9c825e5dda91", "test_pytest_mpl.test_parametrized[50]": "fcf0566ef5514674e2b4bf1e9b4c7f52451c6f98abdc75dc876f43c97a23bc32", "test_pytest_mpl.test_parametrized[500]": "38dccccfc980b44359bc1b325bef48471bc084db37ed622af00a553792a8b093", - "test_pytest_mpl.test_hash_succeeds": "8b8ff9ce044bc9075876278781667a708414460209bba25a39d8262ed73d0f04" -} \ No newline at end of file + "test_pytest_mpl.test_hash_succeeds": "8b8ff9ce044bc9075876278781667a708414460209bba25a39d8262ed73d0f04", + "test.test_modified": "3b7db65812fd59403d17a2fba3ebe1fd0abdfde8633df06636e4e1daea259da0", + "test.test_new": "3b7db65812fd59403d17a2fba3ebe1fd0abdfde8633df06636e4e1daea259da0", + "test.test_unmodified": "3b7db65812fd59403d17a2fba3ebe1fd0abdfde8633df06636e4e1daea259da0" +} diff --git a/tests/baseline/hashes/mpl33_ft261.json b/tests/baseline/hashes/mpl33_ft261.json index e8502633..77c5f03f 100644 --- a/tests/baseline/hashes/mpl33_ft261.json +++ b/tests/baseline/hashes/mpl33_ft261.json @@ -9,5 +9,8 @@ "test_pytest_mpl.test_parametrized[5]": "04c998af2d7932ca4a851d610e8a020d94a2f623d1301dbe9b59fe6efd28a5f7", "test_pytest_mpl.test_parametrized[50]": "937d986ab6b209e7d48eb30cc30e9db62c93bbc4c86768e276a5b454e63bca93", "test_pytest_mpl.test_parametrized[500]": "e39ed724b0762b8736879801e32dc0c1525afd03c0567a43b119435aaa608498", - "test_pytest_mpl.test_hash_succeeds": "55ad74a44c99606f1df1e79f67a59a4986bddc2b48ea2b2d7ea8365db91dc7ca" + "test_pytest_mpl.test_hash_succeeds": "55ad74a44c99606f1df1e79f67a59a4986bddc2b48ea2b2d7ea8365db91dc7ca", + "test.test_modified": "ce07de6b726c3b01afb03aa7c9e939d584bc71a54b9737d69853a0d915cd6181", + "test.test_new": "ce07de6b726c3b01afb03aa7c9e939d584bc71a54b9737d69853a0d915cd6181", + "test.test_unmodified": "ce07de6b726c3b01afb03aa7c9e939d584bc71a54b9737d69853a0d915cd6181" } diff --git a/tests/test_pytest_mpl.py b/tests/test_pytest_mpl.py index 735defd9..559e0499 100644 --- a/tests/test_pytest_mpl.py +++ b/tests/test_pytest_mpl.py @@ -411,3 +411,61 @@ def test_hash_missing(tmpdir): # If we don't use --mpl option, the test should succeed code = call_pytest([test_file]) assert code == 0 + + +TEST_RESULTS_ALWAYS = """ +import pytest +import matplotlib.pyplot as plt +def plot(): + fig = plt.figure() + ax = fig.add_subplot(1,1,1) + ax.plot([1,2,2]) + return fig +@pytest.mark.mpl_image_compare +def test_modified(): return plot() +@pytest.mark.mpl_image_compare +def test_new(): return plot() +@pytest.mark.mpl_image_compare +def test_unmodified(): return plot() +""" + + +@pytest.mark.skipif(not hash_library.exists(), reason="No hash library for this mpl version") +def test_results_always(tmpdir): + + test_file = tmpdir.join('test.py').strpath + with open(test_file, 'w') as f: + f.write(TEST_RESULTS_ALWAYS) + results_path = tmpdir.mkdir('results') + + code = call_pytest(['--mpl', test_file, '--mpl-results-always', + rf'--mpl-hash-library={hash_library}', + rf'--mpl-baseline-path={baseline_dir_abs}', + '--mpl-generate-summary=html', + rf'--mpl-results-path={results_path.strpath}']) + assert code == 0 # hashes correct, so all should pass + + comparison_file = results_path.join('fig_comparison.html') + with open(comparison_file, 'r') as f: + html = f.read() + + # each test, and which images should exist + for test, exists in [ + ('test_modified', ['baseline', 'result-failed-diff', 'result']), + ('test_new', ['result']), + ('test_unmodified', ['baseline', 'result']), + ]: + + test_name = f'test.{test}' + + summary = f'{test_name} (passed)' + assert summary in html + + for image_type in ['baseline', 'result-failed-diff', 'result']: + image = f'{test_name}/{image_type}.png' + assert image in html # is present even if 404 + image_exists = results_path.join(*image.split('/')).exists() + if image_type in exists: # assert image so pytest prints it on error + assert image and image_exists + else: + assert image and not image_exists