Use snapshot tests for lint tests

.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: debug-statements
       - id: end-of-file-fixer
       - id: trailing-whitespace
-
+        exclude: .*ambr
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v3.0.3
     hooks:

requirements-test.txt

@@ -3,3 +3,4 @@
 coverage[toml]
 pytest>7.0
 pytest-cov
+syrupy

tests/__snapshots__/test_validation.ambr

@@ -0,0 +1,193 @@
+# serializer version: 1
+# name: test_bad_examples_cli[endpoint-names-invalid.yaml]
+  '''
+  >>> endpoint-names-invalid.yaml
+  error:   Pattern validation on endpoint.name.pattern: 'Server-endpoint' does not match '^[a-z][a-z0-9-]+$' (0.endpoint.name)
+  ------------------------------------------------------------
+  error:   Pattern validation on endpoint.name.pattern: '3-server-endpoint' does not match '^[a-z][a-z0-9-]+$' (1.endpoint.name)
+  ------------------------------------------------------------
+  error:   Pattern validation on endpoint.name.pattern: 'server@endpoint' does not match '^[a-z][a-z0-9-]+$' (2.endpoint.name)
+  ------------------------------------------------------------
+  error:   Pattern validation on endpoint.name.pattern: 'wsgi_endpoint' does not match '^[a-z][a-z0-9-]+$' (3.endpoint.name)
+  ------------------------------------------------------------
+  *** 4 errors, 0 warnings
+
+  '''
+# ---
+# name: test_bad_examples_cli[input-name-too-long.yaml]
+  '''
+  >>> input-name-too-long.yaml
+  error:   Maxlength validation on step.inputs.name.maxLength: 'this-input-name-is-way-too-long-and-will-cause-the-validation-to-fail' is too long (0.step.inputs.0.name)
+  ------------------------------------------------------------
+  *** 1 errors, 0 warnings
+
+  '''
+# ---
+# name: test_bad_examples_cli[invalid-YAML-indentation.yaml]
+  '''
+  >>> invalid-YAML-indentation.yaml
+  error: Indentation Error at line 3, column 10
+  ------------------------------------------------------------
+  *** 1 errors, 0 warnings
+
+  '''
+# ---
+# name: test_bad_examples_cli[invalid-edge-parts.yaml]
+  '''
+  >>> invalid-edge-parts.yaml
+  error: Target specifier 'batch2.input' must have 3 parts (it has 2)
+  ------------------------------------------------------------
+  *** 1 errors, 0 warnings
+
+  '''
+# ---
+# name: test_bad_examples_cli[invalid-indentation-with-valid-YAML.yaml]
+  '''
+  >>> invalid-indentation-with-valid-YAML.yaml
+  error:   Type validation on step.type: None is not of type 'object' (0.step)
+  ------------------------------------------------------------
+  hint: File contains valid YAML but there might be an indentation error in following configuration: 0.step
+  ------------------------------------------------------------
+  *** 1 errors, 0 warnings
+
+  '''
+# ---
+# name: test_bad_examples_cli[invalid-on-error-value.yaml]
+  '''
+  >>> invalid-on-error-value.yaml
+  error:   Anyof validation on pipeline.nodes.anyOf: {'name': 'train-with-errors', 'step': 'train', 'type': 'task', 'on-error': 'report-to-sentry'} is not valid under any of the given schemas (2.pipeline.nodes.1)
+  ------------------------------------------------------------
+  *** 1 errors, 0 warnings
+
+  '''
+# ---
+# name: test_bad_examples_cli[invalid-pipeline-with-override-example.yaml]
+  '''
+  >>> invalid-pipeline-with-override-example.yaml
+  error: Pipeline My overriden input pipeline node merged step base step: input training-labels-error does not exist in step
+  ------------------------------------------------------------
+  error: Pipeline My overriden input pipeline node merged step base step: parameter d does not exist in step
+  ------------------------------------------------------------
+  error: Pipeline My overriden input pipeline node overridden step secon step does not exist
+  ------------------------------------------------------------
+  *** 3 errors, 0 warnings
+
+  '''
+# ---
+# name: test_bad_examples_cli[invalid-shorthand-edge.yaml]
+  '''
+  >>> invalid-shorthand-edge.yaml
+  error: Malformed edge shorthand ['batch1.input.training-labels*']
+  ------------------------------------------------------------
+  *** 1 errors, 0 warnings
+
+  '''
+# ---
+# name: test_bad_examples_cli[invalid-upload-store-value.yaml]
+  '''
+  >>> invalid-upload-store-value.yaml
+  error:   Maxlength validation on step.upload-store.maxLength: 'This is a test string exactly with  65 characters long in length.' is too long (0.step.upload-store)
+  ------------------------------------------------------------
+  *** 1 errors, 0 warnings
+
+  '''
+# ---
+# name: test_bad_examples_cli[step-invalid-resources.yaml]
+  '''
+  >>> step-invalid-resources.yaml
+  error:   Type validation on step.resources.cpu.min.type: 'Not a number' is not of type 'number' (0.step.resources.cpu.min)
+  ------------------------------------------------------------
+  *** 1 errors, 0 warnings
+
+  '''
+# ---
+# name: test_bad_examples_cli[step-missing-required-properties.yaml]
+  '''
+  >>> step-missing-required-properties.yaml
+  error:   Oneof validation on step.command.oneOf: 8 is not valid under any of the given schemas (1.step.command)
+  ------------------------------------------------------------
+  error:   Type validation on step.image.type: True is not of type 'string' (0.step.image)
+  ------------------------------------------------------------
+  error:   Required validation on step.required: 'command' is a required property (0.step)
+  ------------------------------------------------------------
+  error:   Required validation on step.required: 'name' is a required property (0.step)
+  ------------------------------------------------------------
+  *** 4 errors, 0 warnings
+
+  '''
+# ---
+# name: test_bad_examples_cli[step-stop-condition.yaml]
+  '''
+  >>> step-stop-condition.yaml
+  error: Step no-stop, `stop-condition` is not a valid expression: invalid syntax (<expression>, line 1)
+  ------------------------------------------------------------
+  *** 1 errors, 0 warnings
+
+  '''
+# ---
+# name: test_bad_examples_cli[step-with-error-tasks.yaml]
+  '''
+  >>> step-with-error-tasks.yaml
+  error:   Type validation on task.parameters.rules.type: None is not of type 'object' (1.task.parameters.0.rules)
+  ------------------------------------------------------------
+  error:   Type validation on task.parameters.name.type: None is not of type 'string' (2.task.parameters.0.name)
+  ------------------------------------------------------------
+  error:   Required validation on task.parameters.required: 'style' is a required property (2.task.parameters.0)
+  ------------------------------------------------------------
+  error:   Required validation on task.parameters.required: 'name' is a required property (5.task.parameters.0)
+  ------------------------------------------------------------
+  error:   Required validation on task.parameters.required: 'style' is a required property (5.task.parameters.0)
+  ------------------------------------------------------------
+  error:   Required validation on task.required: 'name' is a required property (1.task)
+  ------------------------------------------------------------
+  error:   Required validation on task.required: 'step' is a required property (4.task)
+  ------------------------------------------------------------
+  *** 7 errors, 0 warnings
+
+  '''
+# ---
+# name: test_bad_examples_cli[task-stop-condition.yaml]
+  '''
+  >>> task-stop-condition.yaml
+  error: Task no-stop, `stop-condition` is not a valid expression: EOL while scanning string literal (<expression>, line 1)
+  ------------------------------------------------------------
+  *** 1 errors, 0 warnings
+
+  '''
+# ---
+# name: test_warning_examples_cli[invalid-numeric-parameter-default.yaml]
+  '''
+  >>> invalid-numeric-parameter-default.yaml
+  warning: Step test, parameter value: default zoom is not an integer
+  ------------------------------------------------------------
+  *** 0 errors, 1 warnings
+
+  '''
+# ---
+# name: test_warning_examples_cli[invalid-numeric-range-parameter-default.yaml]
+  '''
+  >>> invalid-numeric-range-parameter-default.yaml
+  warning: Step test, parameter value: default 1337 is greater than the maximum allowed (100)
+  ------------------------------------------------------------
+  *** 0 errors, 1 warnings
+
+  '''
+# ---
+# name: test_warning_examples_cli[invalid-string-parameter-default.yaml]
+  '''
+  >>> invalid-string-parameter-default.yaml
+  warning: Step test, parameter ctime: default `default` value datetime.datetime(2020, 9, 2, 0, 0, tzinfo=datetime.timezone.utc) is not a string (got a <class 'datetime.datetime'>)
+  ------------------------------------------------------------
+  *** 0 errors, 1 warnings
+
+  '''
+# ---
+# name: test_warning_examples_cli[override-with-extra-fields-warning.yaml]
+  '''
+  >>> override-with-extra-fields-warning.yaml
+  error:   Anyof validation on pipeline.nodes.anyOf: {'name': 'merged', 'type': 'execution', 'step': 'base step', 'override': {'training-images': 's3://hello/hello.jpg'}} is not valid under any of the given schemas (1.pipeline.nodes.0)
+  ------------------------------------------------------------
+  *** 1 errors, 0 warnings
+
+  '''
+# ---

tests/test_validation.py

@@ -1,5 +1,6 @@
 import glob
 import os
+import re
 
 import pytest
 
@@ -8,83 +9,69 @@
     examples_path,
     invalid_obj,
     valid_bytes,
+    valid_examples_path,
     valid_obj,
     warning_examples_path,
 )
-from tests.utils import get_error_example_path, get_valid_example_path
 from valohai_yaml import ValidationErrors, validate
 from valohai_yaml.__main__ import main
 
 
+def assert_validation_output(capsys, snapshot, yaml_path: str) -> bool:
+    out, err = capsys.readouterr()
+    # Replace the path with just the filename,
+    # so this is agnostic to where the tests are run from.
+    out = out.replace(yaml_path, os.path.basename(yaml_path))
+    # Normalize Python version differences in certain error messages.
+    out = re.sub(
+        r"unterminated string literal \(.+?\)",
+        "EOL while scanning string literal",
+        out,
+    )
+    assert out == snapshot
+    return True
+
+
 @pytest.mark.parametrize(
     "good_example_path",
     glob.glob(os.path.join(examples_path, "*.yaml")),
+    ids=os.path.basename,
 )
 def test_good_examples_cli(good_example_path):
     """Test that known-good files validate via the CLI."""
     assert main([good_example_path]) == 0
 
 
 @pytest.mark.parametrize(
-    "bad_example_path",
+    "yaml_path",
     glob.glob(os.path.join(error_examples_path, "*.yaml")),
+    ids=os.path.basename,
 )
-def test_bad_examples_cli(capsys, bad_example_path):
+def test_bad_examples_cli(capsys, yaml_path, snapshot):
     """Test that bad examples don't validate via the CLI."""
-    assert main([bad_example_path]) == 1
-    out, err = capsys.readouterr()
-    assert out
+    assert main([yaml_path]) == 1
+    assert assert_validation_output(capsys, snapshot, yaml_path)
 
 
 @pytest.mark.parametrize(
     "yaml_path",
     glob.glob(os.path.join(warning_examples_path, "*.yaml")),
+    ids=os.path.basename,
 )
-def test_warning_examples_cli(capsys, yaml_path):
+def test_warning_examples_cli(capsys, yaml_path, snapshot):
     """Test that warning examples don't validate via the CLI in strict mode."""
     assert main(["--strict-warnings", yaml_path]) == 1
-    out, err = capsys.readouterr()
-    assert out
-
-
-def test_invalid_file_missing_properties_cli(capsys):
-    assert main([get_error_example_path("step-missing-required-properties.yaml")]) == 1
-    out, err = capsys.readouterr()
-    assert "step-missing-required-properties.yaml" in out
-    assert "'command' is a required property" in out
-    assert "'name' is a required property" in out
-    assert "8 is not valid under any of the given schemas" in out
-
-
-def test_invalid_file_too_long_input_name_cli(capsys):
-    assert main([get_error_example_path("input-name-too-long.yaml")]) == 1
-    out, err = capsys.readouterr()
-    assert "input-name-too-long.yaml" in out
-    assert (
-        "'this-input-name-is-way-too-long-and-will-cause-the-validation-to-fail' is too long"
-        in out
-    )
-
-
-def test_invalid_on_error_value(capsys):
-    assert main([get_error_example_path("invalid-on-error-value.yaml")]) == 1
-    out, err = capsys.readouterr()
-    assert "invalid-on-error-value.yaml" in out
-    assert "is not valid under any of the given schemas" in out
+    assert assert_validation_output(capsys, snapshot, yaml_path)
 
 
-def test_valid_endpoint_name_cli(capsys):
-    assert main([get_valid_example_path("endpoint-names-valid.yaml")]) == 0
-
-
-def test_invalid_endpoint_name_cli(capsys):
-    assert main([get_error_example_path("endpoint-names-invalid.yaml")]) == 1
-    out, err = capsys.readouterr()
-    assert "endpoint-names-invalid.yaml" in out
-    assert "'wsgi_endpoint' does not match '^[a-z][a-z0-9-]+$'" in out
-    assert "'server@endpoint' does not match '^[a-z][a-z0-9-]+$'" in out
-    assert "'3-server-endpoint' does not match '^[a-z][a-z0-9-]+$'" in out
-    assert "'Server-endpoint' does not match '^[a-z][a-z0-9-]+$'" in out
+@pytest.mark.parametrize(
+    "valid_example_path",
+    glob.glob(os.path.join(valid_examples_path, "*.yaml")),
+    ids=os.path.basename,
+)
+def test_valid_examples_cli(valid_example_path):
+    """Test that valid example files validate via the CLI."""
+    assert main([valid_example_path]) == 0
 
 
 def test_bytes_validation():