User defined test generator#

canary generates test cases from .pyt, .vvt, and CTestTestFile.cmake files. Each generator is implemented as a subclass of AbstractTestGenerator. User defined test generators can also be created by subclassing AbstractTestGenerator and defining the matches(), describe(), and lock() methods. User defined test generators are registered with the canary_testcase_generator() plugin hook.

Consider the following YAML test input:

tests:
  hello_world:
    description: "A Hello world test"
    script:
    - echo "Hello, ${location}!"
    - echo "n = ${n}"
    keywords:
    - "hello"
    - "world"
    parameters:
      n: [2, 4, 8]
      location: ["World", "U.S.A", "Canada", "Mexico"]

The cartesian product of parameters should be taken and each combination used to generate a test case. Each test case should execute the script, first expanding variables of the form $variable or ${variable} with the parameter values.

In the sections that follow, a test generator will be developed that parses this and other similar test files.

Note

The completed test file generator can be seen at sandialabs/canary-yaml

Example implementation#

import io
import os
import re
from itertools import product
from pathlib import Path

import yaml

import canary


@canary.hookimpl
def canary_testcase_generator(root: str, path: str | None) -> "YAMLTestGenerator":
    if YAMLTestGenerator.matches(root if path is None else os.path.join(root, path)):
        return YAMLTestGenerator(root, path)


@canary.hookimpl
def canary_runtest_setup(case: canary.TestCase) -> None:
    if not YAMLTestGenerator.matches(case.spec.file):
        return
    sh = canary.filesystem.which("sh")
    if script := case.attributes.get("script"):
        with case.workspace.openfile("runtest.sh", "w") as fh:
            fh.write(f"#!{sh}\n")
            fh.write(f"cd {case.workspace.dir}\n")
            fh.write("\n".join(script))
        canary.filesystem.set_executable(case.workspace.joinpath("runtest.sh"))
    else:
        raise ValueError(f"{case}: script attribute not found")


@canary.hookimpl
def canary_runtest_execution_policy(case: canary.TestCase) -> canary.ExecutionPolicy | None:
    if YAMLTestGenerator.matches(case.spec.file):
        return canary.SubprocessExecutionPolicy(["./runtest.sh"])
    return None


class YAMLTestGenerator(canary.AbstractTestGenerator):
    """Define a YAML defined test case with the following schema:

    .. code-block:: yaml

       tests:

         str:
           description: str
           script: list[str]
           keywords: list[str]
           parameters: dict[str, list[float | int | str | None]]

    """

    @classmethod
    def matches(cls, path: str | Path) -> bool:
        """Is ``path`` a YAMLTestGenerator?"""
        path = Path(path)
        return re.match("test_.*\.yaml", path.name) is not None

    def lock(self, on_options: list[str] | None = None) -> list[canary.UnresolvedSpec]:
        """Take the cartesian product of parameters and from each combination create a test case."""

        with open(self.file, "r") as fh:
            fd = yaml.safe_load(fh)

        specs: list[canary.UnresolvedSpec] = []
        for name, details in fd["tests"].items():
            kwds = dict(
                file_root=Path(self.root),
                file_path=Path(self.path),
                family=name,
                keywords=details.get("keywords", []),
                attributes={"details": details.get("description"), "script": details["script"]},
            )

            if parameters := details.get("parameters"):
                keys = list(parameters.keys())
                for values in product(*parameters.values()):
                    params = dict(zip(keys, values))
                    spec = canary.UnresolvedSpec(parameters=params, **kwds)
                    specs.append(spec)
            else:
                spec = canary.UnresolvedSpec(**kwds)
                specs.append(spec)

        return specs

    def describe(self, on_options: list[str] | None = None) -> str:
        cases = self.lock(on_options=on_options)
        file = io.StringIO()
        file.write(f"--- {self.name} ------------\n")
        file.write(f"File: {self.file}\n")
        file.write(f"{len(cases)} test cases:\n")
        canary.graph.print(cases, file=file)
        return file.getvalue()
On this page