From 2aae1c84893d9793debdfd41be595c8adeea426f Mon Sep 17 00:00:00 2001
From: Miroslav Vadkerti <mvadkert@redhat.com>
Date: Tue, 21 Apr 2020 13:17:59 +0200
Subject: [PATCH] Add additionalk test checks

This PR introduces additional test checks that make sense
to run while runing the tests.

The checks are by default controlled on the test level,
but can be overridden in plans.

Signed-off-by: Miroslav Vadkerti <mvadkert@redhat.com>
---
 spec/steps/discover.fmf |  1 +
 spec/steps/execute.fmf  | 29 +++++++++++++++++++++++++++++
 spec/steps/report.fmf   |  3 +++
 spec/tests/checks.fmf   | 26 ++++++++++++++++++++++++++
 4 files changed, 59 insertions(+)
 create mode 100644 spec/tests/checks.fmf

diff --git a/spec/steps/discover.fmf b/spec/steps/discover.fmf
index 1f8788e0ab..45165dbc87 100644
--- a/spec/steps/discover.fmf
+++ b/spec/steps/discover.fmf
@@ -32,6 +32,7 @@ description: |
             tier: 1
             relevancy: |
                 distro < rhel-8: False
+            checks: -avc
 
         /test/two:
             summary: Short test summary.
diff --git a/spec/steps/execute.fmf b/spec/steps/execute.fmf
index 9a2aba8089..445702252e 100644
--- a/spec/steps/execute.fmf
+++ b/spec/steps/execute.fmf
@@ -13,6 +13,10 @@ description: |
         /test/one:
             result: TEST_RESULT
             log: PATH
+            checks:
+                CHECK_NAME:
+                    result: TEST_RESULT
+                    log: PATH
 
         /test/two:
             result: TEST_RESULT
@@ -24,6 +28,8 @@ description: |
         * info - test finished and produced only information
           message
         * passed - test finished and passed
+        * warning - test finished and passed with a warning,
+          like an ``avc`` check failure
         * failed - test finished and failed
         * error - a problem encountered during test execution,
           treated as failed
@@ -31,6 +37,13 @@ description: |
     Where ``PATH`` is the test log output path, relative to the
     execute step plan run directory.
 
+    Where ``CHECK_NAME`` specifies a check which was executed during
+    the testing. Multiple checks can end up here.
+
+    Checks can be overridden by the ``checks`` property of the
+    ``execute`` step. The property in execute overrides all checks
+    from the test attribute.
+
 example: |
     execute:
         how: beakerlib
@@ -98,6 +111,22 @@ example: |
                     echo foo > /var/www/html/index.html
                     curl http://localhost/ | grep foo
 
+    /script-no-avc:
+        summary: Multi-line shell script without the default AVC check
+        description:
+            Providing a multi-line shell script is also
+            supported. In that case executor will store given
+            script into a file and execute. The ``avc`` check is disabled
+            when running this test.
+        example: |
+            execute:
+                checks: -avc
+                script: |
+                    dnf -y install httpd curl
+                    systemctl start httpd
+                    echo foo > /var/www/html/index.html
+                    curl http://localhost/ | grep foo
+
 /beakerlib:
     story: As a user I want to easily run beakerlib tests.
     summary: Execute beakerlib tests
diff --git a/spec/steps/report.fmf b/spec/steps/report.fmf
index 446a830faf..566b47897b 100644
--- a/spec/steps/report.fmf
+++ b/spec/steps/report.fmf
@@ -34,9 +34,12 @@ description: |
         * info - test finished and produced only information
           message
         * passed - test finished and passed
+        * warning -  warning - test finished and passed with a warning,
+          like an ``avc`` check failure
         * failed - test finished and failed
         * error - a problem encountered during test execution
 
+
     Note the priority  of test results is as written above, with
     ``info`` having the lowest priority and ``error`` has the
     highest. This is important for ``PLAN_RESULT``.
diff --git a/spec/tests/checks.fmf b/spec/tests/checks.fmf
new file mode 100644
index 0000000000..06ccfa402f
--- /dev/null
+++ b/spec/tests/checks.fmf
@@ -0,0 +1,26 @@
+summary: Specify additional checks which should be enabled by the test runner.
+
+story: As a tester I want to enable additional checks when running my tests.
+
+description: |
+    In some cases we want to run additional checks while running a test. A nice
+    example is a check for unexpected SELinux AVCs produced during the test,
+    this can point to additional issues a user can run into and is very valuable
+    for Red Hat based and Fedora distros.
+
+    These checks can alter the test result and usually provide a separate log
+    output. Note that the value set here can be overridden by L2 metadata.
+
+    Currently the following additional checks are recognized:
+
+    avc
+        SELinux AVCs are inspected during the test time
+    test-inspector
+        Run test inspector
+
+    The default value is ``avc``.
+
+    If the check start's with ``-`` character, it should be disabled.
+
+example: |
+    checks: -avc test-inspector