Expand situations in which 13.3 will terminate test runs

Detect critical failures earlier in test run and bail
to provide engineering with VM in failed state.

tests/api2/test_001_ssh.py

@@ -10,7 +10,7 @@ from time import sleep
 from pytest_dependency import depends
 apifolder = os.getcwd()
 sys.path.append(apifolder)
-from functions import PUT, POST, GET, is_agent_setup, if_key_listed, SSH_TEST
+from functions import fail, PUT, POST, GET, is_agent_setup, if_key_listed, SSH_TEST
 from auto_config import sshKey, user, password, ha
 
 if "controller1_ip" in os.environ:
@@ -52,7 +52,8 @@ def test_05_Checking_if_ssh_is_running():
 def test_06_test_ssh():
     cmd = 'ls -la'
     results = SSH_TEST(cmd, user, password, ip)
-    assert results['result'] is True, f'out: {results["output"]}, err: {results["stderr"]}'
+    if not results['result']:
+        fail(f'Failed to set up SSH output: {results["output"]}, err: {results["stderr"]}')
 
 
 def test_07_Ensure_ssh_agent_is_setup():

tests/api2/test_003_network.py

@@ -10,7 +10,7 @@ from pytest_dependency import depends
 apifolder = os.getcwd()
 sys.path.append(apifolder)
 from auto_config import ha, interface, user, password
-from functions import GET, PUT, SSH_TEST
+from functions import fail, GET, PUT, SSH_TEST
 
 # Only run HA test on HA and vice versa
 if ha and "domain" in os.environ:
@@ -61,7 +61,9 @@ else:
         for num, nameserver in enumerate(nameservers, start=1):
             payload[f'nameserver{num}'] = nameserver
         results = PUT("/network/configuration/", payload)
-        assert results.status_code == 200, results.text
+        if results.status_code != 200:
+            fail(f'Network setup failed with config {payload}: {results.text}')
+
         assert isinstance(results.json(), dict), results.text
 
     @pytest.mark.parametrize('dkeys', ["domain", "hostname", "ipv4gateway", "nameserver1"])

tests/api2/test_007_systemdataset.py

@@ -6,7 +6,7 @@ import os
 from pytest_dependency import depends
 apifolder = os.getcwd()
 sys.path.append(apifolder)
-from functions import POST, GET, PUT, SSH_TEST, wait_on_job
+from functions import fail, POST, GET, PUT, SSH_TEST, wait_on_job
 from auto_config import ha, dev_test, hostname, user, password
 # comment pytestmark for development testing with --dev-test
 pytestmark = pytest.mark.skipif(dev_test, reason='Skipping for test development testing')
@@ -121,7 +121,9 @@ def test_05_verify_the_first_pool_created_become_sysds(request, pool_data):
     assert results.status_code == 200, results.text
     job_id = results.json()
     job_status = wait_on_job(job_id, 180)
-    assert job_status['state'] == 'SUCCESS', str(job_status['results'])
+    if job_status['state'] != SUCCESS:
+        fail(f'Failed to create first pool: {job_status["state"]}: {job_status["results"]}')
+
     pool_data['first_pool'] = job_status['results']['result']
 
     results = GET("/systemdataset/")

tests/api2/test_008_pool.py

@@ -44,6 +44,8 @@ def test_02_wipe_all_pool_disk():
         job_id = results.json()
         job_status = wait_on_job(job_id, 180)
         assert job_status['state'] == 'SUCCESS', str(job_status['results'])
+        if job_status['state'] != 'SUCCESS':
+            fail(f'{disk}: failed to wipe disk: {job_status["state"]}: {job_status["results"]}')
 
 
 @pytest.mark.dependency(name="pool_04")
@@ -65,7 +67,7 @@ def test_04_creating_a_pool(pool_data):
     job_id = results.json()
     job_status = wait_on_job(job_id, 180)
     if job_status['state'] != 'SUCCESS':
-        fail(f'Failed to create first pool: {job_status["results"]}')
+        fail(f'Failed to create first pool: {job_status["state"]}: {job_status["results"]}')
 
     pool_data['id'] = job_status['results']['result']['id']