1. Enhanced Test Selection & Patterns Current Limitation: required-tests: - /exact/test/name - /another/exact/name Apply to __init__.py Improvement: Support Patterns/Wildcards required-tests: - /critical/** # All tests under /critical/ - "*security*" # Any test with "security" in name - pattern: "/smoke/**" platforms: ["rhel", "fedora"] # Conditional requirements Apply to __init__.py 2. Better Error Reporting & Aggregation Current Implementation: # Fails on first missing test raise tmt.utils.ExecuteError(f"Required test '{required_test}' not discovered") Apply to __init__.py Improvement: Comprehensive Error Reporting class RequiredTestsError(tmt.utils.ExecuteError): def __init__(self, missing_tests, skipped_tests, pending_tests): self.missing_tests = missing_tests self.skipped_tests = skipped_tests self.pending_tests = pending_tests # Create detailed error message with suggestions message = self._build_comprehensive_message() super().__init__(message) def _build_comprehensive_message(self): """Build detailed error message with actionable suggestions""" # Implementation would provide: # - List of all failed requirements # - Suggestions for fixing each type of failure # - Links to documentation Apply to __init__.py 3. Hierarchical Requirements & Inheritance Current Limitation: Requirements are flat per phase. # Global requirements (inherited by all plans) discover: required-tests: - /smoke/** # Plan-specific requirements (additive) plans: /integration: discover: required-tests: - /integration/critical/** inherit: true # Inherit global requirements Improvement: Hierarchical Requirements # Allow runtime overrides for emergency situations tmt run --ignore-required-tests tmt run --additional-required-tests="/emergency/fix/**" tmt run --required-tests-level=critical # Only enforce critical level Apply to __init__.py 4. Runtime Flexibility & Overrides Current Limitation: Requirements are static in configuration. Improvement: Runtime Control # Allow runtime overrides for emergency situations tmt run --ignore-required-tests tmt run --additional-required-tests="/emergency/fix/**" tmt run --required-tests-level=critical # Only enforce critical level Apply to __init__.py suggested test: import pytest import tmt import tmt.result import tmt.steps.execute import tmt.utils from tmt.result import ResultOutcome class TestExecuteRequiredTests: """Test cases for required-tests validation in execute step.""" def test_assert_required_tests_executed_success(self, root_logger): """Test that validation passes when all required tests are executed successfully.""" from unittest.mock import Mock # Mock execute step mock_step = Mock() mock_step.plan = Mock() mock_step.plan.provision = Mock() # Mock guest mock_guest = Mock() mock_guest.name = "guest1" mock_step.plan.provision.ready_guests = [mock_guest] execute = tmt.steps.execute.Execute( logger=root_logger, parent=mock_step.plan, name="execute", data=tmt.steps.execute.ExecuteStepData(), ) # Mock required tests from discover step mock_test1 = Mock() mock_test1.test.name = "test1" mock_test1.test.serial_number = 1 mock_test1.test.enabled_on_guest.return_value = True mock_test2 = Mock() mock_test2.test.name = "test2" mock_test2.test.serial_number = 2 mock_test2.test.enabled_on_guest.return_value = True execute.plan.discover = Mock() execute.plan.discover.required_tests = [mock_test1, mock_test2] # Mock results for the required tests result1 = Mock() result1.name = "test1" result1.serial_number = 1 result1.guest = mock_guest result1.result = ResultOutcome.PASS result2 = Mock() result2.name = "test2" result2.serial_number = 2 result2.guest = mock_guest result2.result = ResultOutcome.FAIL # Failed but executed def mock_results_for_tests(test_origins): results = [] for test_origin in test_origins: if test_origin.test.serial_number == 1: results.append((result1, test_origin)) elif test_origin.test.serial_number == 2: results.append((result2, test_origin)) return results execute.results_for_tests = mock_results_for_tests # Should not raise any exception execute._assert_required_tests_executed() def test_assert_required_tests_executed_missing_execution(self, root_logger): """Test that ExecuteError is raised when required test was not executed.""" from unittest.mock import Mock # Mock execute step mock_step = Mock() mock_step.plan = Mock() mock_step.plan.provision = Mock() # Mock guest mock_guest = Mock() mock_guest.name = "guest1" mock_step.plan.provision.ready_guests = [mock_guest] execute = tmt.steps.execute.Execute( logger=root_logger, parent=mock_step.plan, name="execute", data=tmt.steps.execute.ExecuteStepData(), ) # Mock required test from discover step mock_test = Mock() mock_test.test.name = "missing-test" mock_test.test.serial_number = 1 mock_test.test.enabled_on_guest.return_value = True execute.plan.discover = Mock() execute.plan.discover.required_tests = [mock_test] # Mock results_for_tests to return None (test not executed) def mock_results_for_tests(test_origins): return [(None, test_origin) for test_origin in test_origins] execute.results_for_tests = mock_results_for_tests with pytest.raises(tmt.utils.ExecuteError, match="Required test 'missing-test' was not executed"): execute._assert_required_tests_executed() def test_assert_required_tests_executed_pending_status(self, root_logger): """Test that ExecuteError is raised when required test is still pending.""" from unittest.mock import Mock # Mock execute step mock_step = Mock() mock_step.plan = Mock() mock_step.plan.provision = Mock() # Mock guest mock_guest = Mock() mock_guest.name = "guest1" mock_step.plan.provision.ready_guests = [mock_guest] execute = tmt.steps.execute.Execute( logger=root_logger, parent=mock_step.plan, name="execute", data=tmt.steps.execute.ExecuteStepData(), ) # Mock required test from discover step mock_test = Mock() mock_test.test.name = "pending-test" mock_test.test.serial_number = 1 mock_test.test.enabled_on_guest.return_value = True execute.plan.discover = Mock() execute.plan.discover.required_tests = [mock_test] # Mock result with pending status result = Mock() result.name = "pending-test" result.serial_number = 1 result.guest = mock_guest result.result = ResultOutcome.PENDING def mock_results_for_tests(test_origins): return [(result, test_origin) for test_origin in test_origins] execute.results_for_tests = mock_results_for_tests with pytest.raises(tmt.utils.ExecuteError, match="Required test 'pending-test' is still pending"): execute._assert_required_tests_executed() def test_assert_required_tests_executed_skipped_status(self, root_logger): """Test that ExecuteError is raised when required test was skipped.""" from unittest.mock import Mock # Mock execute step mock_step = Mock() mock_step.plan = Mock() mock_step.plan.provision = Mock() # Mock guest mock_guest = Mock() mock_guest.name = "guest1" mock_step.plan.provision.ready_guests = [mock_guest] execute = tmt.steps.execute.Execute( logger=root_logger, parent=mock_step.plan, name="execute", data=tmt.steps.execute.ExecuteStepData(), ) # Mock required test from discover step mock_test = Mock() mock_test.test.name = "skipped-test" mock_test.test.serial_number = 1 mock_test.test.enabled_on_guest.return_value = True execute.plan.discover = Mock() execute.plan.discover.required_tests = [mock_test] # Mock result with skip status result = Mock() result.name = "skipped-test" result.serial_number = 1 result.guest = mock_guest result.result = ResultOutcome.SKIP def mock_results_for_tests(test_origins): return [(result, test_origin) for test_origin in test_origins] execute.results_for_tests = mock_results_for_tests with pytest.raises(tmt.utils.ExecuteError, match="Required test 'skipped-test' was skipped"): execute._assert_required_tests_executed() def test_assert_required_tests_multihost_success(self, root_logger): """Test multihost scenario where required test runs on multiple guests.""" from unittest.mock import Mock # Mock execute step mock_step = Mock() mock_step.plan = Mock() mock_step.plan.provision = Mock() # Mock multiple guests mock_guest1 = Mock() mock_guest1.name = "guest1" mock_guest2 = Mock() mock_guest2.name = "guest2" mock_step.plan.provision.ready_guests = [mock_guest1, mock_guest2] execute = tmt.steps.execute.Execute( logger=root_logger, parent=mock_step.plan, name="execute", data=tmt.steps.execute.ExecuteStepData(), ) # Mock required test that runs on both guests mock_test = Mock() mock_test.test.name = "multihost-test" mock_test.test.serial_number = 1 mock_test.test.enabled_on_guest.return_value = True execute.plan.discover = Mock() execute.plan.discover.required_tests = [mock_test] # Mock results for both guests result1 = Mock() result1.name = "multihost-test" result1.serial_number = 1 result1.guest = mock_guest1 result1.result = ResultOutcome.PASS result2 = Mock() result2.name = "multihost-test" result2.serial_number = 1 result2.guest = mock_guest2 result2.result = ResultOutcome.PASS execute._results = [result1, result2] def mock_results_for_tests(test_origins): return [(result1, test_origins[0]), (result2, test_origins[0])] execute.results_for_tests = mock_results_for_tests # Should not raise any exception execute._assert_required_tests_executed() def test_assert_required_tests_multihost_missing_guest(self, root_logger): """Test multihost scenario where required test missing on one guest.""" from unittest.mock import Mock # Mock execute step mock_step = Mock() mock_step.plan = Mock() mock_step.plan.provision = Mock() # Mock multiple guests mock_guest1 = Mock() mock_guest1.name = "guest1" mock_guest2 = Mock() mock_guest2.name = "guest2" mock_step.plan.provision.ready_guests = [mock_guest1, mock_guest2] execute = tmt.steps.execute.Execute( logger=root_logger, parent=mock_step.plan, name="execute", data=tmt.steps.execute.ExecuteStepData(), ) # Mock required test that should run on both guests mock_test = Mock() mock_test.test.name = "multihost-test" mock_test.test.serial_number = 1 mock_test.test.enabled_on_guest.return_value = True execute.plan.discover = Mock() execute.plan.discover.required_tests = [mock_test] # Mock result for only one guest (missing on guest2) result1 = Mock() result1.name = "multihost-test" result1.serial_number = 1 result1.guest = mock_guest1 result1.result = ResultOutcome.PASS execute._results = [result1] def mock_results_for_tests(test_origins): return [(result1, test_origins[0])] execute.results_for_tests = mock_results_for_tests with pytest.raises(tmt.utils.ExecuteError, match="Required test 'multihost-test' was not executed on guest 'guest2'"): execute._assert_required_tests_executed() def test_assert_required_tests_disabled_on_guest(self, root_logger): """Test that disabled tests on specific guests are handled correctly.""" from unittest.mock import Mock # Mock execute step mock_step = Mock() mock_step.plan = Mock() mock_step.plan.provision = Mock() # Mock multiple guests mock_guest1 = Mock() mock_guest1.name = "guest1" mock_guest2 = Mock() mock_guest2.name = "guest2" mock_step.plan.provision.ready_guests = [mock_guest1, mock_guest2] execute = tmt.steps.execute.Execute( logger=root_logger, parent=mock_step.plan, name="execute", data=tmt.steps.execute.ExecuteStepData(), ) # Mock required test that's only enabled on guest1 mock_test = Mock() mock_test.test.name = "selective-test" mock_test.test.serial_number = 1 def mock_enabled_on_guest(guest): return guest.name == "guest1" mock_test.test.enabled_on_guest = mock_enabled_on_guest execute.plan.discover = Mock() execute.plan.discover.required_tests = [mock_test] # Mock result for only guest1 (should be enough since test is disabled on guest2) result1 = Mock() result1.name = "selective-test" result1.serial_number = 1 result1.guest = mock_guest1 result1.result = ResultOutcome.PASS execute._results = [result1] def mock_results_for_tests(test_origins): return [(result1, test_origins[0])] execute.results_for_tests = mock_results_for_tests # Should not raise any exception since test is disabled on guest2 execute._assert_required_tests_executed() def test_assert_required_tests_no_required_tests(self, root_logger): """Test that validation passes when no tests are required.""" from unittest.mock import Mock # Mock execute step mock_step = Mock() mock_step.plan = Mock() mock_step.plan.provision = Mock() execute = tmt.steps.execute.Execute( logger=root_logger, parent=mock_step.plan, name="execute", data=tmt.steps.execute.ExecuteStepData(), ) # Mock discover step with no required tests execute.plan.discover = Mock() execute.plan.discover.required_tests = [] def mock_results_for_tests(test_origins): return [] execute.results_for_tests = mock_results_for_tests # Should not raise any exception execute._assert_required_tests_executed() def test_assert_required_tests_none_test_origin(self, root_logger): """Test that validation skips tests with None test_origin.""" from unittest.mock import Mock # Mock execute step mock_step = Mock() mock_step.plan = Mock() mock_step.plan.provision = Mock() execute = tmt.steps.execute.Execute( logger=root_logger, parent=mock_step.plan, name="execute", data=tmt.steps.execute.ExecuteStepData(), ) # Mock required test from discover step mock_test = Mock() mock_test.test.name = "test-with-none-origin" mock_test.test.serial_number = 1 execute.plan.discover = Mock() execute.plan.discover.required_tests = [mock_test] # Mock results_for_tests to return None test_origin def mock_results_for_tests(test_origins): result = Mock() result.name = "test-with-none-origin" result.serial_number = 1 return [(result, None)] # None test_origin should be skipped execute.results_for_tests = mock_results_for_tests # Should not raise any exception since None test_origin is skipped execute._assert_required_tests_executed() if __name__ == "__main__": pytest.main([__file__])