From ed01ad3a2fac8fff63f441ead4f8653da053c65b Mon Sep 17 00:00:00 2001 From: Rae Moar Date: Thu, 1 Jul 2021 17:52:31 +0000 Subject: [PATCH 1/4] kunit: tool: Fix error messages for cases of no tests and wrong TAP header This patch addresses misleading error messages reported by kunit_tool in two cases. First, in the case of TAP output having an incorrect header format or missing a header, the parser used to output an error message of 'no tests run!'. Now the parser outputs an error message of 'could not parse test results!'. As an example: Before: $ ./tools/testing/kunit/kunit.py parse /dev/null [ERROR] no tests run! ... After: $ ./tools/testing/kunit/kunit.py parse /dev/null [ERROR] could not parse test results! ... Second, in the case of TAP output with the correct header but no tests, the parser used to output an error message of 'could not parse test results!'. Now the parser outputs an error message of 'no tests run!'. As an example: Before: $ echo -e 'TAP version 14\n1..0' | ./tools/testing/kunit/kunit.py parse [ERROR] could not parse test results! After: $ echo -e 'TAP version 14\n1..0' | ./tools/testing/kunit/kunit.py parse [ERROR] no tests run! Additionally, this patch also corrects the tests in kunit_tool_test.py and adds a test to check the error in the case of TAP output with the correct header but no tests. Signed-off-by: Rae Moar Reviewed-by: David Gow Reviewed-by: Daniel Latypov Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit_parser.py | 6 ++++-- tools/testing/kunit/kunit_tool_test.py | 16 +++++++++++++--- ...st_is_test_passed-no_tests_run_no_header.log} | 0 ...t_is_test_passed-no_tests_run_with_header.log | 2 ++ 4 files changed, 19 insertions(+), 5 deletions(-) rename tools/testing/kunit/test_data/{test_is_test_passed-no_tests_run.log => test_is_test_passed-no_tests_run_no_header.log} (100%) create mode 100644 tools/testing/kunit/test_data/test_is_test_passed-no_tests_run_with_header.log diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py index c3c524b79db8..b88db3f51dc5 100644 --- a/tools/testing/kunit/kunit_parser.py +++ b/tools/testing/kunit/kunit_parser.py @@ -338,9 +338,11 @@ def bubble_up_suite_errors(test_suites: Iterable[TestSuite]) -> TestStatus: def parse_test_result(lines: LineStream) -> TestResult: consume_non_diagnostic(lines) if not lines or not parse_tap_header(lines): - return TestResult(TestStatus.NO_TESTS, [], lines) + return TestResult(TestStatus.FAILURE_TO_PARSE_TESTS, [], lines) expected_test_suite_num = parse_test_plan(lines) - if not expected_test_suite_num: + if expected_test_suite_num == 0: + return TestResult(TestStatus.NO_TESTS, [], lines) + elif expected_test_suite_num is None: return TestResult(TestStatus.FAILURE_TO_PARSE_TESTS, [], lines) test_suites = [] for i in range(1, expected_test_suite_num + 1): diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index bdae0e5f6197..75045aa0f8a1 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -157,8 +157,18 @@ class KUnitParserTest(unittest.TestCase): kunit_parser.TestStatus.FAILURE, result.status) + def test_no_header(self): + empty_log = test_data_path('test_is_test_passed-no_tests_run_no_header.log') + with open(empty_log) as file: + result = kunit_parser.parse_run_tests( + kunit_parser.extract_tap_lines(file.readlines())) + self.assertEqual(0, len(result.suites)) + self.assertEqual( + kunit_parser.TestStatus.FAILURE_TO_PARSE_TESTS, + result.status) + def test_no_tests(self): - empty_log = test_data_path('test_is_test_passed-no_tests_run.log') + empty_log = test_data_path('test_is_test_passed-no_tests_run_with_header.log') with open(empty_log) as file: result = kunit_parser.parse_run_tests( kunit_parser.extract_tap_lines(file.readlines())) @@ -173,7 +183,7 @@ class KUnitParserTest(unittest.TestCase): with open(crash_log) as file: result = kunit_parser.parse_run_tests( kunit_parser.extract_tap_lines(file.readlines())) - print_mock.assert_any_call(StrContains('no tests run!')) + print_mock.assert_any_call(StrContains('could not parse test results!')) print_mock.stop() file.close() @@ -309,7 +319,7 @@ class KUnitJsonTest(unittest.TestCase): result["sub_groups"][1]["test_cases"][0]) def test_no_tests_json(self): - result = self._json_for('test_is_test_passed-no_tests_run.log') + result = self._json_for('test_is_test_passed-no_tests_run_with_header.log') self.assertEqual(0, len(result['sub_groups'])) class StrContains(str): diff --git a/tools/testing/kunit/test_data/test_is_test_passed-no_tests_run.log b/tools/testing/kunit/test_data/test_is_test_passed-no_tests_run_no_header.log similarity index 100% rename from tools/testing/kunit/test_data/test_is_test_passed-no_tests_run.log rename to tools/testing/kunit/test_data/test_is_test_passed-no_tests_run_no_header.log diff --git a/tools/testing/kunit/test_data/test_is_test_passed-no_tests_run_with_header.log b/tools/testing/kunit/test_data/test_is_test_passed-no_tests_run_with_header.log new file mode 100644 index 000000000000..5f48ee659d40 --- /dev/null +++ b/tools/testing/kunit/test_data/test_is_test_passed-no_tests_run_with_header.log @@ -0,0 +1,2 @@ +TAP version 14 +1..0 From a72fa6304342a3ad749955fd9007695f761e12d6 Mon Sep 17 00:00:00 2001 From: Daniel Latypov Date: Wed, 23 Jun 2021 13:17:21 -0700 Subject: [PATCH 2/4] Documentation: kunit: drop obsolete note about uml_abort for coverage Commit b6d5799b0b58 ("kunit: Add 'kunit_shutdown' option") changes KUnit to call kernel_halt() by default when done testing. This fixes the issue with not having .gcda files due to not calling atexit() handlers, and therefore we can stop recommending people manually tweak UML code. The need to use older versions of GCC (<=6) remains however, due to linktime issues, same as before. Note: There also might still be issues with .gcda files as well in newer versions. Signed-off-by: Daniel Latypov Reviewed-by: David Gow Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- Documentation/dev-tools/kunit/running_tips.rst | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/Documentation/dev-tools/kunit/running_tips.rst b/Documentation/dev-tools/kunit/running_tips.rst index 7d99386cf94a..d1626d548fa5 100644 --- a/Documentation/dev-tools/kunit/running_tips.rst +++ b/Documentation/dev-tools/kunit/running_tips.rst @@ -86,19 +86,7 @@ Generating code coverage reports under UML .. note:: TODO(brendanhiggins@google.com): There are various issues with UML and versions of gcc 7 and up. You're likely to run into missing ``.gcda`` - files or compile errors. We know one `faulty GCC commit - `_ - but not how we'd go about getting this fixed. The compile errors still - need some investigation. - -.. note:: - TODO(brendanhiggins@google.com): for recent versions of Linux - (5.10-5.12, maybe earlier), there's a bug with gcov counters not being - flushed in UML. This translates to very low (<1%) reported coverage. This is - related to the above issue and can be worked around by replacing the - one call to ``uml_abort()`` (it's in ``os_dump_core()``) with a plain - ``exit()``. - + files or compile errors. This is different from the "normal" way of getting coverage information that is documented in Documentation/dev-tools/gcov.rst. From 58c965d84e14196e762c803c50c7b207a9e352bc Mon Sep 17 00:00:00 2001 From: Daniel Latypov Date: Wed, 23 Jun 2021 12:09:19 -0700 Subject: [PATCH 3/4] kunit: tool: remove unnecessary "annotations" import The import was working around the fact "tuple[T]" was used instead of typing.Tuple[T]. Convert it to use type.Tuple to be consistent with how the rest of the code is anotated. Signed-off-by: Daniel Latypov Reviewed-by: David Gow Reviewed-by: Brendan Higgins Tested-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit_kernel.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py index 90bc007f1f93..2c6f916ccbaf 100644 --- a/tools/testing/kunit/kunit_kernel.py +++ b/tools/testing/kunit/kunit_kernel.py @@ -6,15 +6,13 @@ # Author: Felix Guo # Author: Brendan Higgins -from __future__ import annotations import importlib.util import logging import subprocess import os import shutil import signal -from typing import Iterator -from typing import Optional +from typing import Iterator, Optional, Tuple from contextlib import ExitStack @@ -208,7 +206,7 @@ def get_source_tree_ops(arch: str, cross_compile: Optional[str]) -> LinuxSourceT raise ConfigError(arch + ' is not a valid arch') def get_source_tree_ops_from_qemu_config(config_path: str, - cross_compile: Optional[str]) -> tuple[ + cross_compile: Optional[str]) -> Tuple[ str, LinuxSourceTreeOperations]: # The module name/path has very little to do with where the actual file # exists (I learned this through experimentation and could not find it From df4b0807ca1a62822342d404b863eff933d15762 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Mon, 12 Jul 2021 19:52:58 +0000 Subject: [PATCH 4/4] kunit: tool: Assert the version requirement Commit 87c9c1631788 ("kunit: tool: add support for QEMU") on the 'next' tree adds 'from __future__ import annotations' in 'kunit_kernel.py'. Because it is supported on only >=3.7 Python, people using older Python will get below error: Traceback (most recent call last): File "./tools/testing/kunit/kunit.py", line 20, in import kunit_kernel File "/home/sjpark/linux/tools/testing/kunit/kunit_kernel.py", line 9 from __future__ import annotations ^ SyntaxError: future feature annotations is not defined This commit adds a version assertion in 'kunit.py', so that people get more explicit error message like below: Traceback (most recent call last): File "./tools/testing/kunit/kunit.py", line 15, in assert sys.version_info >= (3, 7), "Python version is too old" AssertionError: Python version is too old Signed-off-by: SeongJae Park Acked-by: Daniel Latypov Reviewed-by: Brendan Higgins Signed-off-by: Shuah Khan --- tools/testing/kunit/kunit.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index be8d8d4a4e08..6276ce0c0196 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -12,6 +12,8 @@ import sys import os import time +assert sys.version_info >= (3, 7), "Python version is too old" + from collections import namedtuple from enum import Enum, auto