pw_unit_test: Add test suite filter
This updates the unit test framework to accept an optional list of test
suites to run. If set, RUN_ALL_TESTS will only run test cases which
match the provided suites. Additionally, tracking of skipped and
disabled tests in a test run is added.
The unit test RPC service is also updated to accept a list of suites in
a test run request.
Change-Id: Ia1aefdcf2314c24431bff75378c30fcdfdd24c27
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/40061
Commit-Queue: Alexei Frolov <frolv@google.com>
Reviewed-by: Keir Mierle <keir@google.com>
Reviewed-by: Joe Ethier <jethier@google.com>
diff --git a/pw_unit_test/BUILD.gn b/pw_unit_test/BUILD.gn
index a693e09..aa80fb9 100644
--- a/pw_unit_test/BUILD.gn
+++ b/pw_unit_test/BUILD.gn
@@ -30,9 +30,9 @@
pw_source_set("pw_unit_test") {
public_configs = [ ":default_config" ]
public_deps = [
- "$dir_pw_polyfill",
- "$dir_pw_preprocessor",
- "$dir_pw_string",
+ dir_pw_polyfill,
+ dir_pw_preprocessor,
+ dir_pw_string,
]
public = [
"public/pw_unit_test/event_handler.h",
@@ -90,6 +90,7 @@
":pw_unit_test",
":unit_test_proto.pwpb",
":unit_test_proto.raw_rpc",
+ "$dir_pw_containers:vector",
]
deps = [ dir_pw_log ]
public = [
diff --git a/pw_unit_test/docs.rst b/pw_unit_test/docs.rst
index 27fda58..5459484 100644
--- a/pw_unit_test/docs.rst
+++ b/pw_unit_test/docs.rst
@@ -109,6 +109,18 @@
return RUN_ALL_TESTS();
}
+Test filtering
+^^^^^^^^^^^^^^
+If using C++17, filters can be set on the test framework to run only a subset of
+the registered unit tests. This is useful when many tests are bundled into a
+single application image.
+
+Currently, only a test suite filter is supported. This is set by calling
+``pw::unit_test::SetTestSuitesToRun`` with a list of suite names.
+
+.. note::
+ Test filtering is only supported in C++17.
+
Build system integration
^^^^^^^^^^^^^^^^^^^^^^^^
``pw_unit_test`` integrates directly into Pigweed's GN build system. To define
@@ -271,3 +283,8 @@
client = HdlcRpcClient(serial.Serial(device, baud), PROTO)
run_tests(client.rpcs())
+
+pw_unit_test.rpc
+^^^^^^^^^^^^^^^^
+.. automodule:: pw_unit_test.rpc
+ :members: EventHandler, run_tests
diff --git a/pw_unit_test/framework.cc b/pw_unit_test/framework.cc
index 69529bc..70b36d3 100644
--- a/pw_unit_test/framework.cc
+++ b/pw_unit_test/framework.cc
@@ -14,6 +14,7 @@
#include "pw_unit_test/framework.h"
+#include <algorithm>
#include <cstring>
namespace pw {
@@ -49,15 +50,23 @@
int Framework::RunAllTests() {
run_tests_summary_.passed_tests = 0;
run_tests_summary_.failed_tests = 0;
+ run_tests_summary_.skipped_tests = 0;
+ run_tests_summary_.disabled_tests = 0;
if (event_handler_ != nullptr) {
event_handler_->RunAllTestsStart();
}
for (const TestInfo* test = tests_; test != nullptr; test = test->next()) {
- if (test->enabled()) {
+ if (ShouldRunTest(*test)) {
test->run();
- } else if (event_handler_ != nullptr) {
- event_handler_->TestCaseDisabled(test->test_case());
+ } else if (!test->enabled()) {
+ run_tests_summary_.disabled_tests++;
+
+ if (event_handler_ != nullptr) {
+ event_handler_->TestCaseDisabled(test->test_case());
+ }
+ } else {
+ run_tests_summary_.skipped_tests++;
}
}
if (event_handler_ != nullptr) {
@@ -115,6 +124,26 @@
event_handler_->TestCaseExpect(current_test_->test_case(), expectation);
}
+bool Framework::ShouldRunTest(const TestInfo& test_info) {
+#if PW_CXX_STANDARD_IS_SUPPORTED(17)
+ // Test suite filtering is only supported if using C++17.
+ if (!test_suites_to_run_.empty()) {
+ std::string_view test_suite(test_info.test_case().suite_name);
+
+ bool suite_matches =
+ std::any_of(test_suites_to_run_.begin(),
+ test_suites_to_run_.end(),
+ [&](auto& name) { return test_suite == name; });
+
+ if (!suite_matches) {
+ return false;
+ }
+ }
+#endif // PW_CXX_STANDARD_IS_SUPPORTED(17)
+
+ return test_info.enabled();
+}
+
bool TestInfo::enabled() const {
constexpr size_t kStringSize = sizeof("DISABLED_") - 1;
return std::strncmp("DISABLED_", test_case().test_name, kStringSize) != 0 &&
diff --git a/pw_unit_test/public/pw_unit_test/event_handler.h b/pw_unit_test/public/pw_unit_test/event_handler.h
index b7b931d..3a5a710 100644
--- a/pw_unit_test/public/pw_unit_test/event_handler.h
+++ b/pw_unit_test/public/pw_unit_test/event_handler.h
@@ -84,6 +84,12 @@
// The number of passed tests among the run tests.
int failed_tests;
+
+ // The number of tests skipped or filtered out.
+ int skipped_tests;
+
+ // The number of disabled tests encountered.
+ int disabled_tests;
};
// An event handler is responsible for collecting and processing the results of
diff --git a/pw_unit_test/public/pw_unit_test/framework.h b/pw_unit_test/public/pw_unit_test/framework.h
index b9db35b..959cef8 100644
--- a/pw_unit_test/public/pw_unit_test/framework.h
+++ b/pw_unit_test/public/pw_unit_test/framework.h
@@ -29,6 +29,8 @@
#include "pw_unit_test/event_handler.h"
#if PW_CXX_STANDARD_IS_SUPPORTED(17)
+#include <string_view>
+
#include "pw_string/string_builder.h"
#endif // PW_CXX_STANDARD_IS_SUPPORTED(17)
@@ -155,7 +157,10 @@
constexpr Framework()
: current_test_(nullptr),
current_result_(TestResult::kSuccess),
- run_tests_summary_{.passed_tests = 0, .failed_tests = 0},
+ run_tests_summary_{.passed_tests = 0,
+ .failed_tests = 0,
+ .skipped_tests = 0,
+ .disabled_tests = 0},
exit_status_(0),
event_handler_(nullptr),
memory_pool_() {}
@@ -177,6 +182,17 @@
// are sent to the registered event handler, if any.
int RunAllTests();
+#if PW_CXX_STANDARD_IS_SUPPORTED(17)
+ // Only run test suites whose names are included in the provided list during
+ // the next test run. This is C++17 only; older versions of C++ will run all
+ // non-disabled tests.
+ void SetTestSuitesToRun(std::span<std::string_view> test_suites) {
+ test_suites_to_run_ = test_suites;
+ }
+#endif // PW_CXX_STANDARD_IS_SUPPORTED(17)
+
+ bool ShouldRunTest(const TestInfo& test_info);
+
// Constructs an instance of a unit test class and runs the test.
//
// Tests are constructed within a static memory pool at run time instead of
@@ -280,6 +296,10 @@
// Handler to which to dispatch test events.
EventHandler* event_handler_;
+#if PW_CXX_STANDARD_IS_SUPPORTED(17)
+ std::span<std::string_view> test_suites_to_run_;
+#endif // PW_CXX_STANDARD_IS_SUPPORTED(17)
+
// Memory region in which to construct test case classes as they are run.
// TODO(frolv): Make the memory pool size configurable.
static constexpr size_t kTestMemoryPoolSizeBytes = 16384;
@@ -376,6 +396,12 @@
virtual void PigweedTestBody() = 0;
};
+#if PW_CXX_STANDARD_IS_SUPPORTED(17)
+inline void SetTestSuitesToRun(std::span<std::string_view> test_suites) {
+ internal::Framework::Get().SetTestSuitesToRun(test_suites);
+}
+#endif // PW_CXX_STANDARD_IS_SUPPORTED(17)
+
} // namespace unit_test
} // namespace pw
diff --git a/pw_unit_test/pw_unit_test_proto/unit_test.proto b/pw_unit_test/pw_unit_test_proto/unit_test.proto
index ace948e..4a10e8b 100644
--- a/pw_unit_test/pw_unit_test_proto/unit_test.proto
+++ b/pw_unit_test/pw_unit_test_proto/unit_test.proto
@@ -81,6 +81,9 @@
message TestRunRequest {
// Whether to send expectation events for successful checks.
bool report_passed_expectations = 1;
+
+ // Optional list of test suites to run.
+ repeated string test_suite = 2;
}
service UnitTest {
diff --git a/pw_unit_test/py/pw_unit_test/rpc.py b/pw_unit_test/py/pw_unit_test/rpc.py
index d1294d5..9fc8280 100644
--- a/pw_unit_test/py/pw_unit_test/rpc.py
+++ b/pw_unit_test/py/pw_unit_test/rpc.py
@@ -113,6 +113,7 @@
def run_tests(rpcs: pw_rpc.client.Services,
report_passed_expectations: bool = False,
+ test_suites: Iterable[str] = (),
event_handlers: Iterable[EventHandler] = (
LoggingEventHandler(), ),
timeout_s: OptionalTimeout = UseDefault.VALUE) -> bool:
@@ -126,6 +127,7 @@
test_responses = iter(
unit_test_service.Run(
report_passed_expectations=report_passed_expectations,
+ test_suite=test_suites,
pw_rpc_timeout_s=timeout_s))
# Read the first response, which must be a test_run_start message.
diff --git a/pw_unit_test/unit_test_service.cc b/pw_unit_test/unit_test_service.cc
index 86fb37c..b43537f 100644
--- a/pw_unit_test/unit_test_service.cc
+++ b/pw_unit_test/unit_test_service.cc
@@ -14,6 +14,7 @@
#include "pw_unit_test/unit_test_service.h"
+#include "pw_containers/vector.h"
#include "pw_log/log.h"
#include "pw_protobuf/decoder.h"
#include "pw_unit_test/framework.h"
@@ -26,6 +27,11 @@
writer_ = std::move(writer);
verbose_ = false;
+ // List of test suite names to run. The string views in this vector point to
+ // data in the raw protobuf request message, so it is only valid for the
+ // duration of this function.
+ pw::Vector<std::string_view, 16> suites_to_run;
+
protobuf::Decoder decoder(request);
Status status;
@@ -34,6 +40,24 @@
case TestRunRequest::Fields::REPORT_PASSED_EXPECTATIONS:
decoder.ReadBool(&verbose_);
break;
+
+ case TestRunRequest::Fields::TEST_SUITE: {
+ std::string_view suite_name;
+ if (!decoder.ReadString(&suite_name).ok()) {
+ break;
+ }
+
+ if (!suites_to_run.full()) {
+ suites_to_run.push_back(suite_name);
+ } else {
+ PW_LOG_ERROR("Maximum of %d test suite filters supported",
+ suites_to_run.max_size());
+ writer_.Finish(Status::InvalidArgument());
+ return;
+ }
+
+ break;
+ }
}
}
@@ -42,12 +66,19 @@
return;
}
- PW_LOG_DEBUG("Starting unit test run");
+ PW_LOG_INFO("Starting unit test run");
- pw::unit_test::RegisterEventHandler(&handler_);
+ RegisterEventHandler(&handler_);
+ SetTestSuitesToRun(suites_to_run);
+ PW_LOG_DEBUG("%u test suite filters applied",
+ static_cast<unsigned>(suites_to_run.size()));
+
RUN_ALL_TESTS();
- pw::unit_test::RegisterEventHandler(nullptr);
- PW_LOG_DEBUG("Unit test run complete");
+
+ RegisterEventHandler(nullptr);
+ SetTestSuitesToRun({});
+
+ PW_LOG_INFO("Unit test run complete");
writer_.Finish();
}
@@ -62,6 +93,8 @@
TestRunEnd::Encoder test_run_end = event.GetTestRunEndEncoder();
test_run_end.WritePassed(summary.passed_tests);
test_run_end.WriteFailed(summary.failed_tests);
+ test_run_end.WriteSkipped(summary.skipped_tests);
+ test_run_end.WriteDisabled(summary.disabled_tests);
});
}
