Extract utils for running test binaries from fuctional_test to separate library. PiperOrigin-RevId: 687437701
diff --git a/e2e_tests/BUILD b/e2e_tests/BUILD index 868ac5f..ebd11cb 100644 --- a/e2e_tests/BUILD +++ b/e2e_tests/BUILD
@@ -18,6 +18,22 @@ licenses(["notice"]) +cc_library( + name = "test_binary_util", + testonly = 1, + srcs = ["test_binary_util.cc"], + hdrs = ["test_binary_util.h"], + deps = [ + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + "@com_google_absl//absl/time", + "@com_google_fuzztest//fuzztest:flag_name", + "@com_google_fuzztest//fuzztest:subprocess", + ], +) + # Can be run with or without `--config=fuzztest`. When running without, the # fuzzing mode tests are skipped. cc_test( @@ -39,6 +55,7 @@ }), shard_count = 50, deps = [ + ":test_binary_util", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format",
diff --git a/e2e_tests/CMakeLists.txt b/e2e_tests/CMakeLists.txt index 2ff4c60..7e9a9d8 100644 --- a/e2e_tests/CMakeLists.txt +++ b/e2e_tests/CMakeLists.txt
@@ -4,6 +4,24 @@ add_subdirectory(testdata) +fuzztest_cc_library( + NAME + test_binary_util + HDRS + test_binary_util.h + SRCS + test_binary_util.cc + DEPS + absl::check + absl::flat_hash_map + absl::strings + absl::string_view + absl::time + fuzztest_flag_name + fuzztest_subprocess + TESTONLY +) + add_executable( functional_test functional_test.cc @@ -17,6 +35,7 @@ fuzztest_printer fuzztest_serialization fuzztest_subprocess + fuzztest_test_binary_util fuzztest_type_support re2 absl::flat_hash_map
diff --git a/e2e_tests/functional_test.cc b/e2e_tests/functional_test.cc index f327451..9b0aed5 100644 --- a/e2e_tests/functional_test.cc +++ b/e2e_tests/functional_test.cc
@@ -27,7 +27,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/container/flat_hash_map.h" -#include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" @@ -35,6 +34,7 @@ #include "absl/time/clock.h" #include "absl/time/time.h" #include "./domain_tests/domain_testing.h" +#include "./e2e_tests/test_binary_util.h" #include "./fuzztest/internal/io.h" #include "./fuzztest/internal/logging.h" #include "./fuzztest/internal/printer.h" @@ -46,8 +46,6 @@ namespace fuzztest::internal { namespace { -#define FUZZTEST_FLAG_PREFIX_ "" - using ::fuzztest::domain_implementor::PrintMode; using ::testing::_; using ::testing::AllOf; @@ -68,26 +66,6 @@ constexpr absl::string_view kDefaultTargetBinary = "testdata/fuzz_tests_for_functional_testing"; -std::string CreateFuzzTestFlag(absl::string_view flag_name, - absl::string_view flag_value) { - return absl::StrCat("--", FUZZTEST_FLAG_PREFIX_, flag_name, "=", flag_value); -} - -std::string BinaryPath(const absl::string_view name) { - const auto test_srcdir = absl::NullSafeStringView(getenv("TEST_SRCDIR")); - FUZZTEST_INTERNAL_CHECK_PRECONDITION( - !test_srcdir.empty(), - "Please set TEST_SRCDIR to non-empty value or use bazel to run the " - "test."); - const std::string binary_path = absl::StrCat( - test_srcdir, "/_main/e2e_tests/", name, - absl::EndsWith(name, ".stripped") ? "" : ".stripped"); - - FUZZTEST_INTERNAL_CHECK(std::filesystem::exists(binary_path), - absl::StrCat("Can't find ", binary_path)); - return binary_path; -} - absl::flat_hash_map<std::string, std::string> WithTestSanitizerOptions( absl::flat_hash_map<std::string, std::string> env) { if (!env.contains("ASAN_OPTIONS")) @@ -97,22 +75,6 @@ return env; } -class TempDir { - public: - TempDir() { - dirname_ = "/tmp/replay_test_XXXXXX"; - dirname_ = mkdtemp(dirname_.data()); - EXPECT_TRUE(std::filesystem::is_directory(dirname_)); - } - - const std::string& dirname() const { return dirname_; } - - ~TempDir() { std::filesystem::remove_all(dirname_); } - - private: - std::string dirname_; -}; - class UnitTestModeTest : public ::testing::Test { protected: RunResults Run( @@ -120,15 +82,11 @@ absl::string_view target_binary = kDefaultTargetBinary, const absl::flat_hash_map<std::string, std::string>& env = {}, const absl::flat_hash_map<std::string, std::string>& fuzzer_flags = {}) { - std::vector<std::string> commandline = { + return RunBinary( BinaryPath(target_binary), - absl::StrCat("--", GTEST_FLAG_PREFIX_, "filter=", test_filter)}; - - for (const auto& flag : fuzzer_flags) { - commandline.push_back(CreateFuzzTestFlag(flag.first, flag.second)); - } - return RunCommand(commandline, WithTestSanitizerOptions(env), - absl::Minutes(10)); + {.flags = {{GTEST_FLAG_PREFIX_ "filter", std::string(test_filter)}}, + .fuzztest_flags = fuzzer_flags, + .env = WithTestSanitizerOptions(env)}); } }; @@ -673,21 +631,6 @@ EXPECT_EQ(val, other_val); } -std::string CentipedePath() { - const auto test_srcdir = absl::NullSafeStringView(getenv("TEST_SRCDIR")); - FUZZTEST_INTERNAL_CHECK_PRECONDITION( - !test_srcdir.empty(), - "Please set TEST_SRCDIR to non-empty value or use bazel to run the " - "test."); - const std::string binary_path = absl::StrCat( - test_srcdir, - "/_main/centipede/centipede_uninstrumented"); - - FUZZTEST_INTERNAL_CHECK(std::filesystem::exists(binary_path), - absl::StrCat("Can't find ", binary_path)); - return binary_path; -} - // Tests for the FuzzTest command line interface. class GenericCommandLineInterfaceTest : public ::testing::Test { protected: @@ -698,14 +641,11 @@ absl::string_view binary = kDefaultTargetBinary, const absl::flat_hash_map<std::string, std::string>& non_fuzztest_flags = {}) { - std::vector<std::string> args = {BinaryPath(binary)}; - for (const auto& [key, value] : flags) { - args.push_back(CreateFuzzTestFlag(key, value)); - } - for (const auto& [key, value] : non_fuzztest_flags) { - args.push_back(absl::StrCat("--", key, "=", value)); - } - return RunCommand(args, WithTestSanitizerOptions(env), timeout); + return RunBinary(BinaryPath(binary), + RunOptions{.flags = non_fuzztest_flags, + .fuzztest_flags = flags, + .env = WithTestSanitizerOptions(env), + .timeout = timeout}); } }; @@ -1132,10 +1072,10 @@ {{"FUZZTEST_STACK_LIMIT", "512000"}}); EXPECT_THAT(std_err, HasSubstr("argument 0: ")); EXPECT_THAT(std_err, - HasSubstr("Stack limit is set by FUZZTEST_STACK_LIMIT env var " - "- this is going to be deprecated soon. Consider " - "switching to --" FUZZTEST_FLAG_PREFIX_ - "stack_limit_kb flag.")); + HasSubstr(absl::StrCat( + "Stack limit is set by FUZZTEST_STACK_LIMIT env var - this " + "is going to be deprecated soon. Consider switching to ", + CreateFuzzTestFlag("stack_limit_kb", ""), " flag."))); ExpectStackLimitExceededMessage(std_err, 512000); EXPECT_THAT(status, Eq(Signal(SIGABRT))); } @@ -1279,20 +1219,22 @@ RunResults Run(absl::string_view test_name, int iterations) { if (GetParam().multi_process) { TempDir workdir; - return RunCommand( - {CentipedePath(), "--print_runner_log", "--exit_on_crash", - absl::StrCat("--workdir=", workdir.dirname()), - absl::StrCat("--binary=", BinaryPath(kDefaultTargetBinary), " ", - CreateFuzzTestFlag("fuzz", test_name)), - absl::StrCat("--num_runs=", iterations)}, - /*environment=*/{}, - /*timeout=*/absl::InfiniteDuration()); + return RunBinary( + CentipedePath(), + {.flags = {{"print_runner_log", "true"}, + {"exit_on_crash", "true"}, + {"workdir", workdir.dirname()}, + {"binary", + absl::StrCat(BinaryPath(kDefaultTargetBinary), " ", + CreateFuzzTestFlag("fuzz", test_name))}, + {"num_runs", absl::StrCat(iterations)}}, + .timeout = absl::InfiniteDuration()}); } else { - return RunCommand( - {BinaryPath(kDefaultTargetBinary), - CreateFuzzTestFlag("fuzz", test_name)}, - {{"FUZZTEST_MAX_FUZZING_RUNS", absl::StrCat(iterations)}}, - /*timeout=*/absl::InfiniteDuration()); + return RunBinary( + BinaryPath(kDefaultTargetBinary), + {.fuzztest_flags = {{"fuzz", std::string(test_name)}}, + .env = {{"FUZZTEST_MAX_FUZZING_RUNS", absl::StrCat(iterations)}}, + .timeout = absl::InfiniteDuration()}); } } @@ -1423,17 +1365,22 @@ env = WithTestSanitizerOptions(std::move(env)); if (GetParam().multi_process) { TempDir workdir; - return RunCommand( - {CentipedePath(), "--exit_on_crash", "--timeout_per_input=0", - absl::StrCat("--stop_at=", absl::Now() + timeout), - absl::StrCat("--workdir=", workdir.dirname()), - absl::StrCat("--binary=", BinaryPath(target_binary), " ", - CreateFuzzTestFlag("fuzz", test_name))}, - env, timeout + absl::Seconds(10)); + return RunBinary( + CentipedePath(), + {.flags = {{"exit_on_crash", "true"}, + {"timeout_per_input", "0"}, + {"stop_at", absl::StrCat(absl::Now() + timeout)}, + {"workdir", workdir.dirname()}, + {"binary", + absl::StrCat(BinaryPath(target_binary), " ", + CreateFuzzTestFlag("fuzz", test_name))}}, + .env = std::move(env), + .timeout = timeout + absl::Seconds(10)}); } else { - return RunCommand( - {BinaryPath(target_binary), CreateFuzzTestFlag("fuzz", test_name)}, - env, timeout); + return RunBinary(BinaryPath(target_binary), + {.fuzztest_flags = {{"fuzz", std::string(test_name)}}, + .env = std::move(env), + .timeout = timeout}); } }
diff --git a/e2e_tests/test_binary_util.cc b/e2e_tests/test_binary_util.cc new file mode 100644 index 0000000..cf8a61b --- /dev/null +++ b/e2e_tests/test_binary_util.cc
@@ -0,0 +1,90 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "./e2e_tests/test_binary_util.h" + +#include <cstdlib> +#include <filesystem> // NOLINT +#include <string> +#include <system_error> // NOLINT +#include <vector> + +#include "absl/log/check.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "./fuzztest/internal/flag_name.h" +#include "./fuzztest/internal/subprocess.h" + +namespace fuzztest::internal { +namespace { + +// Returns the full path for `relative_path` given relative to the FuzzTest +// root. +std::string GetFullPath(const std::filesystem::path& relative_path) { + const auto test_srcdir = absl::NullSafeStringView(std::getenv("TEST_SRCDIR")); + CHECK(!test_srcdir.empty()) << "Please set TEST_SRCDIR to non-empty value or " + "use bazel to run the test."; + const std::string full_path = + std::filesystem::path(test_srcdir) / "_main" + / relative_path; + CHECK(std::filesystem::exists(full_path)) << "Can't find " << full_path; + return full_path; +} + +} // namespace + +TempDir::TempDir() { + std::error_code error; + dirname_ = std::filesystem::temp_directory_path(error) / "temp_dir_XXXXXX"; + CHECK(!error) << "Failed to get the root temp directory path: " << error; + dirname_ = mkdtemp(dirname_.data()); + CHECK(std::filesystem::is_directory(dirname_)); +} + +TempDir::~TempDir() { std::filesystem::remove_all(dirname_); } + +std::string CreateFuzzTestFlag(absl::string_view flag_name, + absl::string_view flag_value) { + return absl::StrCat("--", FUZZTEST_FLAG_PREFIX, flag_name, + (flag_value.empty() ? "" : "="), flag_value); +} + +std::string BinaryPath(const absl::string_view relative_path) { + return GetFullPath( + std::filesystem::path("e2e_tests") / + absl::StrCat(relative_path, absl::EndsWith(relative_path, ".stripped") + ? "" + : ".stripped")); +} + +std::string CentipedePath() { + return GetFullPath(std::filesystem::path("centipede") / + "centipede_uninstrumented"); +} + +RunResults RunBinary(absl::string_view binary_path, const RunOptions& options) { + std::vector<std::string> args; + args.reserve(1 + options.fuzztest_flags.size() + options.flags.size()); + args.push_back(std::string(binary_path)); + for (const auto& [key, value] : options.fuzztest_flags) { + args.push_back(CreateFuzzTestFlag(key, value)); + } + for (const auto& [key, value] : options.flags) { + args.push_back(absl::StrCat("--", key, "=", value)); + } + return RunCommand(args, options.env, options.timeout); +} + +} // namespace fuzztest::internal
diff --git a/e2e_tests/test_binary_util.h b/e2e_tests/test_binary_util.h new file mode 100644 index 0000000..85042bd --- /dev/null +++ b/e2e_tests/test_binary_util.h
@@ -0,0 +1,79 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef FUZZTEST_E2E_TESTS_TEST_BINARY_UTIL_H_ +#define FUZZTEST_E2E_TESTS_TEST_BINARY_UTIL_H_ + +#include <string> + +#include "absl/container/flat_hash_map.h" +#include "absl/strings/string_view.h" +#include "absl/time/time.h" +#include "./fuzztest/internal/subprocess.h" + +namespace fuzztest::internal { + +// A helper class for creating a temporary directory. Removes the directory +// when it goes out of scope. +class TempDir { + public: + TempDir(); + ~TempDir(); + + TempDir(const TempDir& other) = delete; + TempDir& operator=(const TempDir& other) = delete; + + const std::string& dirname() const { return dirname_; } + + private: + std::string dirname_; +}; + +// Returns a string of the form "--flag_name=flag_value" (or "--flag_name" if +// `flag_value` is empty), but with the flag name transformed to take into +// account any internal flag prefix. +std::string CreateFuzzTestFlag(absl::string_view flag_name, + absl::string_view flag_value); + +// Returns the full path to the test binary based on the path relative to the +// e2e_tests directory. The test binary's name must end with the suffix +// ".stripped", but the function doesn't require the suffix in `relative_path`. +// If the suffix is missing, it will be added. +std::string BinaryPath(absl::string_view relative_path); + +// Returns the full path to the Centipede binary. +std::string CentipedePath(); + +struct RunOptions { + // General flags to pass to the binary. Useful when passing flags to + // binaries like Centipede. + absl::flat_hash_map<std::string, std::string> flags; + // Flags to pass to a FuzzTest binary. These should be given without the + // internal flag prefix, e.g., just ("fuzz_for", "1s"). + absl::flat_hash_map<std::string, std::string> fuzztest_flags; + // Environment variables to pass to the binary. + absl::flat_hash_map<std::string, std::string> env; + // Duration after which the binary will be terminated. + absl::Duration timeout = absl::Minutes(10); +}; + +// Runs the binary given by its full `binary_path` in a subprocess and returns +// the results. The `options` parameter can be used to pass flags and +// environment variables to the binary. +RunResults RunBinary(absl::string_view binary_path, + const RunOptions& options = {}); + +} // namespace fuzztest::internal + +#endif // FUZZTEST_E2E_TESTS_TEST_BINARY_UTIL_H_
diff --git a/fuzztest/CMakeLists.txt b/fuzztest/CMakeLists.txt index db7ea53..834c254 100644 --- a/fuzztest/CMakeLists.txt +++ b/fuzztest/CMakeLists.txt
@@ -83,6 +83,7 @@ "init_fuzztest.cc" DEPS fuzztest::configuration + fuzztest::flag_name fuzztest::googletest_adaptor fuzztest::io fuzztest::logging @@ -240,6 +241,7 @@ SRCS "internal/coverage.cc" DEPS + fuzztest::flag_name fuzztest::logging fuzztest::table_of_recent_compares absl::core_headers @@ -372,6 +374,13 @@ fuzztest_cc_library( NAME + flag_name + HDRS + "internal/flag_name.h" +) + +fuzztest_cc_library( + NAME googletest_adaptor HDRS "internal/googletest_adaptor.h" @@ -380,6 +389,7 @@ DEPS fuzztest::configuration fuzztest::corpus_database + fuzztest::flag_name fuzztest::io fuzztest::registry fuzztest::runtime @@ -529,6 +539,7 @@ fuzztest::coverage fuzztest::domain_core fuzztest::fixture_driver + fuzztest::flag_name fuzztest::io fuzztest::logging fuzztest::printer