Support Centipede using corpus seeding lib without globbing. This involve adding new non-globbing interface in the seeding lib. This is needed because not all platform support globbing. The standalone seed_corpus_maker still takes globbing options from the proto input. The plan is to migrate the proto too. PiperOrigin-RevId: 966020770
diff --git a/.github/workflows/bazel_test_centipede.yml b/.github/workflows/bazel_test_centipede.yml index 1bbcba0..ce34e4b 100644 --- a/.github/workflows/bazel_test_centipede.yml +++ b/.github/workflows/bazel_test_centipede.yml
@@ -30,6 +30,8 @@ # TODO(xinhaoyuan): Bump to 24.04 after https://github.com/llvm/llvm-project/issues/102443 # is fixed. runs-on: ubuntu-22.04 + permissions: + contents: read timeout-minutes: 60 strategy: matrix: @@ -39,13 +41,13 @@ run: | sudo sysctl -w kernel.core_pattern="" - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install dependencies run: | sudo apt-get update && sudo apt-get install -yq \ clang llvm libssl-dev - name: Restore latest cache - uses: actions/cache/restore@v4 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: "~/.cache/bazel" key: bazel-centipede-cache-${{ matrix.config }} @@ -84,19 +86,21 @@ bazel test --no//fuzztest:use_riegeli --test_output=errors --linkopt=-fsanitize=address --copt=-fsanitize=address --test_env=ASAN_OPTIONS=detect_leaks=0 --platform_suffix=asan --test_timeout=600 centipede/puzzles:all - name: Save new cache based on main if: github.ref == 'refs/heads/main' - uses: actions/cache/save@v4 + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: "~/.cache/bazel" key: bazel-centipede-cache-${{ matrix.config }}-${{ github.run_id }} run_tests_mac: name: Run Centipede tests (MacOS) runs-on: macos-15 + permissions: + contents: read timeout-minutes: 60 steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Restore latest cache - uses: actions/cache/restore@v4 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: "~/.cache/bazel" key: bazel-centipede-cache-mac @@ -127,7 +131,38 @@ bazel --output_user_root="${HOME}/.cache/bazel" test --test_output=errors --no//fuzztest:use_riegeli --linkopt=-fsanitize=address --copt=-fsanitize=address --test_env=ASAN_OPTIONS=detect_leaks=0 --platform_suffix=asan --test_timeout=600 centipede/puzzles:all - name: Save new cache based on main if: github.ref == 'refs/heads/main' - uses: actions/cache/save@v4 + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: "~/.cache/bazel" key: bazel-centipede-cache-mac-${{ github.run_id }} + run_tests_win: + name: Run Centipede tests (Windows) + runs-on: windows-latest + permissions: + contents: read + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Restore latest cache + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: "~/.cache/bazel" + key: bazel-centipede-cache-win + restore-keys: bazel-centipede-cache-win- + - name: Set environment variable + run: echo "USE_BAZEL_VERSION=8.7.0" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + - name: Run unit tests + if: ${{ !cancelled() }} + run: | + <# Only supported libraries are tested here. #> ` + bazelisk test --disk_cache=~/.cache/bazel --local_test_jobs=1 --test_output=errors --no//fuzztest:use_riegeli ` + --extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows-clang-cl ` + --extra_execution_platforms=//:x64_windows-clang-cl --enable_runfiles ` + -- centipede:util_test centipede:command_test centipede:seed_corpus_maker_lib_test + - name: Save new cache based on main + if: github.ref == 'refs/heads/main' + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: "~/.cache/bazel" + key: bazel-centipede-cache-win-${{ github.run_id }}
diff --git a/BUILD b/BUILD index 17cbe3f..e8b953e 100644 --- a/BUILD +++ b/BUILD
@@ -13,3 +13,12 @@ # limitations under the License. exports_files(["MODULE.bazel"]) + +platform( + name = "x64_windows-clang-cl", + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:windows", + "@bazel_tools//tools/cpp:clang-cl", + ], +)
diff --git a/MODULE.bazel b/MODULE.bazel index 0a0dc3e..f99e39c 100644 --- a/MODULE.bazel +++ b/MODULE.bazel
@@ -23,6 +23,10 @@ name = "rules_cc", version = "0.2.17", ) + +cc_configure = use_extension("@rules_cc//cc:extensions.bzl", "cc_configure_extension") +use_repo(cc_configure, "local_config_cc") + bazel_dep( name = "rules_shell", version = "0.6.1",
diff --git a/centipede/BUILD b/centipede/BUILD index 6279af0..ec5776a 100644 --- a/centipede/BUILD +++ b/centipede/BUILD
@@ -228,6 +228,7 @@ "@com_google_fuzztest//common:hash", "@com_google_fuzztest//common:logging", "@com_google_fuzztest//common:remote_file", + "@com_google_fuzztest//common:windows_includes", ], ) @@ -598,6 +599,8 @@ ":stop", ":util", "@abseil-cpp//absl/base:core_headers", + "@abseil-cpp//absl/cleanup", + "@abseil-cpp//absl/container:btree", "@abseil-cpp//absl/status", "@abseil-cpp//absl/status:statusor", "@abseil-cpp//absl/strings", @@ -605,6 +608,8 @@ "@abseil-cpp//absl/synchronization", "@abseil-cpp//absl/time", "@com_google_fuzztest//common:logging", + "@com_google_fuzztest//common:windows_includes", + "@com_google_fuzztest//fuzztest/internal:escaping", ], ) @@ -1392,7 +1397,12 @@ cc_test( name = "util_test", srcs = ["util_test.cc"], - copts = ["-fno-signed-char"], + copts = select({ + "@platforms//os:windows": [ + "/J", # Make unsigned char the default + ], + "//conditions:default": ["-fno-signed-char"], + }), deps = [ ":feature", ":thread_pool", @@ -1739,10 +1749,14 @@ name = "command_test_helper", srcs = ["command_test_helper.cc"], deps = [ - ":runner_fork_server", "@abseil-cpp//absl/base:nullability", + "@abseil-cpp//absl/strings", "@abseil-cpp//absl/time", - ], + "@com_google_fuzztest//common:windows_includes", + ] + select({ + "@platforms//os:windows": [], + "//conditions:default": [":runner_fork_server"], + }), ) cc_test(
diff --git a/centipede/centipede_interface.cc b/centipede/centipede_interface.cc index e6ba1eb..670c2b7 100644 --- a/centipede/centipede_interface.cc +++ b/centipede/centipede_interface.cc
@@ -282,30 +282,31 @@ std::string_view coverage_dir) { const WorkDir workdir{env}; SeedCorpusSource regression; - regression.dir_glob = std::string(regression_dir); + regression.src_dirs = {std::string(regression_dir)}; regression.num_recent_dirs = 1; - regression.individual_input_rel_glob = "*"; + regression.individual_input_rel_prefix = ""; regression.sampled_fraction_or_count = 1.0f; std::vector<SeedCorpusSource> sources = {std::move(regression)}; if (!coverage_dir.empty()) { SeedCorpusSource coverage; - coverage.dir_glob = std::string(coverage_dir); + coverage.src_dirs = {std::string(coverage_dir)}; coverage.num_recent_dirs = 1; // We're using the previously distilled corpus files as seeds. - coverage.shard_rel_glob = - std::filesystem::path{ - workdir.DistilledCorpusFilePaths().AllShardsGlob()} - .filename(); - coverage.individual_input_rel_glob = "*"; + coverage.shard_rel_prefix = + std::filesystem::path{workdir.DistilledCorpusFilePaths().prefix()} + .filename() + .string(); + coverage.individual_input_rel_prefix = ""; coverage.sampled_fraction_or_count = 1.0f; sources.push_back(std::move(coverage)); } SeedCorpusDestination destination; destination.dir_path = env.workdir; // We're seeding the current corpus files. - destination.shard_rel_glob = - std::filesystem::path{workdir.CorpusFilePaths().AllShardsGlob()} - .filename(); + destination.shard_rel_prefix = + std::filesystem::path{workdir.CorpusFilePaths().prefix()} + .filename() + .string(); destination.shard_index_digits = WorkDir::kDigitsInShardIndex; destination.num_shards = static_cast<uint32_t>(env.num_threads); return { @@ -653,21 +654,24 @@ "crashing"; const WorkDir workdir{env}; SeedCorpusSource crash_corpus_source; - crash_corpus_source.dir_glob = crash_dir; + crash_corpus_source.src_dirs = {crash_dir.string()}; crash_corpus_source.num_recent_dirs = 1; - crash_corpus_source.individual_input_rel_glob = env.crash_id; + crash_corpus_source.individual_input_rel_prefix = env.crash_id; crash_corpus_source.sampled_fraction_or_count = 1.0f; - const SeedCorpusConfig crash_corpus_config = { - /*sources=*/{crash_corpus_source}, - /*destination=*/{ - /*dir_path=*/env.workdir, - /*shard_rel_glob=*/ - std::filesystem::path{workdir.CorpusFilePaths().AllShardsGlob()} - .filename(), - /*shard_index_digits=*/WorkDir::kDigitsInShardIndex, - /*num_shards=*/1}}; - FUZZTEST_CHECK_OK(GenerateSeedCorpusFromConfig( - crash_corpus_config, env.binary_name, env.binary_hash)); + { + SeedCorpusDestination crash_corpus_destination; + crash_corpus_destination.dir_path = env.workdir; + crash_corpus_destination.shard_rel_prefix = + std::filesystem::path{workdir.CorpusFilePaths().prefix()} + .filename() + .string(); + crash_corpus_destination.shard_index_digits = WorkDir::kDigitsInShardIndex; + crash_corpus_destination.num_shards = 1; + FUZZTEST_CHECK_OK( + GenerateSeedCorpusFromConfig({/*sources=*/{crash_corpus_source}, + /*destination=*/crash_corpus_destination}, + env.binary_name, env.binary_hash)); + } Environment run_crash_env = env; run_crash_env.load_shards_only = true; run_crash_env.persistent_mode = false;
diff --git a/centipede/command.cc b/centipede/command.cc index 43100e4..ed66e71 100644 --- a/centipede/command.cc +++ b/centipede/command.cc
@@ -14,6 +14,7 @@ #include "./centipede/command.h" +#ifndef _WIN32 #include <errno.h> #include <fcntl.h> #include <spawn.h> @@ -22,6 +23,7 @@ #include <sys/types.h> #include <sys/wait.h> #include <unistd.h> +#endif // _WIN32 #ifdef __APPLE__ #include <inttypes.h> @@ -35,6 +37,7 @@ #include <cstdlib> #include <filesystem> // NOLINT #include <fstream> +#include <memory> #include <optional> #include <string> #include <string_view> @@ -58,8 +61,15 @@ #include "./centipede/stop.h" #include "./centipede/util.h" #include "./common/logging.h" +#include "./fuzztest/internal/escaping.h" +#ifdef _WIN32 +#include "absl/cleanup/cleanup.h" +#include "absl/container/btree_map.h" +#include "absl/strings/ascii.h" // NOLINT +#include "./common/windows_includes.h" +#endif -#if !defined(_MSC_VER) +#ifndef _WIN32 // Needed to pass the current environment to posix_spawn, which needs an // explicit envp without an option to inherit implicitly. extern char** environ; @@ -72,8 +82,11 @@ constexpr std::string_view kCommandLineSeparator(" \\\n"); constexpr std::string_view kNoForkServerRequestPrefix("%f"); +#ifdef _WIN32 +// Do not define `GetProcessCreationStamp`, which is for fork servers. +#else absl::StatusOr<std::string> GetProcessCreationStamp(pid_t pid) { -#ifdef __APPLE__ +#if defined(__APPLE__) struct proc_bsdinfo info = {}; if (proc_pidinfo(pid, PROC_PIDTBSDINFO, 0, &info, PROC_PIDTBSDINFO_SIZE) != PROC_PIDTBSDINFO_SIZE) { @@ -110,8 +123,9 @@ ": ", proc_stat_line)); } return std::string(fields[kFieldIndexOfStartTimeAfterComm]); -#endif +#endif // __APPLE__ } +#endif // _WIN32 std::string GetUniqueSuffix() { static std::atomic<uint64_t> suffix_counter = {0}; @@ -122,7 +136,17 @@ // TODO(ussuri): Encapsulate as much of the fork server functionality from // this source as possible in this struct, and make it a class. -struct Command::ForkServerProps { +#ifdef _WIN32 +struct Command::PlatformContext { + HANDLE win_process_handle = INVALID_HANDLE_VALUE; + ~PlatformContext() { + if (win_process_handle != INVALID_HANDLE_VALUE) { + CloseHandle(win_process_handle); + } + } +}; +#else +struct ForkServerProps { // The file paths of the comms pipes. std::string fifo_path_[2]; // The file descriptors of the comms pipes. @@ -177,17 +201,30 @@ } }; +struct Command::PlatformContext { + pid_t pid = -1; + std::unique_ptr<ForkServerProps> fork_server; +}; +#endif // _WIN32 + // NOTE: Because std::unique_ptr<T> requires T to be a complete type wherever // the deleter is instantiated, the special member functions must be defined -// out-of-line here, now that ForkServerProps is complete (that's by-the-book +// out-of-line here, now that PlatformContext is complete (that's by-the-book // PIMPL). Command::~Command() { if (is_executing()) { FUZZTEST_LOG(WARNING) << "Destructing Command object for " << path() << " with " - << (fork_server_ ? absl::StrCat("fork server PID ", - fork_server_->pid_) - : absl::StrCat("PID ", pid_)) +#ifdef _WIN32 + << GetProcessId(platform_context_->win_process_handle) +#else + << (platform_context_->fork_server + ? absl::StrCat( + "fork server PID ", + platform_context_->fork_server->pid_) + : absl::StrCat("PID ", + platform_context_->pid)) +#endif << " still running. Requesting it to force-stop " "without waiting for it..."; RequestStop(/*force=*/true); @@ -196,14 +233,51 @@ } Command::Command(std::string_view path, Options options) - : path_(path), options_(std::move(options)) {} + : path_(path), + options_(std::move(options)), + platform_context_(std::make_unique<PlatformContext>()) {} Command::Command(std::string_view path) : Command{path, {}} {} std::string Command::ToString() const { +#ifdef _WIN32 + std::string path = path_; + if (absl::StartsWith(path, kNoForkServerRequestPrefix)) { + path = path.substr(kNoForkServerRequestPrefix.size()); + } + constexpr std::string_view kTempFileWildCard = "@@"; + if (absl::StrContains(path, kTempFileWildCard)) { + FUZZTEST_CHECK(!options_.temp_file_path.empty()); + std::string temp_file = options_.temp_file_path; + path = absl::StrReplaceAll(path, {{kTempFileWildCard, temp_file}}); + } + std::string binary_cmd = path; + auto Escape = [](std::string_view s) { + std::string r = "\""; + size_t num_bs = 0; + for (size_t i = 0; i < s.size(); ++i) { + if (s[i] == '"') { + r.append(num_bs + 1, '\\'); + num_bs = 0; + } else if (s[i] == '\\') { + ++num_bs; + } else { + num_bs = 0; + } + r += s[i]; + } + r.append(num_bs, '\\'); + r += '"'; + return r; + }; + for (const auto& arg : options_.args) { + absl::StrAppend(&binary_cmd, " ", Escape(arg)); + } + return binary_cmd; +#else // _WIN32 std::vector<std::string> ss; ss.reserve(/*env*/ 1 + options_.env_diff.size() + /*path*/ 1 + - /*args*/ options_.args.size() + /*out/err*/ 2); + /*args*/ options_.args.size() + /*in/out/err*/ 3); // env. ss.push_back("exec env"); std::vector<std::string> env_to_set; @@ -217,7 +291,7 @@ } } for (auto& var : env_to_set) { - ss.push_back(std::move(var)); + ss.push_back(ShellEscape(var)); } // path. std::string path = path_; @@ -235,46 +309,57 @@ ss.push_back(std::move(path)); // args. for (const auto& arg : options_.args) { - ss.push_back(arg); + ss.push_back(ShellEscape(arg)); } - // out/err. + // in/out/err. + if (!options_.stdin_file_path.empty()) { + ss.push_back(absl::StrCat("< ", ShellEscape(options_.stdin_file_path))); + } if (!stdout_file_.empty()) { - ss.push_back(absl::StrCat("> ", stdout_file_)); + ss.push_back(absl::StrCat("> ", ShellEscape(stdout_file_))); } if (!stderr_file_.empty()) { if (stdout_file_ != stderr_file_) { - ss.push_back(absl::StrCat("2> ", stderr_file_)); + ss.push_back(absl::StrCat("2> ", ShellEscape(stderr_file_))); } else { ss.push_back("2>&1"); } } // Trim trailing space and return. return absl::StrJoin(ss, kCommandLineSeparator); +#endif // _WIN32 } bool Command::StartForkServer(std::string_view temp_dir_path, std::string_view prefix) { +#ifdef _WIN32 + return false; +#else if (absl::StartsWith(path_, kNoForkServerRequestPrefix)) { FUZZTEST_VLOG(2) << "Fork server disabled for " << path(); return false; } - FUZZTEST_CHECK(!is_executing_ && !fork_server_); + FUZZTEST_CHECK(!is_executing_ && !platform_context_->fork_server); FUZZTEST_VLOG(2) << "Starting fork server for " << path(); ResetRedirectionFiles(GetUniqueSuffix()); command_line_ = ToString(); - fork_server_.reset(new ForkServerProps); - fork_server_->fifo_path_[0] = std::filesystem::path(temp_dir_path) - .append(absl::StrCat(prefix, "_FIFO0")); - fork_server_->fifo_path_[1] = std::filesystem::path(temp_dir_path) - .append(absl::StrCat(prefix, "_FIFO1")); + platform_context_->fork_server = std::make_unique<ForkServerProps>(); + platform_context_->fork_server->fifo_path_[0] = + std::filesystem::path(temp_dir_path) + .append(absl::StrCat(prefix, "_FIFO0")); + platform_context_->fork_server->fifo_path_[1] = + std::filesystem::path(temp_dir_path) + .append(absl::StrCat(prefix, "_FIFO1")); const std::string pid_file_path = std::filesystem::path(temp_dir_path).append("pid"); (void)std::filesystem::create_directory(temp_dir_path); // it may not exist. for (int i = 0; i < 2; ++i) { - FUZZTEST_PCHECK(mkfifo(fork_server_->fifo_path_[i].c_str(), 0600) == 0) - << VV(i) << VV(fork_server_->fifo_path_[i]); + FUZZTEST_PCHECK( + mkfifo(platform_context_->fork_server->fifo_path_[i].c_str(), 0600) == + 0) + << VV(i) << VV(platform_context_->fork_server->fifo_path_[i]); } // NOTE: A background process does not return its exit status to the subshell, @@ -290,8 +375,9 @@ printf "%%s" $! > "%s" )sh"; const std::string fork_server_command = absl::StrFormat( - kForkServerCommandStub, fork_server_->fifo_path_[0], - fork_server_->fifo_path_[1], command_line_, pid_file_path); + kForkServerCommandStub, platform_context_->fork_server->fifo_path_[0], + platform_context_->fork_server->fifo_path_[1], command_line_, + pid_file_path); FUZZTEST_VLOG(1) << "Fork server command:" << fork_server_command; const int exit_code = system(fork_server_command.c_str()); @@ -316,10 +402,12 @@ // it. // See more at // https://www.gnu.org/software/libc/manual/html_node/Operating-Modes.html. - if ((fork_server_->pipe_[0] = open(fork_server_->fifo_path_[0].c_str(), - O_RDWR | O_NONBLOCK)) < 0 || - (fork_server_->pipe_[1] = open(fork_server_->fifo_path_[1].c_str(), - O_RDONLY | O_NONBLOCK)) < 0) { + if ((platform_context_->fork_server->pipe_[0] = + open(platform_context_->fork_server->fifo_path_[0].c_str(), + O_RDWR | O_NONBLOCK)) < 0 || + (platform_context_->fork_server->pipe_[1] = + open(platform_context_->fork_server->fifo_path_[1].c_str(), + O_RDONLY | O_NONBLOCK)) < 0) { LogProblemInfo( "Failed to establish communication with fork server; will proceed " "without it"); @@ -328,8 +416,11 @@ std::string pid_str; ReadFromLocalFile(pid_file_path, pid_str); - FUZZTEST_CHECK(absl::SimpleAtoi(pid_str, &fork_server_->pid_)) << VV(pid_str); - auto creation_stamp = GetProcessCreationStamp(fork_server_->pid_); + FUZZTEST_CHECK( + absl::SimpleAtoi(pid_str, &platform_context_->fork_server->pid_)) + << VV(pid_str); + auto creation_stamp = + GetProcessCreationStamp(platform_context_->fork_server->pid_); if (!creation_stamp.ok()) { LogProblemInfo( absl::StrCat("Failed to get the fork server's creation stamp; will " @@ -338,8 +429,9 @@ creation_stamp.status(), ")")); return false; } - fork_server_->creation_stamp = *std::move(creation_stamp); + platform_context_->fork_server->creation_stamp = *std::move(creation_stamp); return true; +#endif // _WIN32 } void Command::ResetRedirectionFiles(std::string_view new_suffix) { @@ -364,37 +456,218 @@ } absl::Status Command::VerifyForkServerIsHealthy() { +#ifdef _WIN32 + return absl::UnimplementedError("Fork server not supported on Windows"); +#else // Preconditions: the callers (`Execute()`) should call us only when the fork // server is presumed to be running (`fork_server_pid_` >= 0). If it is, the // comms pipes are guaranteed to be opened by `StartForkServer()`. - FUZZTEST_CHECK(fork_server_ != nullptr) << "Fork server wasn't started"; - FUZZTEST_CHECK(fork_server_->pid_ >= 0) + FUZZTEST_CHECK(platform_context_->fork_server != nullptr) + << "Fork server wasn't started"; + FUZZTEST_CHECK(platform_context_->fork_server->pid_ >= 0) << "Fork server process failed to start"; - FUZZTEST_CHECK(fork_server_->pipe_[0] >= 0 && fork_server_->pipe_[1] >= 0) + FUZZTEST_CHECK(platform_context_->fork_server->pipe_[0] >= 0 && + platform_context_->fork_server->pipe_[1] >= 0) << "Failed to connect to fork server"; // A process with the fork server PID exists (_some_ process, possibly with a // recycled PID)... - if (kill(fork_server_->pid_, 0) != EXIT_SUCCESS) { - return absl::UnknownError(absl::StrCat( - "Can't communicate with fork server, PID=", fork_server_->pid_)); + if (kill(platform_context_->fork_server->pid_, 0) != EXIT_SUCCESS) { + return absl::UnknownError( + absl::StrCat("Can't communicate with fork server, PID=", + platform_context_->fork_server->pid_)); } // ...and it is a process has the same creation stamp, so it's practically // guaranteed to be our original fork server process. - const auto creation_stamp = GetProcessCreationStamp(fork_server_->pid_); + const auto creation_stamp = + GetProcessCreationStamp(platform_context_->fork_server->pid_); if (!creation_stamp.ok()) return creation_stamp.status(); - if (*creation_stamp != fork_server_->creation_stamp) { + if (*creation_stamp != platform_context_->fork_server->creation_stamp) { return absl::UnknownError(absl::StrCat( "Fork server's creation stamp changed (new process?) - expected ", - fork_server_->creation_stamp, ", but got ", *creation_stamp)); + platform_context_->fork_server->creation_stamp, ", but got ", + *creation_stamp)); } return absl::OkStatus(); +#endif // _WIN32 } bool Command::ExecuteAsync() { FUZZTEST_CHECK(!is_executing()); - if (fork_server_ != nullptr) { +#ifdef _WIN32 + FUZZTEST_CHECK_EQ(platform_context_->win_process_handle, + INVALID_HANDLE_VALUE); + ResetRedirectionFiles(GetUniqueSuffix()); + command_line_ = ToString(); + + struct CaseInsensitiveCompare { + using is_transparent = void; + bool operator()(std::string_view a, std::string_view b) const { + return std::lexicographical_compare( + a.begin(), a.end(), b.begin(), b.end(), + [](unsigned char ca, unsigned char cb) { + return static_cast<unsigned char>(absl::ascii_tolower(ca)) < + static_cast<unsigned char>(absl::ascii_tolower(cb)); + }); + } + }; + + absl::btree_map<std::string, std::string, CaseInsensitiveCompare> env_map; + + LPCH env_strings = GetEnvironmentStringsA(); + if (env_strings != nullptr) { + const char* ptr = env_strings; + while (*ptr != '\0') { + std::string_view entry(ptr); + ptr += entry.size() + 1; + size_t eq_pos = entry.find('=', 1); + if (eq_pos != entry.npos) { + env_map[entry.substr(0, eq_pos)] = + std::string(entry.substr(eq_pos + 1)); + } else { + env_map[entry] = ""; + } + } + FreeEnvironmentStringsA(env_strings); + } + + for (std::string_view env_var : options_.env_diff) { + if (absl::StartsWith(env_var, "-")) { + std::string_view key = env_var.substr(1); + if (absl::EndsWith(key, "=")) { + key = key.substr(0, key.size() - 1); + } + env_map.erase(key); + } else { + auto pos = env_var.find('='); + if (pos != env_var.npos) { + env_map[env_var.substr(0, pos)] = std::string(env_var.substr(pos + 1)); + } + } + } + + std::string env_block = absl::StrJoin(env_map, absl::string_view("\0", 1), + absl::PairFormatter("=")); + env_block.append(2, '\0'); + + STARTUPINFOEXA si = {}; + si.StartupInfo.cb = sizeof(si); + si.StartupInfo.dwFlags = STARTF_USESTDHANDLES; + PROCESS_INFORMATION pi = {}; + + SECURITY_ATTRIBUTES sa = {sizeof(sa), NULL, TRUE}; + HANDLE handle_in = INVALID_HANDLE_VALUE; + HANDLE handle_out = INVALID_HANDLE_VALUE; + HANDLE handle_err = INVALID_HANDLE_VALUE; + + absl::Cleanup close_handles = [&] { + if (handle_in != INVALID_HANDLE_VALUE) CloseHandle(handle_in); + if (handle_out != INVALID_HANDLE_VALUE) CloseHandle(handle_out); + if (handle_err != INVALID_HANDLE_VALUE) CloseHandle(handle_err); + }; + + if (!options_.stdin_file_path.empty()) { + handle_in = CreateFileA(options_.stdin_file_path.c_str(), FILE_READ_DATA, + /*dwShareMode=*/0, &sa, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, NULL); + if (handle_in == INVALID_HANDLE_VALUE) { + FUZZTEST_LOG(ERROR) << "Failed to open stdin file: " + << options_.stdin_file_path; + return false; + } + si.StartupInfo.hStdInput = handle_in; + } else { + si.StartupInfo.hStdInput = GetStdHandle(STD_INPUT_HANDLE); + } + + if (!stdout_file_.empty()) { + handle_out = + CreateFileA(stdout_file_.c_str(), GENERIC_WRITE, FILE_SHARE_READ, &sa, + CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (handle_out == INVALID_HANDLE_VALUE) { + FUZZTEST_LOG(ERROR) << "Failed to open stdout file: " << stdout_file_; + return false; + } + si.StartupInfo.hStdOutput = handle_out; + } else { + si.StartupInfo.hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE); + } + + if (!stderr_file_.empty()) { + if (stderr_file_ == stdout_file_) { + si.StartupInfo.hStdError = si.StartupInfo.hStdOutput; + } else { + handle_err = + CreateFileA(stderr_file_.c_str(), GENERIC_WRITE, FILE_SHARE_READ, &sa, + CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (handle_err == INVALID_HANDLE_VALUE) { + FUZZTEST_LOG(ERROR) << "Failed to open stderr file: " << stderr_file_; + return false; + } + si.StartupInfo.hStdError = handle_err; + } + } else { + si.StartupInfo.hStdError = GetStdHandle(STD_ERROR_HANDLE); + } + + std::vector<HANDLE> handles_to_inherit; + for (HANDLE h : {si.StartupInfo.hStdInput, si.StartupInfo.hStdOutput, + si.StartupInfo.hStdError}) { + if (h != nullptr && h != INVALID_HANDLE_VALUE && + std::find(handles_to_inherit.begin(), handles_to_inherit.end(), h) == + handles_to_inherit.end()) { + SetHandleInformation(h, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT); + handles_to_inherit.push_back(h); + } + } + + SIZE_T attr_list_size = 0; + InitializeProcThreadAttributeList(nullptr, 1, 0, &attr_list_size); + auto attr_list_buffer = std::make_unique<uint8_t[]>(attr_list_size); + si.lpAttributeList = + reinterpret_cast<PPROC_THREAD_ATTRIBUTE_LIST>(attr_list_buffer.get()); + if (!InitializeProcThreadAttributeList(si.lpAttributeList, 1, 0, + &attr_list_size)) { + DWORD err = GetLastError(); + FUZZTEST_LOG(ERROR) << "InitializeProcThreadAttributeList failed: " << err; + return false; + } + absl::Cleanup delete_attr_list = [&] { + DeleteProcThreadAttributeList(si.lpAttributeList); + }; + + if (!handles_to_inherit.empty()) { + if (!UpdateProcThreadAttribute( + si.lpAttributeList, 0, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, + handles_to_inherit.data(), + handles_to_inherit.size() * sizeof(HANDLE), nullptr, nullptr)) { + DWORD err = GetLastError(); + FUZZTEST_LOG(ERROR) << "UpdateProcThreadAttribute failed: " << err; + return false; + } + } + + std::string cmd = command_line_; + const BOOL cp_res = CreateProcessA( + NULL, cmd.data(), NULL, NULL, + /*bInheritHandles=*/!handles_to_inherit.empty(), + EXTENDED_STARTUPINFO_PRESENT, env_block.empty() ? NULL : env_block.data(), + NULL, &si.StartupInfo, &pi); + + if (!cp_res) { + DWORD err = GetLastError(); + FUZZTEST_LOG(ERROR) << "CreateProcessA failed for '" << cmd + << "': error=" << err; + return false; + } + + platform_context_->win_process_handle = pi.hProcess; + CloseHandle(pi.hThread); + is_executing_ = true; + return true; +#else // _WIN32 + if (platform_context_->fork_server != nullptr) { FUZZTEST_VLOG(1) << "Sending execution request to fork server"; if (const auto status = VerifyForkServerIsHealthy(); !status.ok()) { @@ -405,7 +678,7 @@ // Wake up the fork server. char x = ' '; - if (write(fork_server_->pipe_[0], &x, 1) != 1) { + if (write(platform_context_->fork_server->pipe_[0], &x, 1) != 1) { LogProblemInfo( absl::StrCat("Failed to write to fork server pipe. Errno: ", errno)); return false; @@ -413,12 +686,13 @@ // Read the one-byte ack. // Use 60s as an arbitrary duration to wait for the process to load and // enter the fork server. - if (!fork_server_->ReadPipe(absl::Now() + absl::Seconds(60), x)) { + if (!platform_context_->fork_server->ReadPipe( + absl::Now() + absl::Seconds(60), x)) { LogProblemInfo("Failed to read from fork server pipe."); return false; } } else { - FUZZTEST_CHECK_EQ(pid_, -1); + FUZZTEST_CHECK_EQ(platform_context_->pid, -1); ResetRedirectionFiles(GetUniqueSuffix()); command_line_ = ToString(); @@ -431,24 +705,58 @@ argv.push_back(argv_str.data()); } argv.push_back(nullptr); - FUZZTEST_PCHECK(posix_spawn(&pid_, argv[0], /*file_actions=*/nullptr, + FUZZTEST_PCHECK(posix_spawn(&platform_context_->pid, argv[0], + /*file_actions=*/nullptr, /*attrp=*/nullptr, argv.data(), environ) == 0); } is_executing_ = true; return true; +#endif // _WIN32 } std::optional<int> Command::Wait(absl::Time deadline, StopCondition* stop_condition) { FUZZTEST_CHECK(is_executing()); +#ifdef _WIN32 + FUZZTEST_CHECK_NE(platform_context_->win_process_handle, + INVALID_HANDLE_VALUE); + DWORD timeout_ms = INFINITE; + if (deadline != absl::InfiniteFuture()) { + auto dur = deadline - absl::Now(); + if (dur <= absl::ZeroDuration()) { + timeout_ms = 0; + } else { + timeout_ms = static_cast<DWORD>(absl::ToInt64Milliseconds(dur)); + } + } + DWORD res = + WaitForSingleObject(platform_context_->win_process_handle, timeout_ms); + if (res == WAIT_TIMEOUT) { + VlogProblemInfo( + absl::StrCat("Timeout while waiting for command process: deadline is ", + deadline), + /*vlog_level=*/1); + return std::nullopt; + } + DWORD exit_code = 0; + GetExitCodeProcess(platform_context_->win_process_handle, &exit_code); + CloseHandle(platform_context_->win_process_handle); + platform_context_->win_process_handle = INVALID_HANDLE_VALUE; + is_executing_ = false; + if (exit_code == STATUS_CONTROL_C_EXIT && stop_condition != nullptr) { + stop_condition->RequestStop( + EXIT_FAILURE, "Command killed: signal=SIGINT (likely Ctrl-C)"); + } + return static_cast<int>(exit_code); +#else // _WIN32 int exit_code = EXIT_SUCCESS; - if (fork_server_ != nullptr) { + if (platform_context_->fork_server != nullptr) { // The fork server forks, the child is running. Block until some readable // data appears in the pipe (that is, after the fork server writes the // execution result to it). - if (!fork_server_->ReadPipe(deadline, exit_code)) { + if (!platform_context_->fork_server->ReadPipe(deadline, exit_code)) { VlogProblemInfo( absl::StrCat("Waiting for fork server failed, deadline is ", deadline), @@ -456,11 +764,13 @@ return std::nullopt; } } else { - FUZZTEST_CHECK_NE(pid_, -1); + FUZZTEST_CHECK_NE(platform_context_->pid, -1); while (true) { - const pid_t r = waitpid(pid_, &exit_code, WNOHANG); + const pid_t r = waitpid(platform_context_->pid, &exit_code, WNOHANG); FUZZTEST_CHECK_NE(r, -1); - if (r == pid_ && (WIFEXITED(exit_code) || WIFSIGNALED(exit_code))) break; + if (r == platform_context_->pid && + (WIFEXITED(exit_code) || WIFSIGNALED(exit_code))) + break; FUZZTEST_CHECK_EQ(r, 0); const auto timeout = deadline - absl::Now(); if (timeout > absl::ZeroDuration()) { @@ -477,7 +787,7 @@ return std::nullopt; } } - pid_ = -1; + platform_context_->pid = -1; } is_executing_ = false; @@ -535,20 +845,27 @@ } return exit_code; +#endif // _WIN32 } void Command::RequestStop(bool force) { FUZZTEST_CHECK(is_executing()); - if (fork_server_) { - FUZZTEST_CHECK_NE(fork_server_->pid_, -1); +#ifdef _WIN32 + FUZZTEST_CHECK_NE(platform_context_->win_process_handle, + INVALID_HANDLE_VALUE); + TerminateProcess(platform_context_->win_process_handle, 1); +#else + if (platform_context_->fork_server) { + FUZZTEST_CHECK_NE(platform_context_->fork_server->pid_, -1); // Cannot send SIGKILL to the fork server as it kills only the parent // process, but not the child. The fork server would send SIGKILL to the // child on SIGUSR1. - kill(fork_server_->pid_, force ? SIGUSR1 : SIGTERM); + kill(platform_context_->fork_server->pid_, force ? SIGUSR1 : SIGTERM); return; } - FUZZTEST_CHECK_NE(pid_, -1); - kill(pid_, force ? SIGKILL : SIGTERM); + FUZZTEST_CHECK_NE(platform_context_->pid, -1); + kill(platform_context_->pid, force ? SIGKILL : SIGTERM); +#endif // _WIN32 } std::string Command::ReadRedirectedStdout() const {
diff --git a/centipede/command.h b/centipede/command.h index 33c1d14..15d4d70 100644 --- a/centipede/command.h +++ b/centipede/command.h
@@ -52,6 +52,8 @@ // `Command` automatically unlinks any previous redirected files on // execution and destruction. std::string stderr_file_prefix; + // Redirect stdin from this file path if non-empty. + std::string stdin_file_path; // "@@" in the command will be replaced with `temp_file_path`. std::string temp_file_path; }; @@ -125,9 +127,8 @@ const std::string& stderr_file() const { return stderr_file_; } private: - struct ForkServerProps; + struct PlatformContext; - int pid_ = -1; bool is_executing_ = false; // Derived from Options::{stdout,stderr}_file_prefix, with the realized suffix @@ -163,7 +164,7 @@ const Options options_; std::string command_line_; - std::unique_ptr<ForkServerProps> fork_server_; + std::unique_ptr<PlatformContext> platform_context_; }; // Get the shared mutex for execution logging for preventing confusing
diff --git a/centipede/command_test.cc b/centipede/command_test.cc index 48f4d07..0c91cb1 100644 --- a/centipede/command_test.cc +++ b/centipede/command_test.cc
@@ -15,7 +15,9 @@ #include "./centipede/command.h" #include <signal.h> +#ifndef _WIN32 #include <sys/wait.h> // NOLINT(for WTERMSIG) +#endif #include <cstdlib> #include <filesystem> // NOLINT @@ -26,6 +28,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "absl/strings/str_cat.h" #include "absl/strings/substitute.h" #include "absl/time/clock.h" #include "absl/time/time.h" @@ -33,58 +36,120 @@ #include "./centipede/util.h" #include "./common/test_util.h" +#ifdef _WIN32 +#define setenv(n, v, _r) _putenv_s(n, v) +#endif + namespace fuzztest::internal { namespace { +using ::testing::AllOf; +using ::testing::HasSubstr; using ::testing::Optional; -TEST(CommandTest, ToString) { - EXPECT_EQ(Command{"x"}.ToString(), "exec env \\\nx"); - { - Command::Options cmd_options; - cmd_options.args = {"arg1", "arg2"}; - EXPECT_EQ((Command{"path", std::move(cmd_options)}.ToString()), - "exec env \\\npath \\\narg1 \\\narg2"); - } - { - Command::Options cmd_options; - cmd_options.env_diff = {"K1=V1", "K2=V2", "-K3"}; - EXPECT_EQ((Command{"x", std::move(cmd_options)}.ToString()), - "exec env \\\n-u K3 \\\nK1=V1 \\\nK2=V2 \\\nx"); - } -} - TEST(CommandTest, Execute) { + const std::string helper = + GetDataDependencyFilepath("centipede/command_test_helper").string(); StopCondition stop_condition; // Check for default exit code. - Command echo{"echo"}; - EXPECT_EQ(echo.Execute(&stop_condition), 0); + Command::Options options_success; + options_success.args = {"success"}; + Command success_cmd{helper, std::move(options_success)}; + EXPECT_EQ(success_cmd.Execute(&stop_condition), 0); EXPECT_FALSE(stop_condition.ShouldStop()); // Check for exit code 7. - Command exit7{"bash -c 'exit 7'"}; + Command::Options options_ret7; + options_ret7.args = {"ret7"}; + Command exit7{helper, std::move(options_ret7)}; EXPECT_EQ(exit7.Execute(&stop_condition), 7); EXPECT_FALSE(stop_condition.ShouldStop()); } TEST(CommandTest, HandlesInterruptedCommand) { + const std::string helper = + GetDataDependencyFilepath("centipede/command_test_helper").string(); StopCondition stop_condition; - Command self_sigint{"bash -c 'kill -SIGINT $$'"}; - self_sigint.ExecuteAsync(); - self_sigint.Wait(absl::InfiniteFuture(), &stop_condition); + Command::Options options_ctrlc; + options_ctrlc.args = {"ctrlc"}; + Command self_ctrlc{helper, std::move(options_ctrlc)}; + // Cannot set to SIG_IGN as the command would inherit that. + signal(SIGINT, [](int) {}); + self_ctrlc.ExecuteAsync(); + signal(SIGINT, SIG_DFL); + self_ctrlc.Wait(absl::InfiniteFuture(), &stop_condition); EXPECT_TRUE(stop_condition.ShouldStop()); } -TEST(CommandTest, InputFileWildCard) { - Command::Options cmd_options; - cmd_options.temp_file_path = "TEMP_FILE"; - Command cmd{"foo bar @@ baz", std::move(cmd_options)}; - EXPECT_EQ(cmd.ToString(), "exec env \\\nfoo bar TEMP_FILE baz"); +TEST(CommandTest, ExecuteWithOptions) { + const std::filesystem::path test_tmpdir = GetTestTempDir(test_info_->name()); + const std::string helper = + GetDataDependencyFilepath("centipede/command_test_helper").string(); + + { + const std::string log_prefix = (test_tmpdir / "args").string(); + Command::Options cmd_options; + cmd_options.args = {"echo_args", "arg1", "arg2"}; + cmd_options.stdout_file_prefix = log_prefix; + Command cmd{helper, std::move(cmd_options)}; + EXPECT_EQ(cmd.Execute(), 0); + std::string log_contents; + ReadFromLocalFile(cmd.stdout_file(), log_contents); + EXPECT_THAT(log_contents, + AllOf(HasSubstr("arg[0]=arg1\n"), HasSubstr("arg[1]=arg2\n"))); + } + + { + setenv("K3", "V3", 1); + const std::string log_prefix = (test_tmpdir / "env").string(); + Command::Options cmd_options; + cmd_options.args = {"echo_env", "K1", "K2", "K3"}; + cmd_options.env_diff = {"K1=V1", "K2=V2", "-K3"}; + cmd_options.stdout_file_prefix = log_prefix; + Command cmd{helper, std::move(cmd_options)}; + EXPECT_EQ(cmd.Execute(), 0); + std::string log_contents; + ReadFromLocalFile(cmd.stdout_file(), log_contents); + EXPECT_THAT(log_contents, AllOf(HasSubstr("K1=V1\n"), HasSubstr("K2=V2\n"), + HasSubstr("K3=<UNSET>\n"))); + } + + { + const std::string stdin_file = (test_tmpdir / "input.txt").string(); + WriteToLocalFile(stdin_file, "hello stdin"); + const std::string log_prefix = (test_tmpdir / "stdin").string(); + Command::Options cmd_options; + cmd_options.args = {"echo_stdin"}; + cmd_options.stdin_file_path = stdin_file; + cmd_options.stdout_file_prefix = log_prefix; + Command cmd{helper, std::move(cmd_options)}; + EXPECT_EQ(cmd.Execute(), 0); + std::string log_contents; + ReadFromLocalFile(cmd.stdout_file(), log_contents); + EXPECT_THAT(log_contents, HasSubstr("hello stdin")); + } } +TEST(CommandTest, InputFileWildCard) { + const std::filesystem::path test_tmpdir = GetTestTempDir(test_info_->name()); + const std::string helper = + GetDataDependencyFilepath("centipede/command_test_helper").string(); + const std::string log_prefix = (test_tmpdir / "wildcard").string(); + + Command::Options cmd_options; + cmd_options.temp_file_path = "TEMP_FILE"; + cmd_options.stdout_file_prefix = log_prefix; + Command cmd{absl::StrCat(helper, " @@"), std::move(cmd_options)}; + EXPECT_EQ(cmd.Execute(), 17); + std::string log_contents; + ReadFromLocalFile(cmd.stdout_file(), log_contents); + EXPECT_EQ(log_contents, "Got input: TEMP_FILE\n"); +} + +#ifndef _WIN32 TEST(CommandTest, ForkServer) { - const std::string test_tmpdir = GetTestTempDir(test_info_->name()); + const std::filesystem::path test_tmpdir = GetTestTempDir(test_info_->name()); const std::string helper = GetDataDependencyFilepath("centipede/command_test_helper"); @@ -92,93 +157,93 @@ { const std::string input = "success"; - const std::string log_prefix = std::filesystem::path{test_tmpdir} / input; + const std::string log_prefix = test_tmpdir / input; Command::Options cmd_options; cmd_options.args = {input}; cmd_options.stdout_file_prefix = log_prefix; cmd_options.stderr_file_prefix = log_prefix; Command cmd{helper, std::move(cmd_options)}; - EXPECT_TRUE(cmd.StartForkServer(test_tmpdir, "ForkServer")); + EXPECT_TRUE(cmd.StartForkServer(test_tmpdir.string(), "ForkServer")); EXPECT_EQ(cmd.Execute(), EXIT_SUCCESS); std::string log_contents; ReadFromLocalFile(cmd.stdout_file(), log_contents); - EXPECT_EQ(log_contents, absl::Substitute("Got input: $0", input)); + EXPECT_EQ(log_contents, absl::Substitute("Got input: $0\n", input)); } { const std::string input = "fail"; - const std::string log_prefix = std::filesystem::path{test_tmpdir} / input; + const std::string log_prefix = test_tmpdir / input; Command::Options cmd_options; cmd_options.args = {input}; cmd_options.stdout_file_prefix = log_prefix; cmd_options.stderr_file_prefix = log_prefix; Command cmd{helper, std::move(cmd_options)}; - EXPECT_TRUE(cmd.StartForkServer(test_tmpdir, "ForkServer")); + EXPECT_TRUE(cmd.StartForkServer(test_tmpdir.string(), "ForkServer")); EXPECT_EQ(cmd.Execute(), EXIT_FAILURE); std::string log_contents; ReadFromLocalFile(cmd.stdout_file(), log_contents); - EXPECT_EQ(log_contents, absl::Substitute("Got input: $0", input)); + EXPECT_EQ(log_contents, absl::Substitute("Got input: $0\n", input)); } { const std::string input = "ret42"; - const std::string log_prefix = std::filesystem::path{test_tmpdir} / input; + const std::string log_prefix = test_tmpdir / input; Command::Options cmd_options; cmd_options.args = {input}; cmd_options.stdout_file_prefix = log_prefix; cmd_options.stderr_file_prefix = log_prefix; Command cmd{helper, std::move(cmd_options)}; - EXPECT_TRUE(cmd.StartForkServer(test_tmpdir, "ForkServer")); + EXPECT_TRUE(cmd.StartForkServer(test_tmpdir.string(), "ForkServer")); EXPECT_EQ(cmd.Execute(), 42); std::string log_contents; ReadFromLocalFile(cmd.stdout_file(), log_contents); - EXPECT_EQ(log_contents, absl::Substitute("Got input: $0", input)); + EXPECT_EQ(log_contents, absl::Substitute("Got input: $0\n", input)); } { const std::string input = "abort"; - const std::string log_prefix = std::filesystem::path{test_tmpdir} / input; + const std::string log_prefix = test_tmpdir / input; Command::Options cmd_options; cmd_options.args = {input}; cmd_options.stdout_file_prefix = log_prefix; cmd_options.stderr_file_prefix = log_prefix; Command cmd{helper, std::move(cmd_options)}; - EXPECT_TRUE(cmd.StartForkServer(test_tmpdir, "ForkServer")); + EXPECT_TRUE(cmd.StartForkServer(test_tmpdir.string(), "ForkServer")); // WTERMSIG() needs an lvalue on some platforms. const int ret = cmd.Execute(); EXPECT_EQ(WTERMSIG(ret), SIGABRT); std::string log_contents; ReadFromLocalFile(cmd.stdout_file(), log_contents); - EXPECT_EQ(log_contents, absl::Substitute("Got input: $0", input)); + EXPECT_EQ(log_contents, absl::Substitute("Got input: $0\n", input)); } { const std::string input = "sleep"; - const std::string log_prefix = std::filesystem::path{test_tmpdir} / input; + const std::string log_prefix = test_tmpdir / input; Command::Options cmd_options; cmd_options.args = {input}; cmd_options.stdout_file_prefix = log_prefix; cmd_options.stderr_file_prefix = log_prefix; Command cmd{helper, std::move(cmd_options)}; - ASSERT_TRUE(cmd.StartForkServer(test_tmpdir, "ForkServer")); + ASSERT_TRUE(cmd.StartForkServer(test_tmpdir.string(), "ForkServer")); ASSERT_TRUE(cmd.ExecuteAsync()); EXPECT_EQ(cmd.Wait(absl::Now() + absl::Seconds(2)), std::nullopt); cmd.RequestStop(/*force=*/false); EXPECT_THAT(cmd.Wait(absl::Now() + absl::Seconds(2)), Optional(SIGTERM)); std::string log_contents; ReadFromLocalFile(cmd.stdout_file(), log_contents); - EXPECT_EQ(log_contents, absl::Substitute("Got input: $0", input)); + EXPECT_EQ(log_contents, absl::Substitute("Got input: $0\n", input)); } { const std::string input = "hang"; - const std::string log_prefix = std::filesystem::path{test_tmpdir} / input; + const std::string log_prefix = test_tmpdir / input; Command::Options cmd_options; cmd_options.args = {input}; cmd_options.stdout_file_prefix = log_prefix; cmd_options.stderr_file_prefix = log_prefix; Command cmd{helper, std::move(cmd_options)}; - ASSERT_TRUE(cmd.StartForkServer(test_tmpdir, "ForkServer")); + ASSERT_TRUE(cmd.StartForkServer(test_tmpdir.string(), "ForkServer")); ASSERT_TRUE(cmd.ExecuteAsync()); EXPECT_EQ(cmd.Wait(absl::Now() + absl::Seconds(2)), std::nullopt); cmd.RequestStop(/*force=*/false); @@ -187,11 +252,12 @@ EXPECT_THAT(cmd.Wait(absl::Now() + absl::Seconds(2)), Optional(SIGKILL)); std::string log_contents; ReadFromLocalFile(cmd.stdout_file(), log_contents); - EXPECT_EQ(log_contents, absl::Substitute("Got input: $0", input)); + EXPECT_EQ(log_contents, absl::Substitute("Got input: $0\n", input)); } // TODO(kcc): [impl] test what happens if the child is interrupted. } +#endif } // namespace } // namespace fuzztest::internal
diff --git a/centipede/command_test_helper.cc b/centipede/command_test_helper.cc index d3153a8..75fe519 100644 --- a/centipede/command_test_helper.cc +++ b/centipede/command_test_helper.cc
@@ -12,33 +12,92 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include <unistd.h> - #include <cassert> #include <csignal> #include <cstdio> #include <cstdlib> #include <cstring> +#ifdef _WIN32 +#include <fcntl.h> +#include <io.h> + +#include "./common/windows_includes.h" +#endif + #include "absl/base/nullability.h" +#include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/string_view.h" #include "absl/time/clock.h" #include "absl/time/time.h" // A binary linked with the fork server that exits/crashes in different ways. int main(int argc, char** absl_nonnull argv) { - assert(argc == 2); - printf("Got input: %s", argv[1]); +#ifdef _WIN32 + // Disable the automatic \n -> \r\n conversion. + _setmode(_fileno(stdout), _O_BINARY); +#endif + + assert(argc >= 2); + printf("Got input: %s\n", argv[1]); + + if (!strcmp(argv[1], "echo_args")) { + for (int i = 2; i < argc; ++i) { + printf("arg[%d]=%s\n", i - 2, argv[i]); + } + fflush(stdout); + return EXIT_SUCCESS; + } + + if (!strcmp(argv[1], "echo_env")) { + for (int i = 2; i < argc; ++i) { + const char* val = getenv(argv[i]); + printf("%s=%s\n", argv[i], val ? val : "<UNSET>"); + } + fflush(stdout); + return EXIT_SUCCESS; + } + + if (!strcmp(argv[1], "echo_stdin")) { + char buf[1024]; + while (fgets(buf, sizeof(buf), stdin)) { + fputs(buf, stdout); + } + fflush(stdout); + return EXIT_SUCCESS; + } + fflush(stdout); + if (!strcmp(argv[1], "success")) return EXIT_SUCCESS; if (!strcmp(argv[1], "fail")) return EXIT_FAILURE; - if (!strcmp(argv[1], "ret42")) return 42; + + int ret_code = 0; + if (absl::StartsWith(argv[1], "ret") && + absl::SimpleAtoi(argv[1] + 3, &ret_code)) { + return ret_code; + } + if (!strcmp(argv[1], "abort")) abort(); + if (!strcmp(argv[1], "ctrlc")) { +#ifdef _WIN32 + SetConsoleCtrlHandler(NULL, FALSE); + GenerateConsoleCtrlEvent(CTRL_C_EVENT, 0); + Sleep(INFINITE); +#else + raise(SIGINT); +#endif + return EXIT_SUCCESS; + } // Sleep longer than kTimeout in CommandDeathTest_ForkServerHangingBinary. if (!strcmp(argv[1], "sleep")) absl::SleepFor(absl::Seconds(5)); if (!strcmp(argv[1], "hang")) { +#ifndef _WIN32 struct sigaction act{}; act.sa_handler = [](int) {}; sigaction(SIGTERM, &act, nullptr); +#endif absl::SleepFor(absl::Seconds(10)); }
diff --git a/centipede/seed_corpus_maker_lib.cc b/centipede/seed_corpus_maker_lib.cc index d9171c7..31b3d82 100644 --- a/centipede/seed_corpus_maker_lib.cc +++ b/centipede/seed_corpus_maker_lib.cc
@@ -105,26 +105,75 @@ // `source.num_recent_dirs()` most recent ones. std::vector<std::string> src_dirs; - if (const auto match_status = RemoteGlobMatch(source.dir_glob, src_dirs); - !match_status.ok() && !absl::IsNotFound(match_status)) { - return match_status; + if (!source.dir_glob.empty()) { + if (const auto match_status = RemoteGlobMatch(source.dir_glob, src_dirs); + !match_status.ok() && !absl::IsNotFound(match_status)) { + return match_status; + } + FUZZTEST_LOG(INFO) << "Found " << src_dirs.size() + << " corpus dir(s) candidates matching " + << source.dir_glob; } - FUZZTEST_LOG(INFO) << "Found " << src_dirs.size() - << " corpus dir(s) matching " << source.dir_glob; + src_dirs.insert(src_dirs.end(), source.src_dirs.begin(), + source.src_dirs.end()); // Sort in the ascending lexicographical order. We expect that dir names // contain timestamps and therefore will be sorted from oldest to newest. std::sort(src_dirs.begin(), src_dirs.end(), std::less<std::string>()); + src_dirs.erase(std::unique(src_dirs.begin(), src_dirs.end()), src_dirs.end()); + src_dirs.erase(std::remove_if(src_dirs.begin(), src_dirs.end(), + [](const auto& path) { + return !RemotePathIsDirectory(path); + }), + src_dirs.end()); if (source.num_recent_dirs < src_dirs.size()) { src_dirs.erase(src_dirs.begin(), src_dirs.end() - source.num_recent_dirs); FUZZTEST_LOG(INFO) << "Selected " << src_dirs.size() << " corpus dir(s)"; } + FUZZTEST_LOG(INFO) + << "Reading/sampling seed corpus elements from source dir(s): "; + for (const auto& src_dir : src_dirs) { + FUZZTEST_LOG(INFO) << " " << src_dir; + } + // Find all the corpus shard and individual input files in the found dirs. std::vector<std::string> corpus_shard_fnames; std::vector<std::string> individual_input_fnames; for (const auto& dir : src_dirs) { absl::flat_hash_set<std::string> current_corpus_shard_fnames; + if (source.shard_rel_prefix.has_value()) { + if (!source.shard_rel_glob.empty()) { + return absl::InvalidArgumentError( + "Must not specify both shard_rel_prefix and shard_rel_glob"); + } + const auto shard_prefix = + std::filesystem::path{fs::path{dir} / *source.shard_rel_prefix}; + const auto candidates = + RemoteListFiles(shard_prefix.parent_path().string(), + /*recursively=*/false); + if (candidates.ok()) { + const auto shard_prefix_filename = shard_prefix.filename().string(); + size_t num_added_shards = 0; + for (const auto& candidate : *candidates) { + if (!absl::StartsWith( + std::filesystem::path{candidate}.filename().string(), + shard_prefix_filename)) { + continue; + } + ++num_added_shards; + corpus_shard_fnames.push_back(candidate); + current_corpus_shard_fnames.insert(candidate); + } + FUZZTEST_LOG(INFO) << "Found " << num_added_shards + << " shard(s) matching prefix [" + << *source.shard_rel_prefix << "]"; + } else { + FUZZTEST_LOG(ERROR) + << "Got error when listing with " << VV(*source.shard_rel_prefix) + << ": " << candidates.status(); + } + } if (!source.shard_rel_glob.empty()) { std::vector<std::string> matched_fnames; const std::string glob = fs::path{dir} / source.shard_rel_glob; @@ -142,6 +191,43 @@ << " shard(s) matching " << glob; } } + if (source.individual_input_rel_prefix.has_value()) { + if (!source.individual_input_rel_glob.empty()) { + return absl::InvalidArgumentError( + "Must not specify both individual_input_rel_prefix and " + "individual_input_rel_glob"); + } + const auto individual_input_prefix = std::filesystem::path{ + fs::path{dir} / *source.individual_input_rel_prefix}; + const auto candidates = + RemoteListFiles(individual_input_prefix.parent_path().string(), + /*recursively=*/false); + if (candidates.ok()) { + const auto individual_input_prefix_filename = + individual_input_prefix.filename().string(); + size_t num_added_individual_inputs = 0; + for (const auto& candidate : *candidates) { + if (RemotePathIsDirectory(candidate)) { + continue; + } + if (!absl::StartsWith( + std::filesystem::path{candidate}.filename().string(), + individual_input_prefix_filename)) { + continue; + } + if (current_corpus_shard_fnames.contains(candidate)) continue; + ++num_added_individual_inputs; + individual_input_fnames.push_back(candidate); + } + FUZZTEST_LOG(INFO) << "Found " << num_added_individual_inputs + << " individual input(s) with prefix: [" + << *source.individual_input_rel_prefix << "]"; + } else { + FUZZTEST_LOG(ERROR) << "Got error when listing with " + << VV(*source.individual_input_rel_prefix) << ": " + << candidates.status(); + } + } if (!source.individual_input_rel_glob.empty()) { std::vector<std::string> matched_fnames; const std::string glob = fs::path{dir} / source.individual_input_rel_glob; @@ -378,10 +464,19 @@ return absl::InvalidArgumentError( "Requested number of destination shards must be > 0"); } - if (!absl::StrContains(destination.shard_rel_glob, "*")) { + if (!destination.shard_rel_glob.empty()) { + if (!absl::StrContains(destination.shard_rel_glob, "*")) { + return absl::InvalidArgumentError( + absl::StrCat("Destination shard pattern must contain '*', got ", + destination.shard_rel_glob)); + } + if (destination.shard_rel_prefix.has_value()) { + return absl::InvalidArgumentError( + "Cannot have both shard_rel_glob and shard_rel_prefix"); + } + } else if (!destination.shard_rel_prefix.has_value()) { return absl::InvalidArgumentError( - absl::StrCat("Destination shard pattern must contain '*', got ", - destination.shard_rel_glob)); + "Missing shard_rel_glob and shard_rel_prefix"); } // Compute shard sizes. If the elements can't be evenly divided between the @@ -424,7 +519,10 @@ const std::string shard_idx = absl::StrFormat("%0*d", destination.shard_index_digits, shard); const std::string corpus_rel_fname = - absl::StrReplaceAll(destination.shard_rel_glob, {{"*", shard_idx}}); + !destination.shard_rel_glob.empty() + ? absl::StrReplaceAll(destination.shard_rel_glob, + {{"*", shard_idx}}) + : absl::StrCat(*destination.shard_rel_prefix, shard_idx); const std::string corpus_fname = fs::path{destination.dir_path} / corpus_rel_fname;
diff --git a/centipede/seed_corpus_maker_lib.h b/centipede/seed_corpus_maker_lib.h index a20e121..e4d15c4 100644 --- a/centipede/seed_corpus_maker_lib.h +++ b/centipede/seed_corpus_maker_lib.h
@@ -17,6 +17,7 @@ #include <cstdint> #include <iostream> +#include <optional> #include <string> #include <string_view> #include <utility> @@ -37,12 +38,18 @@ // difference is commented below. struct SeedCorpusSource { std::string dir_glob; + // Source directory in addition to scanning `dir_glob`. + std::vector<std::string> src_dirs; uint32_t num_recent_dirs; std::string shard_rel_glob; + // A path prefix relative to the source dirs used to find corpus shards. + std::optional<std::string> shard_rel_prefix; // If non-empty, will be used to glob the individual input files (with one // input in each file) in the source dirs. Any files matching `shard_rel_glob` // will be skipped. std::string individual_input_rel_glob; + // A path prefix relative to the source dirs to find individual inputs. + std::optional<std::string> individual_input_rel_prefix; std::variant<float, uint32_t> sampled_fraction_or_count; std::string features_start_point; @@ -56,6 +63,8 @@ struct SeedCorpusDestination { std::string dir_path; std::string shard_rel_glob; + // A path prefix relative to `dir_path` to generate output shards. + std::optional<std::string> shard_rel_prefix; uint32_t shard_index_digits; uint32_t num_shards; };
diff --git a/centipede/seed_corpus_maker_lib_test.cc b/centipede/seed_corpus_maker_lib_test.cc index f7a1501..a98ac49 100644 --- a/centipede/seed_corpus_maker_lib_test.cc +++ b/centipede/seed_corpus_maker_lib_test.cc
@@ -14,11 +14,10 @@ #include "./centipede/seed_corpus_maker_lib.h" -#include <unistd.h> - #include <cmath> #include <cstddef> #include <filesystem> // NOLINT +#include <optional> #include <string> #include <string_view> #include <vector> @@ -40,6 +39,7 @@ using ::testing::IsSubsetOf; using ::testing::IsSupersetOf; +using ::testing::UnitTest; inline constexpr auto kIdxDigits = WorkDir::kDigitsInShardIndex; @@ -93,9 +93,17 @@ << VV(workdir); } -TEST(SeedCorpusMakerLibTest, RoundTripWriteReadWrite) { - const fs::path test_dir = GetTestTempDir(test_info_->name()); - chdir(test_dir.c_str()); +struct SeedCorpusMakerLibTestParam { + bool use_globs; +}; + +class SeedCorpusMakerLibTest + : public testing::TestWithParam<SeedCorpusMakerLibTestParam> {}; + +TEST_P(SeedCorpusMakerLibTest, RoundTripWriteReadWrite) { + const fs::path test_dir = + GetTestTempDir(UnitTest::GetInstance()->current_test_info()->name()); + fs::current_path(test_dir); const InputAndFeaturesVec kElements = { {{0}, {}}, @@ -117,13 +125,18 @@ constexpr size_t kNumShards = 2; const SeedCorpusDestination destination = { /*dir_path=*/std::string(kRelDir1), - /*shard_rel_glob=*/absl::StrCat("distilled-", kCovBin, ".*"), + /*shard_rel_glob=*/GetParam().use_globs + ? absl::StrCat("distilled-", kCovBin, ".*") + : "", + /*shard_rel_prefix=*/GetParam().use_globs + ? std::nullopt + : std::make_optional(absl::StrCat("distilled-", kCovBin, ".")), /*shard_index_digits=*/kIdxDigits, /*num_shards=*/kNumShards, }; ASSERT_OK(WriteSeedCorpusElementsToDestination( // kElements, kCovBin, kCovHash, destination)); - const std::string workdir = (test_dir / kRelDir1).c_str(); + const std::string workdir = (test_dir / kRelDir1).string(); ASSERT_NO_FATAL_FAILURE(VerifyShardsExist( // workdir, kCovBin, kCovHash, kNumShards, ShardType::kDistilled)); } @@ -133,9 +146,14 @@ { for (const float fraction : {1.0, 0.5, 0.2}) { SeedCorpusSource source; - source.dir_glob = std::string(kRelDir1); source.num_recent_dirs = 2; - source.shard_rel_glob = absl::StrCat("distilled-", kCovBin, ".*"); + if (GetParam().use_globs) { + source.dir_glob = std::string(kRelDir1); + source.shard_rel_glob = absl::StrCat("distilled-", kCovBin, ".*"); + } else { + source.src_dirs = {std::string(kRelDir1)}; + source.shard_rel_prefix = absl::StrCat("distilled-", kCovBin, "."); + } source.sampled_fraction_or_count = fraction; InputAndFeaturesVec elements; @@ -154,16 +172,24 @@ constexpr size_t kNumShards = 3; SeedCorpusSource source; - source.dir_glob = std::string(kRelDir1); source.num_recent_dirs = 1; - source.shard_rel_glob = absl::StrCat("distilled-", kCovBin, ".*"); + if (GetParam().use_globs) { + source.dir_glob = std::string(kRelDir1); + source.shard_rel_glob = absl::StrCat("distilled-", kCovBin, ".*"); + } else { + source.src_dirs = {std::string(kRelDir1)}; + source.shard_rel_prefix = absl::StrCat("distilled-", kCovBin, "."); + } source.sampled_fraction_or_count = 1.0f; const SeedCorpusConfig config = { /*sources=*/{{source}}, /*destination=*/ { /*dir_path=*/std::string(kRelDir2), - /*shard_rel_glob=*/"corpus.*", + /*shard_rel_glob=*/GetParam().use_globs ? "corpus.*" : "", + /*shard_rel_prefix=*/GetParam().use_globs + ? std::nullopt + : std::optional<std::string>("corpus."), /*shard_index_digits=*/kIdxDigits, /*num_shards=*/kNumShards, }, @@ -172,16 +198,17 @@ { ASSERT_OK(GenerateSeedCorpusFromConfig( // config, kCovBin, kCovHash)); - const std::string workdir = (test_dir / kRelDir2).c_str(); + const std::string workdir = (test_dir / kRelDir2).string(); ASSERT_NO_FATAL_FAILURE(VerifyShardsExist( // workdir, kCovBin, kCovHash, kNumShards, ShardType::kNormal)); } } } -TEST(SeedCorpusMakerLibTest, LoadsBothIndividualInputsAndShardsFromSource) { - const fs::path test_dir = GetTestTempDir(test_info_->name()); - chdir(test_dir.c_str()); +TEST_P(SeedCorpusMakerLibTest, LoadsBothIndividualInputsAndShardsFromSource) { + const fs::path test_dir = + GetTestTempDir(UnitTest::GetInstance()->current_test_info()->name()); + fs::current_path(test_dir); const InputAndFeaturesVec kShardedInputs = { {{0}, {}}, @@ -202,13 +229,18 @@ constexpr size_t kNumShards = 2; const SeedCorpusDestination destination = { /*dir_path=*/std::string(kRelDir), - /*shard_rel_glob=*/absl::StrCat("distilled-", kCovBin, ".*"), + /*shard_rel_glob=*/GetParam().use_globs + ? absl::StrCat("distilled-", kCovBin, ".*") + : "", + /*shard_rel_prefix=*/GetParam().use_globs + ? std::nullopt + : std::make_optional(absl::StrCat("distilled-", kCovBin, ".")), /*shard_index_digits=*/kIdxDigits, /*num_shards=*/kNumShards, }; FUZZTEST_CHECK_OK(WriteSeedCorpusElementsToDestination( // kShardedInputs, kCovBin, kCovHash, destination)); - const std::string workdir = (test_dir / kRelDir).c_str(); + const std::string workdir = (test_dir / kRelDir).string(); ASSERT_NO_FATAL_FAILURE(VerifyShardsExist( // workdir, kCovBin, kCovHash, kNumShards, ShardType::kDistilled)); } @@ -226,12 +258,23 @@ InputAndFeaturesVec elements; ASSERT_OK(SampleSeedCorpusElementsFromSource( // SeedCorpusSource{ - /*dir_glob=*/std::string(kRelDir), + /*dir_glob=*/GetParam().use_globs ? std::string(kRelDir) : "", + /*src_dirs=*/GetParam().use_globs + ? std::vector<std::string>{} + : std::vector<std::string>{std::string{kRelDir}}, /*num_recent_dirs=*/1, - /*shard_rel_glob=*/absl::StrCat("distilled-", kCovBin, ".*"), + /*shard_rel_glob=*/GetParam().use_globs + ? absl::StrCat("distilled-", kCovBin, ".*") + : "", + /*shard_rel_prefix=*/GetParam().use_globs + ? std::nullopt + : std::make_optional(absl::StrCat("distilled-", kCovBin, ".")), // Intentionally try to match the shard files and test if they will // be read as individual inputs. - /*individual_input_rel_glob=*/"*", + /*individual_input_rel_glob=*/GetParam().use_globs ? "*" : "", + /*individual_input_rel_prefix=*/GetParam().use_globs + ? std::nullopt + : std::optional<std::string>{""}, /*sampled_fraction_or_count=*/1.0f, }, kCovBin, kCovHash, elements)); @@ -244,9 +287,10 @@ } } -TEST(SeedCorpusMakerLibTest, FeaturesStartPointCanDropFeatures) { - const fs::path test_dir = GetTestTempDir(test_info_->name()); - chdir(test_dir.c_str()); +TEST_P(SeedCorpusMakerLibTest, FeaturesStartPointCanDropFeatures) { + const fs::path test_dir = + GetTestTempDir(UnitTest::GetInstance()->current_test_info()->name()); + fs::current_path(test_dir); const InputAndFeaturesVec kElementsSrc1 = { {{0}, {}}, @@ -269,7 +313,12 @@ constexpr size_t kNumShards = 2; const SeedCorpusDestination destination_src1 = { /*dir_path=*/std::string(kSrcDir1), - /*shard_rel_glob=*/absl::StrCat("distilled-", kCovBin, ".*"), + /*shard_rel_glob=*/GetParam().use_globs + ? absl::StrCat("distilled-", kCovBin, ".*") + : "", + /*shard_rel_prefix=*/GetParam().use_globs + ? std::nullopt + : std::make_optional(absl::StrCat("distilled-", kCovBin, ".")), /*shard_index_digits=*/kIdxDigits, /*num_shards=*/kNumShards, }; @@ -278,7 +327,12 @@ const SeedCorpusDestination destination_src2 = { /*dir_path=*/std::string(kSrcDir2), - /*shard_rel_glob=*/absl::StrCat("distilled-", kCovBin, ".*"), + /*shard_rel_glob=*/GetParam().use_globs + ? absl::StrCat("distilled-", kCovBin, ".*") + : "", + /*shard_rel_prefix=*/GetParam().use_globs + ? std::nullopt + : std::make_optional(absl::StrCat("distilled-", kCovBin, ".")), /*shard_index_digits=*/kIdxDigits, /*num_shards=*/kNumShards, }; @@ -287,9 +341,17 @@ } SeedCorpusSource source; - source.dir_glob = std::string("dir/src/*"); + if (GetParam().use_globs) { + source.dir_glob = std::string("dir/src/*"); + } else { + source.src_dirs = {std::string{kSrcDir1}, std::string{kSrcDir2}}; + } source.num_recent_dirs = 2; - source.shard_rel_glob = absl::StrCat("distilled-", kCovBin, ".*"); + if (GetParam().use_globs) { + source.shard_rel_glob = absl::StrCat("distilled-", kCovBin, ".*"); + } else { + source.shard_rel_prefix = absl::StrCat("distilled-", kCovBin, "."); + } source.sampled_fraction_or_count = 1.0f; InputAndFeaturesVec elements; @@ -316,5 +378,10 @@ ASSERT_EQ(get_num_features(elements), get_num_features(kElementsSrc2)); } +INSTANTIATE_TEST_SUITE_P( + SeedCorpusMakerLibTestWithConfigurations, SeedCorpusMakerLibTest, + testing::Values(SeedCorpusMakerLibTestParam{/*use_globs=*/false}, + SeedCorpusMakerLibTestParam{/*use_globs=*/true})); + } // namespace } // namespace fuzztest::internal
diff --git a/centipede/seed_corpus_maker_proto_lib_test.cc b/centipede/seed_corpus_maker_proto_lib_test.cc index 29ab4f6..f400ebc 100644 --- a/centipede/seed_corpus_maker_proto_lib_test.cc +++ b/centipede/seed_corpus_maker_proto_lib_test.cc
@@ -16,6 +16,7 @@ #include <cstddef> #include <filesystem> // NOLINT +#include <optional> #include <string> #include <string_view> @@ -111,6 +112,7 @@ const SeedCorpusDestination destination = { /*dir_path=*/std::string(kRelDir1), /*shard_rel_glob=*/absl::StrCat("distilled-", kCovBin, ".*"), + /*shard_rel_prefix=*/std::nullopt, /*shard_index_digits=*/kIdxDigits, /*num_shards=*/2, };
diff --git a/centipede/symbol_table.cc b/centipede/symbol_table.cc index e693b48..6e58ebd 100644 --- a/centipede/symbol_table.cc +++ b/centipede/symbol_table.cc
@@ -120,10 +120,9 @@ "--no-inlines", "-e", std::string(dso_path), - "<", - std::string(pcs_file.path()), }; - cmd_options.stdout_file_prefix = symbols_file_prefix; + cmd_options.stdin_file_path = std::string(pcs_file.path()); + cmd_options.stdout_file_prefix = symbols_file_prefix.string(); Command cmd{symbolizer_path, std::move(cmd_options)}; int exit_code = cmd.Execute(); if (exit_code != EXIT_SUCCESS) {
diff --git a/centipede/util.cc b/centipede/util.cc index 583cc1f..8a9064e 100644 --- a/centipede/util.cc +++ b/centipede/util.cc
@@ -17,8 +17,12 @@ #include "./centipede/util.h" +#ifdef _WIN32 +#include "./common/windows_includes.h" +#else #include <sys/mman.h> #include <unistd.h> +#endif #include <algorithm> #include <cctype> @@ -47,6 +51,7 @@ #include "absl/base/const_init.h" #include "absl/base/nullability.h" #include "absl/base/thread_annotations.h" +#include "absl/strings/match.h" #include "absl/strings/str_format.h" #include "absl/strings/str_replace.h" #include "absl/strings/str_split.h" @@ -63,8 +68,13 @@ size_t GetRandomSeed(size_t seed) { if (seed != 0) return seed; +#ifdef _WIN32 + return time(nullptr) + GetCurrentProcessId() + + std::hash<std::thread::id>{}(std::this_thread::get_id()); +#else return time(nullptr) + getpid() + std::hash<std::thread::id>{}(std::this_thread::get_id()); +#endif } std::string AsPrintableString(ByteSpan data, size_t max_len) { @@ -83,7 +93,7 @@ template <typename Container> void ReadFromLocalFile(std::string_view file_path, Container &data) { - std::ifstream f(std::string{file_path}); + std::ifstream f(std::string{file_path}, std::ios::in | std::ios::binary); if (!f) return; f.seekg(0, std::ios_base::end); auto size = f.tellg(); @@ -112,12 +122,13 @@ } void ClearLocalFileContents(std::string_view file_path) { - std::ofstream f(std::string{file_path}, std::ios::out | std::ios::trunc); + std::ofstream f(std::string{file_path}, + std::ios::out | std::ios::trunc | std::ios::binary); FUZZTEST_CHECK(f) << "Failed to clear the file: " << file_path; } void WriteToLocalFile(std::string_view file_path, ByteSpan data) { - std::ofstream f(std::string{file_path}); + std::ofstream f(std::string{file_path}, std::ios::out | std::ios::binary); FUZZTEST_CHECK(f) << "Failed to open local file: " << file_path; f.write(reinterpret_cast<const char *>(data.data()), static_cast<int64_t>(data.size())); @@ -136,13 +147,15 @@ void WriteToLocalHashedFileInDir(std::string_view dir_path, ByteSpan data) { if (dir_path.empty()) return; - std::string file_path = std::filesystem::path(dir_path).append(Hash(data)); + std::string file_path = + std::filesystem::path(dir_path).append(Hash(data)).string(); WriteToLocalFile(file_path, data); } void WriteToRemoteHashedFileInDir(std::string_view dir_path, ByteSpan data) { if (dir_path.empty()) return; - std::string file_path = std::filesystem::path(dir_path).append(Hash(data)); + std::string file_path = + std::filesystem::path(dir_path).append(Hash(data)).string(); FUZZTEST_CHECK_OK( RemoteFileSetContents(file_path, std::string(data.begin(), data.end()))); } @@ -155,17 +168,24 @@ } std::string ProcessAndThreadUniqueID(std::string_view prefix) { - // operator << is the only way to serialize std::this_thread::get_id(). std::ostringstream oss; +#ifdef _WIN32 + oss << prefix << GetCurrentProcessId() << "-" << GetCurrentThreadId(); +#else + // operator << is the only way to serialize std::this_thread::get_id(). oss << prefix << getpid() << "-" << std::this_thread::get_id(); +#endif return oss.str(); } std::string TemporaryLocalDirPath() { const char *TMPDIR = getenv("TMPDIR"); + if (!TMPDIR) TMPDIR = getenv("TEMP"); + if (!TMPDIR) TMPDIR = getenv("TMP"); std::string tmp = TMPDIR ? TMPDIR : "/tmp"; - return std::filesystem::path(tmp).append( - ProcessAndThreadUniqueID("centipede-")); + return std::filesystem::path(tmp) + .append(ProcessAndThreadUniqueID("centipede-")) + .string(); } // We need to maintain a global set of dirs that CreateLocalDirRemovedAtExit() @@ -189,13 +209,19 @@ void CreateLocalDirRemovedAtExit(std::string_view path) { // Safeguard against removing dirs not created by TemporaryLocalDirPath(). - FUZZTEST_CHECK_NE(path.find("/centipede-"), std::string::npos); + FUZZTEST_CHECK(absl::StrContains(path, "/centipede-") || + absl::StrContains(path, "\\centipede-")); // Create the dir. std::error_code error; - std::filesystem::remove_all(path, error); - FUZZTEST_LOG_IF(ERROR, error) - << "Unable to clean up existing dir " << path << ": " << error.message(); - std::filesystem::create_directories(path); + std::filesystem::path p(path); + if (std::filesystem::exists(p, error)) { + std::filesystem::remove_all(p, error); + FUZZTEST_LOG_IF(ERROR, error) + << "Unable to clean up existing dir " << p << ": " << error.message(); + } + std::filesystem::create_directories(p, error); + FUZZTEST_CHECK(!error) << "Failed to create local dir " << p << ": " + << error.message(); // Add to dirs_to_delete_at_exit. absl::MutexLock lock(dirs_to_delete_at_exit_mutex); if (!dirs_to_delete_at_exit) { @@ -206,7 +232,7 @@ } ScopedFile::ScopedFile(std::string_view dir_path, std::string_view name) - : my_path_(std::filesystem::path(dir_path) / name) {} + : my_path_((std::filesystem::path(dir_path) / name).string()) {} ScopedFile::~ScopedFile() { std::error_code error; @@ -359,16 +385,85 @@ return res; } +#ifdef _WIN32 +// On Windows, we use the first page for the magic cookies of mmapped regions so +// that our VEH can handle it properly. +static constexpr std::string_view kMmapMagicCookie = "CENTIPED"; + +static const auto page_size = []() { + SYSTEM_INFO si; + GetSystemInfo(&si); + return static_cast<size_t>(si.dwPageSize); +}(); + +static LONG CALLBACK +AutoCommitPageFaultHandler(PEXCEPTION_POINTERS exception_info) { + auto record = exception_info->ExceptionRecord; + if (record->ExceptionCode != EXCEPTION_ACCESS_VIOLATION || + record->NumberParameters < 2) { + return EXCEPTION_CONTINUE_SEARCH; + } + auto fault_addr = reinterpret_cast<LPVOID>(record->ExceptionInformation[1]); + MEMORY_BASIC_INFORMATION mbi; + if (VirtualQuery(fault_addr, &mbi, sizeof(mbi)) != sizeof(mbi)) { + return EXCEPTION_CONTINUE_SEARCH; + } + if (mbi.State != MEM_RESERVE) { + return EXCEPTION_CONTINUE_SEARCH; + } + auto cookie_addr = reinterpret_cast<const uint8_t*>(mbi.AllocationBase); + if (VirtualQuery(cookie_addr, &mbi, sizeof(mbi)) != sizeof(mbi)) { + return EXCEPTION_CONTINUE_SEARCH; + } + if (mbi.State != MEM_COMMIT) { + return EXCEPTION_CONTINUE_SEARCH; + } + if (std::memcmp(cookie_addr, kMmapMagicCookie.data(), + kMmapMagicCookie.size()) != 0) { + return EXCEPTION_CONTINUE_SEARCH; + } + if (VirtualAlloc(fault_addr, 1, MEM_COMMIT, PAGE_READWRITE) == nullptr) { + return EXCEPTION_CONTINUE_SEARCH; + } + return EXCEPTION_CONTINUE_EXECUTION; +} +#endif + uint8_t *MmapNoReserve(size_t size) { +#ifdef _WIN32 + // Set up page fault handler to commit page on demand. + [[maybe_unused]] static bool installed_veh = []() { + // Must use `First=0` as it could otherwise conflict with e.g. sanitizers. + AddVectoredExceptionHandler(/*First=*/0, AutoCommitPageFaultHandler); + return true; + }(); + // MEM_RESERVE has different semantics and does not contradict with + // MAP_NORESERVE for mmap. + auto result = reinterpret_cast<uint8_t*>( + VirtualAlloc(nullptr, size + page_size, MEM_RESERVE, PAGE_READWRITE)); + FUZZTEST_CHECK(result != nullptr) + << "VirtualAlloc failed for size " << size << " err=" << GetLastError(); + FUZZTEST_CHECK(VirtualAlloc(result, kMmapMagicCookie.size(), MEM_COMMIT, + PAGE_READWRITE) != nullptr) + << "VirtualAlloc failed to commit the memory region cookie"; + std::memcpy(result, kMmapMagicCookie.data(), kMmapMagicCookie.size()); + return result + page_size; +#else auto result = mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); FUZZTEST_CHECK(result != MAP_FAILED); return reinterpret_cast<uint8_t *>(result); +#endif } void Munmap(uint8_t *ptr, size_t size) { +#ifdef _WIN32 + BOOL result = VirtualFree(ptr - page_size, 0, MEM_RELEASE); + FUZZTEST_CHECK(result != 0); +#else auto result = munmap(ptr, size); FUZZTEST_CHECK_EQ(result, 0); +#endif } int PollTimeoutMs(absl::Duration timeout) {
diff --git a/centipede/util_test.cc b/centipede/util_test.cc index 6f26650..52041aa 100644 --- a/centipede/util_test.cc +++ b/centipede/util_test.cc
@@ -32,6 +32,10 @@ #include "./common/hash.h" #include "./common/logging.h" +#ifdef _WIN32 +#define setenv(n, v, _r) _putenv_s(n, v) +#endif + namespace fuzztest::internal { TEST(UtilTest, AsString) { @@ -116,7 +120,8 @@ auto temp_dir = TemporaryLocalDirPath(); // Create dir, create a file there, write to file, read from it, remove dir. std::filesystem::create_directories(temp_dir); - std::string temp_file_path = std::filesystem::path(temp_dir).append("blah"); + std::string temp_file_path = + std::filesystem::path(temp_dir).append("blah").string(); ByteArray written_data{1, 2, 3}; WriteToLocalFile(temp_file_path, written_data); ByteArray read_data; @@ -162,8 +167,8 @@ EXPECT_TRUE(std::filesystem::exists(tmpdir)); setenv("CENTIPEDE_UTIL_TEST_TEMP_DIR", tmpdir.c_str(), 1); // Create two subdirs via CreateLocalDirRemovedAtExit. - std::string subdir1 = std::filesystem::path(tmpdir).append("1"); - std::string subdir2 = std::filesystem::path(tmpdir).append("2"); + std::string subdir1 = std::filesystem::path(tmpdir).append("1").string(); + std::string subdir2 = std::filesystem::path(tmpdir).append("2").string(); CreateLocalDirRemovedAtExit(subdir1); CreateLocalDirRemovedAtExit(subdir2); EXPECT_TRUE(std::filesystem::exists(subdir1)); @@ -293,6 +298,15 @@ testing::ElementsAre(std::vector<int>{1}, std::vector<int>{3})); } +TEST(UtilTest, MmapTest) { + static constexpr size_t kBufSize = 1 << 30; // 1 GiB + auto* buf = MmapNoReserve(kBufSize); + ASSERT_NE(buf, nullptr); + EXPECT_EQ(buf[1234], 0); + EXPECT_EQ(buf[567890], 0); + Munmap(buf, kBufSize); +} + TEST(UtilTest, PollTimeoutMsWorks) { EXPECT_GT(PollTimeoutMs(absl::ZeroDuration()), 0); EXPECT_GT(PollTimeoutMs(-absl::InfiniteDuration()), 0);
diff --git a/centipede/workdir.h b/centipede/workdir.h index 5192b93..d8f6a1d 100644 --- a/centipede/workdir.h +++ b/centipede/workdir.h
@@ -51,6 +51,8 @@ // Returns the shard index of `path` if it is a shard parth, `nullopt` // otherwise. std::optional<size_t> GetShardIndex(std::string_view path) const; + // Gets the common prefix of the sharded path. + const std::string& prefix() const { return prefix_; } private: friend class WorkDir;
diff --git a/common/BUILD b/common/BUILD index bbdb792..296b140 100644 --- a/common/BUILD +++ b/common/BUILD
@@ -208,6 +208,7 @@ ":blob_file", ":defs", ":logging", + ":windows_includes", "@abseil-cpp//absl/strings", "@abseil-cpp//absl/strings:str_format", "@googletest//:gtest", @@ -225,6 +226,11 @@ ], ) +cc_library( + name = "windows_includes", + hdrs = ["windows_includes.h"], +) + ### Tests cc_test(
diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 671195e..b775763 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt
@@ -150,6 +150,13 @@ fuzztest_cc_library( NAME + windows_includes + HDRS + "windows_includes.h" +) + +fuzztest_cc_library( + NAME test_util HDRS "test_util.h" @@ -159,6 +166,7 @@ fuzztest::common_logging fuzztest::blob_file fuzztest::defs + fuzztest::windows_includes absl::strings absl::str_format GTest::gtest
diff --git a/common/test_util.cc b/common/test_util.cc index 3232d55..8bc7063 100644 --- a/common/test_util.cc +++ b/common/test_util.cc
@@ -18,10 +18,20 @@ #include <string_view> #include <system_error> // NOLINT +#ifdef _WIN32 +#include "./common/windows_includes.h" +#else +#include <unistd.h> +#endif + #include "gtest/gtest.h" #include "absl/strings/str_cat.h" #include "./common/logging.h" +#ifdef _WIN32 +#define setenv(n, v, _r) _putenv_s(n, v) +#endif + namespace fuzztest::internal { std::filesystem::path GetTestTempDir(std::string_view subdir) { @@ -35,11 +45,18 @@ FUZZTEST_CHECK(!error) << "Failed to create dir: " VV(dir) << error.message(); } - return std::filesystem::canonical(dir); + return std::filesystem::absolute(dir); } std::string GetTempFilePath(std::string_view subdir, size_t i) { - return GetTestTempDir(subdir) / absl::StrCat("tmp.", getpid(), ".", i); + return (GetTestTempDir(subdir) / absl::StrCat("tmp.", +#ifdef _WIN32 + GetCurrentProcessId(), +#else + getpid(), +#endif + ".", i)) + .string(); } std::filesystem::path GetTestRunfilesDir() { @@ -59,8 +76,15 @@ const auto runfiles_dir = GetTestRunfilesDir(); auto path = runfiles_dir; path.append(rel_path); - FUZZTEST_CHECK(std::filesystem::exists(path)) // - << "No such path: " << VV(path) << VV(runfiles_dir) << VV(rel_path); + std::error_code ec; +#ifdef _WIN32 + auto win_path = path; + win_path += ".exe"; + if (std::filesystem::exists(win_path, ec)) return win_path; +#endif + FUZZTEST_CHECK(std::filesystem::exists(path, ec)) // + << "No such path: " << VV(path) << VV(runfiles_dir) << VV(rel_path) + << VV(ec); return path; }
diff --git a/common/test_util.h b/common/test_util.h index c859bff..e363f18 100644 --- a/common/test_util.h +++ b/common/test_util.h
@@ -79,7 +79,7 @@ const std::filesystem::path& path() const { return path_; } std::string GetFilePath(std::string_view file_name) const { - return path_ / file_name; + return (path_ / file_name).string(); } std::string CreateSubdir(std::string_view name) const {
diff --git a/common/windows_includes.h b/common/windows_includes.h new file mode 100644 index 0000000..e946797 --- /dev/null +++ b/common/windows_includes.h
@@ -0,0 +1,39 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef FUZZTEST_COMMON_WINDOWS_INCLUDES_H_ +#define FUZZTEST_COMMON_WINDOWS_INCLUDES_H_ + +#ifdef _WIN32 + +// clang-format off +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif + +#ifndef NOMINMAX +#define NOMINMAX +#endif + +#ifndef NOGDI +#define NOGDI +#endif + +#include <windows.h> + +// clang-format on + +#endif // _WIN32 + +#endif // FUZZTEST_COMMON_WINDOWS_INCLUDES_H_