Add `--ignore_timeout_reports` to use timeout without reporting errors. #Centipede This is to be used when timeout reports are noisy. In this change, timeouts are almost hidden from the view of the controller (except that the controller can see an incomplete batch execution result). Having more visibility in controller is desired but not done yet. PiperOrigin-RevId: 740033429
diff --git a/centipede/centipede_callbacks.cc b/centipede/centipede_callbacks.cc index 23269e8..20ca6d9 100644 --- a/centipede/centipede_callbacks.cc +++ b/centipede/centipede_callbacks.cc
@@ -131,6 +131,9 @@ absl::StrCat("stack_limit_kb=", env_.stack_limit_kb), absl::StrCat("crossover_level=", env_.crossover_level), }; + if (env_.ignore_timeout_reports) { + flags.emplace_back("ignore_timeout_reports"); + } if (!disable_coverage) { flags.emplace_back(absl::StrCat("path_level=", env_.path_level)); if (env_.use_pc_features) flags.emplace_back("use_pc_features");
diff --git a/centipede/environment.h b/centipede/environment.h index 22d27b8..86fe54f 100644 --- a/centipede/environment.h +++ b/centipede/environment.h
@@ -68,6 +68,7 @@ size_t timeout_per_input = 60; size_t timeout_per_batch = 0; absl::Duration force_abort_timeout = absl::Minutes(15); + bool ignore_timeout_reports = false; absl::Time stop_at = absl::InfiniteFuture(); bool fork_server = true; bool full_sync = false;
diff --git a/centipede/environment_flags.cc b/centipede/environment_flags.cc index 325bac2..a650077 100644 --- a/centipede/environment_flags.cc +++ b/centipede/environment_flags.cc
@@ -166,6 +166,9 @@ "hangs (e.g., during stacktrace dumps). The default value is ", Environment::Default().force_abort_timeout, "; use 'inf' to disable.")); +ABSL_FLAG(bool, ignore_timeout_reports, + Environment::Default().ignore_timeout_reports, + "If set, will ignore reporting timeouts as errors."); ABSL_FLAG(absl::Time, stop_at, Environment::Default().stop_at, "Stop fuzzing in all shards (--total_shards) at approximately this " "time in ISO-8601/RFC-3339 format, e.g. 2023-04-06T23:35:02Z. " @@ -472,6 +475,7 @@ /*timeout_per_input=*/absl::GetFlag(FLAGS_timeout_per_input), /*timeout_per_batch=*/absl::GetFlag(FLAGS_timeout_per_batch), /*force_abort_timeout=*/absl::GetFlag(FLAGS_force_abort_timeout), + /*ignore_timeout_reports=*/absl::GetFlag(FLAGS_ignore_timeout_reports), /*stop_at=*/ GetStopAtTime(absl::GetFlag(FLAGS_stop_at), absl::GetFlag(FLAGS_stop_after)),
diff --git a/centipede/runner.cc b/centipede/runner.cc index 91e76ce..79a423a 100644 --- a/centipede/runner.cc +++ b/centipede/runner.cc
@@ -240,6 +240,7 @@ const char *units; uint64_t value; uint64_t limit; + bool ignore_report; const char *failure; }; const uint64_t input_start_time = state.input_start_time; @@ -247,25 +248,28 @@ if (input_start_time == 0 || batch_start_time == 0) return; const Resource resources[] = { {Resource{ - /*what =*/"Per-input timeout", - /*units =*/"sec", - /*value =*/curr_time - input_start_time, - /*limit =*/state.run_time_flags.timeout_per_input, - /*failure =*/kExecutionFailurePerInputTimeout.data(), + /*what=*/"Per-input timeout", + /*units=*/"sec", + /*value=*/curr_time - input_start_time, + /*limit=*/state.run_time_flags.timeout_per_input, + /*ignore_report=*/state.run_time_flags.ignore_timeout_reports != 0, + /*failure=*/kExecutionFailurePerInputTimeout.data(), }}, {Resource{ - /*what =*/"Per-batch timeout", - /*units =*/"sec", - /*value =*/curr_time - batch_start_time, - /*limit =*/state.run_time_flags.timeout_per_batch, - /*failure =*/kExecutionFailurePerBatchTimeout.data(), + /*what=*/"Per-batch timeout", + /*units=*/"sec", + /*value=*/curr_time - batch_start_time, + /*limit=*/state.run_time_flags.timeout_per_batch, + /*ignore_report=*/state.run_time_flags.ignore_timeout_reports != 0, + /*failure=*/kExecutionFailurePerBatchTimeout.data(), }}, {Resource{ - /*what =*/"RSS limit", - /*units =*/"MB", - /*value =*/GetPeakRSSMb(), - /*limit =*/state.run_time_flags.rss_limit_mb, - /*failure =*/kExecutionFailureRssLimitExceeded.data(), + /*what=*/"RSS limit", + /*units=*/"MB", + /*value=*/GetPeakRSSMb(), + /*limit=*/state.run_time_flags.rss_limit_mb, + /*ignore_report=*/false, + /*failure=*/kExecutionFailureRssLimitExceeded.data(), }}, }; for (const auto &resource : resources) { @@ -275,6 +279,15 @@ // `RunOneInput()` after all the work is done. static std::atomic<bool> already_handling_failure = false; if (!already_handling_failure.exchange(true)) { + if (resource.ignore_report) { + fprintf(stderr, + "========= %s exceeded: %" PRIu64 " > %" PRIu64 + " (%s); exiting without reporting as an error\n", + resource.what, resource.value, resource.limit, + resource.units); + std::_Exit(0); + // should not return here. + } fprintf(stderr, "========= %s exceeded: %" PRIu64 " > %" PRIu64 " (%s); exiting\n",
diff --git a/centipede/runner.h b/centipede/runner.h index fc30e1f..2060386 100644 --- a/centipede/runner.h +++ b/centipede/runner.h
@@ -72,6 +72,7 @@ std::atomic<uint64_t> rss_limit_mb; uint64_t crossover_level; uint64_t skip_seen_features : 1; + uint64_t ignore_timeout_reports : 1; }; // One such object is created in runner's TLS. @@ -177,7 +178,9 @@ /*stack_limit_kb=*/HasIntFlag(":stack_limit_kb=", 0), /*rss_limit_mb=*/HasIntFlag(":rss_limit_mb=", 0), /*crossover_level=*/HasIntFlag(":crossover_level=", 50), - /*skip_seen_features=*/HasFlag(":skip_seen_features:")}; + /*skip_seen_features=*/HasFlag(":skip_seen_features:"), + /*ignore_timeout_reports=*/HasFlag(":ignore_timeout_reports:"), + }; // Returns true iff `flag` is present. // Typical usage: pass ":some_flag:", i.e. the flag name surrounded with ':'.
diff --git a/centipede/testing/centipede_main_test.sh b/centipede/testing/centipede_main_test.sh index e786a42..a098045 100755 --- a/centipede/testing/centipede_main_test.sh +++ b/centipede/testing/centipede_main_test.sh
@@ -176,10 +176,37 @@ centipede::assert_regex_in_file "end-fuzz.*pair: [^0]" "${LOG}" } +test_timeouts() { + FUNC="${FUNCNAME[0]}" + WD="${TEST_TMPDIR}/${FUNC}/WD" + CORPUS="${TEST_TMPDIR}/${FUNC}/corpus" + LOG="${TEST_TMPDIR}/${FUNC}/log" + + centipede::ensure_empty_dir "${WD}" + centipede::ensure_empty_dir "${CORPUS}" + echo -n "slo" >"${CORPUS}"/input + + echo "============ ${FUNC}: fuzz with --timeout_per_input" + test_fuzz --workdir="${WD}" --corpus_dir="${CORPUS}" --num_runs=0 --timeout_per_input=2 | tee "${LOG}" + centipede::assert_regex_in_file "Failure.*: per-input-timeout-exceeded" "${LOG}" + centipede::assert_regex_in_file "end-fuzz:.*crash: 1" "${LOG}" + + centipede::ensure_empty_dir "${WD}" + centipede::ensure_empty_dir "${CORPUS}" + echo -n "slo" >"${CORPUS}"/input + + echo "============ ${FUNC}: fuzz with --timeout_per_input --ignore_timeout_reports" + test_fuzz --workdir="${WD}" --corpus_dir="${CORPUS}" --num_runs=0 --timeout_per_input=2 --ignore_timeout_reports | tee "${LOG}" + centipede::assert_regex_not_in_file "Failure.*: per-input-timeout-exceeded" "${LOG}" + centipede::assert_regex_not_in_file "end-fuzz:.*crash: 1" "${LOG}" +} + + centipede::test_crashing_target abort_test_fuzz "foo" "AbOrT" "I AM ABOUT TO ABORT" test_debug_symbols test_dictionary test_for_each_blob test_pcpair_features +test_timeouts echo "PASS"
diff --git a/centipede/testing/runner_test.sh b/centipede/testing/runner_test.sh index 3f7c693..526185f 100755 --- a/centipede/testing/runner_test.sh +++ b/centipede/testing/runner_test.sh
@@ -107,11 +107,17 @@ echo ======== Check timeout CENTIPEDE_RUNNER_FLAGS=":timeout_per_input=567:" "${target}" \ - 2>&1 | grep "timeout_per_input:.567" + 2>&1 | grep "timeout_per_input: 567" CENTIPEDE_RUNNER_FLAGS=":timeout_per_input=2:" "${target}" "${slo}" \ 2>&1 | grep "Per-input timeout exceeded" +{ + CENTIPEDE_RUNNER_FLAGS=":ignore_timeout_reports:timeout_per_input=2:" "${target}" "${slo}"; + echo "$?" > "${TEST_TMPDIR}/ignore_timeout_reports_exit_code"; +} 2>&1 | grep "Per-input timeout exceeded" | grep "exiting without reporting as an error" +((`cat "${TEST_TMPDIR}/ignore_timeout_reports_exit_code"` == 0)) + echo ======== Check stack limit check with stack_limit CENTIPEDE_RUNNER_FLAGS=":use_pc_features:stack_limit_kb=200:" "${target}" "${stk}" # must pass