Skip to content

Commit

Permalink
Update the Centipede environment's timeout_per_input with target bina…
Browse files Browse the repository at this point in the history 
…ry config.

PiperOrigin-RevId: 693431008
  • Loading branch information
@fniksic @copybara-github
fniksic authored and copybara-github committed Nov 5, 2024
1 parent 4e8f47b commit f4e27f3
Show file tree
Hide file tree
Showing 4 changed files with 98 additions and 38 deletions.
47 changes: 47 additions & 0 deletions centipede/environment.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#include <algorithm>
#include <bitset>
#include <charconv>
#include <cmath>
#include <cstddef>
#include <cstdint>
#include <string>
Expand All @@ -39,6 +40,28 @@
#include "./fuzztest/internal/configuration.h"

namespace centipede {
namespace {

size_t ComputeTimeoutPerBatch(size_t timeout_per_input, size_t batch_size) {
CHECK_GT(batch_size, 0);
// NOTE: If `timeout_per_input` == 0, leave `timeout_per_batch` at 0 too:
// the implementation interprets both as "no limit".
if (timeout_per_input == 0) return 0;
// TODO(ussuri): The formula here is an unscientific heuristic conjured
// up for CPU instruction fuzzing. `timeout_per_input` is interpreted as
// the long tail of the input runtime distribution of yet-unknown nature.
// It might be the exponential, log-normal distribution or similar, and
// the distribution of the total time per batch could be modeled by the
// gamma distribution. Work out the math later. Right now, this naive
// formula gives ~18 min per batch with the input flags' defaults (this
// has worked in test runs so far).
constexpr double kScale = 12;
const double estimated_mean_time_per_input =
std::max(timeout_per_input / kScale, 1.0);
return std::ceil(std::log(estimated_mean_time_per_input + 1.0) * batch_size);
}

} // namespace

const Environment &Environment::Default() {
static absl::NoDestructor<Environment> default_env;
Expand Down Expand Up @@ -225,6 +248,23 @@ void Environment::UpdateWithTargetConfig(
num_threads = config.jobs;
my_shard_index = 0;
}

if (config.time_limit_per_input < absl::InfiniteDuration()) {
CHECK_GE(config.time_limit_per_input, absl::Seconds(1))
<< "Time limit per input must not be less than one second";
const auto time_limit_per_input_sec =
static_cast<size_t>(absl::ToInt64Seconds(config.time_limit_per_input));
CHECK(timeout_per_input == Default().timeout_per_input ||
timeout_per_input == time_limit_per_input_sec)
<< "Value for --timeout_per_input is inconsistent with the value for "
"time_limit_per_input in the target binary:"
<< VV(timeout_per_input) << VV(config.time_limit_per_input);
size_t autocomputed_timeout_per_batch =
ComputeTimeoutPerBatch(timeout_per_input, batch_size);
timeout_per_input = time_limit_per_input_sec;
UpdateTimeoutPerBatchIfEqualTo(autocomputed_timeout_per_batch);
}

if (const auto test_time_limit = config.GetTimeLimitPerTest();
test_time_limit < absl::InfiniteDuration()) {
CHECK(test_time_limit >= absl::Seconds(1))
Expand All @@ -238,4 +278,11 @@ void Environment::UpdateWithTargetConfig(
}
}

void Environment::UpdateTimeoutPerBatchIfEqualTo(size_t val) {
if (timeout_per_batch != val) return;
timeout_per_batch = ComputeTimeoutPerBatch(timeout_per_input, batch_size);
VLOG(1) << "--timeout_per_batch auto-computed: " << timeout_per_batch
<< " sec (see --help for details)";
}

} // namespace centipede
4 changes: 4 additions & 0 deletions centipede/environment.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ struct Environment {
// if the fields are non-default and inconsistent with the corresponding
// values in `config`.
void UpdateWithTargetConfig(const fuzztest::internal::Configuration& config);
// If `timeout_per_batch` is `val`, computes it as a function of
// `timeout_per_input` and `batch_size` and updates it. Otherwise, leaves it
// unchanged.
void UpdateTimeoutPerBatchIfEqualTo(size_t val);
};

} // namespace centipede
Expand Down
41 changes: 4 additions & 37 deletions centipede/environment_flags.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@

#include "./centipede/environment_flags.h"

#include <algorithm>
#include <cmath>
#include <cstdint>
#include <cstdlib>
#include <filesystem> // NOLINT
Expand Down Expand Up @@ -426,37 +424,6 @@ absl::Time GetStopAtTime(absl::Time stop_at, absl::Duration stop_after) {
}
}
// If the passed `timeout_per_batch` is 0, computes its value as a function of
// `timeout_per_input` and `batch_size` and returns it. Otherwise, just returns
// the `timeout_per_batch`.
size_t ComputeTimeoutPerBatch( //
size_t timeout_per_batch, size_t timeout_per_input, size_t batch_size) {
if (timeout_per_batch == 0) {
CHECK_GT(batch_size, 0);
// NOTE: If `timeout_per_input` == 0, leave `timeout_per_batch` at 0 too:
// the implementation interprets both as "no limit".
if (timeout_per_input != 0) {
// TODO(ussuri): The formula here is an unscientific heuristic conjured
// up for CPU instruction fuzzing. `timeout_per_input` is interpreted as
// the long tail of the input runtime distribution of yet-unknown nature.
// It might be the exponential, log-normal distribution or similar, and
// the distribution of the total time per batch could be modeled by the
// gamma distribution. Work out the math later. Right now, this naive
// formula gives ~18 min per batch with the input flags' defaults (this
// has worked in test runs so far).
constexpr double kScale = 12;
const double estimated_mean_time_per_input =
std::max(timeout_per_input / kScale, 1.0);
timeout_per_batch =
std::ceil(std::log(estimated_mean_time_per_input + 1.0) * batch_size);
}
VLOG(1) << "--timeout_per_batch"
<< " not set on command line: auto-computed " << timeout_per_batch
<< " sec (see --help for details)";
}
return timeout_per_batch;
}

} // namespace
Environment CreateEnvironmentFromFlags(const std::vector<std::string> &argv) {
Expand Down Expand Up @@ -490,10 +457,7 @@ Environment CreateEnvironmentFromFlags(const std::vector<std::string> &argv) {
.address_space_limit_mb = absl::GetFlag(FLAGS_address_space_limit_mb),
.rss_limit_mb = absl::GetFlag(FLAGS_rss_limit_mb),
.timeout_per_input = absl::GetFlag(FLAGS_timeout_per_input),
.timeout_per_batch =
ComputeTimeoutPerBatch(absl::GetFlag(FLAGS_timeout_per_batch),
absl::GetFlag(FLAGS_timeout_per_input),
absl::GetFlag(FLAGS_batch_size)),
.timeout_per_batch = absl::GetFlag(FLAGS_timeout_per_batch),
.stop_at = GetStopAtTime(absl::GetFlag(FLAGS_stop_at),
absl::GetFlag(FLAGS_stop_after)),
.fork_server = absl::GetFlag(FLAGS_fork_server),
Expand Down Expand Up @@ -552,6 +516,9 @@ Environment CreateEnvironmentFromFlags(const std::vector<std::string> &argv) {
: absl::GetFlag(FLAGS_binary_hash),
};
env_from_flags.UpdateTimeoutPerBatchIfEqualTo(
Environment::Default().timeout_per_batch);
if (size_t j = absl::GetFlag(FLAGS_j)) {
env_from_flags.total_shards = j;
env_from_flags.num_threads = j;
Expand Down
44 changes: 43 additions & 1 deletion centipede/environment_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,49 @@ TEST(Environment, UpdateForExperiment) {
Experiment(11, true, 30, "E12", "use_cmp_features=true:path_level=30:");
}

TEST(Environment, UpdatesBatchTimeoutFromTargetConfig) {
TEST(Environment, UpdatesTimeoutPerBatchFromTimeoutPerInputAndBatchSize) {
Environment env{
.batch_size = 1000, .timeout_per_input = 100, .timeout_per_batch = 0};
env.UpdateTimeoutPerBatchIfEqualTo(0);
EXPECT_GT(env.timeout_per_batch, 0);

env.timeout_per_batch = 123;
env.UpdateTimeoutPerBatchIfEqualTo(0);
EXPECT_EQ(env.timeout_per_batch, 123);
}

TEST(Environment, UpdatesTimeoutPerInputFromTargetConfigTimeLimitPerInput) {
Environment env{.timeout_per_input =
Environment::Default().timeout_per_input};
fuzztest::internal::Configuration config{.time_limit_per_input =
absl::Seconds(456)};
env.UpdateWithTargetConfig(config);
EXPECT_EQ(env.timeout_per_input, 456);
}

TEST(Environment,
DiesOnInconsistentTimeoutPerInputAndTargetConfigTimeLimitPerInput) {
Environment env{.timeout_per_input = 123};
fuzztest::internal::Configuration config{.time_limit_per_input =
absl::Seconds(456)};
EXPECT_DEATH(
env.UpdateWithTargetConfig(config),
"Value for --timeout_per_input is inconsistent with the value for "
"time_limit_per_input");
}

TEST(Environment, UpdatesTimeoutPerBatchFromTargetConfigTimeLimitPerInput) {
Environment env{.timeout_per_input =
Environment::Default().timeout_per_input};
env.UpdateTimeoutPerBatchIfEqualTo(Environment::Default().timeout_per_batch);
size_t autocomputed_timeout_per_batch = env.timeout_per_batch;
fuzztest::internal::Configuration config{.time_limit_per_input =
absl::Seconds(456)};
env.UpdateWithTargetConfig(config);
EXPECT_NE(env.timeout_per_batch, autocomputed_timeout_per_batch);
}

TEST(Environment, UpdatesTimeoutPerBatchFromTargetConfigTimeLimit) {
Environment env;
fuzztest::internal::Configuration config{
.time_limit = absl::Seconds(123),
Expand Down

0 comments on commit f4e27f3

Please sign in to comment.