domain_tests/in_grammar_domain_test.cc

// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Tests of domain `InJsonGrammar`, the one provided example of an InGrammar
// domain.

#include <string>

#include "gtest/gtest.h"
#include "absl/container/flat_hash_set.h"
#include "absl/random/random.h"
#include "./fuzztest/domain.h"  // IWYU pragma: keep
#include "./domain_tests/domain_testing.h"
#include "./fuzztest/grammars/json_grammar.h"
#include "./fuzztest/internal/serialization.h"
#include "nlohmann/json.hpp"

namespace fuzztest {
namespace {

TEST(InJsonGrammar, InitGeneratesDifferentValidJson) {
  absl::BitGen bitgen;
  auto domain = InJsonGrammar();
  absl::flat_hash_set<std::string> valid_jsons;

  while (valid_jsons.size() < 1000) {
    Value val(domain, bitgen);
    nlohmann::json parsed_json = nlohmann::json::parse(
        val.user_value.begin(), val.user_value.end(), nullptr, false);
    EXPECT_FALSE(parsed_json.is_discarded()) << val;
    valid_jsons.insert(parsed_json.dump());
  }
}

TEST(InJsonGrammar, InitGeneratesShortJson) {
  absl::BitGen bitgen;
  auto domain = InJsonGrammar();

  size_t total_ast_node_num = 0;
  const size_t total_generation_num = 10000;
  for (size_t i = 0; i < total_generation_num; ++i) {
    Value val(domain, bitgen);
    total_ast_node_num += val.corpus_value.NodeCount();
  }

  // The AST generation tries to generate small ASTs. For example, it will only
  // generate one or zero element for a rule `list: element*`. For the current
  // json grammar, the average size (the number of ast nodes) of an ast is 7.
  // TODO(changochen): Say more about the AST size/depth/shape.
  const size_t average_ast_size = total_ast_node_num / total_generation_num;
  EXPECT_NEAR(average_ast_size, 10, 1);
}

TEST(InJsonGrammar, MutateGeneratesValidValues) {
  absl::BitGen bitgen;
  auto domain = InJsonGrammar();
  absl::flat_hash_set<std::string> valid_jsons;
  Value ast(domain, bitgen);

  while (valid_jsons.size() < 5000) {
    ast.Mutate(domain, bitgen, false);
    nlohmann::json parsed_json = nlohmann::json::parse(
        ast.user_value.begin(), ast.user_value.end(), nullptr, false);
    EXPECT_FALSE(parsed_json.is_discarded());
    valid_jsons.insert(parsed_json.dump());
  }
}

TEST(InJsonGrammar, MutateGeneratesDifferentValuesWithHighProb) {
  absl::BitGen bitgen;
  auto domain = InJsonGrammar();
  absl::flat_hash_set<std::string> valid_jsons;
  int num_real_mutation = 0, num_total_mutation = 0;
  Value ast(domain, bitgen);
  nlohmann::json previous_val = nlohmann::json::parse(
      ast.user_value.begin(), ast.user_value.end(), nullptr, false);

  while (valid_jsons.size() < 5000) {
    ++num_total_mutation;
    ast.Mutate(domain, bitgen, false);
    nlohmann::json parsed_json = nlohmann::json::parse(
        ast.user_value.begin(), ast.user_value.end(), nullptr, false);
    valid_jsons.insert(parsed_json.dump());

    // There might be chances that the mutated AST is the same if because
    // mutating regexp doesn't guarantee changes. Otherwise, it guarantees real
    // mutation.
    if (parsed_json != previous_val) ++num_real_mutation;
    previous_val = parsed_json;
  }
  const double probability_of_real_mutation =
      static_cast<double>(num_real_mutation) / num_total_mutation;
  EXPECT_NEAR(probability_of_real_mutation, 0.99, 0.015);
}

TEST(InJsonGrammar, ShrinkModeReducesInputSize) {
  absl::BitGen bitgen;
  auto domain = InJsonGrammar();
  int num_of_successful_shrink = 0;
  int expected_num_of_successful_shrink = 0;
  for (int i = 0; i < 100; ++i) {
    Value val(domain, bitgen);
    size_t original_size = val.corpus_value.NodeCount();
    // If the generated input is big, we expect it to be shrinkable.
    if (original_size > 20) ++expected_num_of_successful_shrink;

    for (int j = 0; j < 10; ++j) {
      val.Mutate(domain, bitgen, true);
      EXPECT_TRUE(val.corpus_value.NodeCount() <= original_size);
    }
    if (original_size > val.corpus_value.NodeCount()) {
      ++num_of_successful_shrink;
    }
  }
  EXPECT_GE(num_of_successful_shrink, expected_num_of_successful_shrink);
}

TEST(InTestGrammar, InGrammarCorpusSerializesCorpusAndParsesCorpusCorrectly) {
  absl::BitGen bitgen;
  auto domain = InJsonGrammar();
  for (int i = 0; i < 1000; ++i) {
    Value val(domain, bitgen);
    std::cout << domain.GetValue(val.corpus_value) << std::endl;
    auto ir_object = domain.SerializeCorpus(val.corpus_value);
    auto ast = domain.ParseCorpus(ir_object);
    ASSERT_TRUE(ast.has_value());
    EXPECT_EQ(domain.GetValue(*ast), val.user_value);
  }
}

}  // namespace
}  // namespace fuzztest