conformance/conformance_test.cc

// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc.  All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include "conformance_test.h"

#include <stdarg.h>

#include <fstream>
#include <set>
#include <string>

#include <google/protobuf/stubs/stringprintf.h>
#include <google/protobuf/message.h>
#include <google/protobuf/text_format.h>
#include <google/protobuf/util/field_comparator.h>
#include <google/protobuf/util/json_util.h>
#include <google/protobuf/util/message_differencer.h>
#include "conformance.pb.h"

using conformance::ConformanceRequest;
using conformance::ConformanceResponse;
using conformance::WireFormat;
using google::protobuf::TextFormat;
using google::protobuf::util::DefaultFieldComparator;
using google::protobuf::util::MessageDifferencer;
using std::string;

namespace {

static string ToOctString(const string& binary_string) {
  string oct_string;
  for (size_t i = 0; i < binary_string.size(); i++) {
    uint8_t c = binary_string.at(i);
    uint8_t high = c / 64;
    uint8_t mid = (c % 64) / 8;
    uint8_t low = c % 8;
    oct_string.push_back('\\');
    oct_string.push_back('0' + high);
    oct_string.push_back('0' + mid);
    oct_string.push_back('0' + low);
  }
  return oct_string;
}

}  // namespace

namespace google {
namespace protobuf {

ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting(
    ConformanceLevel level,
    conformance::WireFormat input_format,
    conformance::WireFormat output_format,
    conformance::TestCategory test_category,
    const Message& prototype_message,
    const string& test_name, const string& input)
    : level_(level),
      input_format_(input_format),
      output_format_(output_format),
      prototype_message_(prototype_message),
      prototype_message_for_compare_(prototype_message.New()),
      test_name_(test_name) {
  switch (input_format) {
    case conformance::PROTOBUF: {
      request_.set_protobuf_payload(input);
      break;
    }

    case conformance::JSON: {
      request_.set_json_payload(input);
      break;
    }

    case conformance::JSPB: {
      request_.set_jspb_payload(input);
      break;
    }

    case conformance::TEXT_FORMAT: {
      request_.set_text_payload(input);
      break;
    }

    default:
      GOOGLE_LOG(FATAL) << "Unspecified input format";
  }

  request_.set_test_category(test_category);

  request_.set_message_type(prototype_message.GetDescriptor()->full_name());
  request_.set_requested_output_format(output_format);
}

std::unique_ptr<Message>
ConformanceTestSuite::ConformanceRequestSetting::NewTestMessage() const {
  return std::unique_ptr<Message>(prototype_message_for_compare_->New());
}

string ConformanceTestSuite::ConformanceRequestSetting::
    GetTestName() const {
  string rname =
      prototype_message_.GetDescriptor()->file()->syntax() ==
        FileDescriptor::SYNTAX_PROTO3 ? "Proto3" : "Proto2";

  return StrCat(ConformanceLevelToString(level_), ".", rname, ".",
                      InputFormatString(input_format_), ".", test_name_, ".",
                      OutputFormatString(output_format_));
}

string ConformanceTestSuite::ConformanceRequestSetting::
    ConformanceLevelToString(
        ConformanceLevel level) const {
  switch (level) {
    case REQUIRED: return "Required";
    case RECOMMENDED: return "Recommended";
  }
  GOOGLE_LOG(FATAL) << "Unknown value: " << level;
  return "";
}

string ConformanceTestSuite::ConformanceRequestSetting::
    InputFormatString(conformance::WireFormat format) const {
  switch (format) {
    case conformance::PROTOBUF:
      return "ProtobufInput";
    case conformance::JSON:
      return "JsonInput";
    case conformance::TEXT_FORMAT:
      return "TextFormatInput";
    default:
      GOOGLE_LOG(FATAL) << "Unspecified output format";
  }
  return "";
}

string ConformanceTestSuite::ConformanceRequestSetting::
    OutputFormatString(conformance::WireFormat format) const {
  switch (format) {
    case conformance::PROTOBUF:
      return "ProtobufOutput";
    case conformance::JSON:
      return "JsonOutput";
    case conformance::TEXT_FORMAT:
      return "TextFormatOutput";
    default:
      GOOGLE_LOG(FATAL) << "Unspecified output format";
  }
  return "";
}

void ConformanceTestSuite::ReportSuccess(const string& test_name) {
  if (expected_to_fail_.erase(test_name) != 0) {
    StringAppendF(&output_,
                  "ERROR: test %s is in the failure list, but test succeeded.  "
                  "Remove it from the failure list.\n",
                  test_name.c_str());
    unexpected_succeeding_tests_.insert(test_name);
  }
  successes_++;
}

void ConformanceTestSuite::ReportFailure(const string& test_name,
                                         ConformanceLevel level,
                                         const ConformanceRequest& request,
                                         const ConformanceResponse& response,
                                         const char* fmt, ...) {
  if (expected_to_fail_.erase(test_name) == 1) {
    expected_failures_++;
    if (!verbose_)
      return;
  } else if (level == RECOMMENDED && !enforce_recommended_) {
    StringAppendF(&output_, "WARNING, test=%s: ", test_name.c_str());
  } else {
    StringAppendF(&output_, "ERROR, test=%s: ", test_name.c_str());
    unexpected_failing_tests_.insert(test_name);
  }
  va_list args;
  va_start(args, fmt);
  StringAppendV(&output_, fmt, args);
  va_end(args);
  StringAppendF(&output_, " request=%s, response=%s\n",
                request.ShortDebugString().c_str(),
                response.ShortDebugString().c_str());
}

void ConformanceTestSuite::ReportSkip(const string& test_name,
                                      const ConformanceRequest& request,
                                      const ConformanceResponse& response) {
  if (verbose_) {
    StringAppendF(&output_, "SKIPPED, test=%s request=%s, response=%s\n",
                  test_name.c_str(), request.ShortDebugString().c_str(),
                  response.ShortDebugString().c_str());
  }
  skipped_.insert(test_name);
}

void ConformanceTestSuite::RunValidInputTest(
    const ConformanceRequestSetting& setting,
    const string& equivalent_text_format) {
  std::unique_ptr<Message> reference_message(setting.NewTestMessage());
  GOOGLE_CHECK(TextFormat::ParseFromString(equivalent_text_format,
                                    reference_message.get()))
      << "Failed to parse data for test case: " << setting.GetTestName()
      << ", data: " << equivalent_text_format;
  const string equivalent_wire_format = reference_message->SerializeAsString();
  RunValidBinaryInputTest(setting, equivalent_wire_format);
}

void ConformanceTestSuite::RunValidBinaryInputTest(
    const ConformanceRequestSetting& setting,
    const string& equivalent_wire_format, bool require_same_wire_format) {
  const ConformanceRequest& request = setting.GetRequest();
  ConformanceResponse response;
  RunTest(setting.GetTestName(), request, &response);
  VerifyResponse(setting, equivalent_wire_format, response, true,
                 require_same_wire_format);
}

void ConformanceTestSuite::VerifyResponse(
    const ConformanceRequestSetting& setting,
    const string& equivalent_wire_format, const ConformanceResponse& response,
    bool need_report_success, bool require_same_wire_format) {
  std::unique_ptr<Message> test_message(setting.NewTestMessage());
  const ConformanceRequest& request = setting.GetRequest();
  const string& test_name = setting.GetTestName();
  ConformanceLevel level = setting.GetLevel();
  std::unique_ptr<Message> reference_message = setting.NewTestMessage();

  GOOGLE_CHECK(reference_message->ParseFromString(equivalent_wire_format))
      << "Failed to parse wire data for test case: " << test_name;

  switch (response.result_case()) {
    case ConformanceResponse::RESULT_NOT_SET:
      ReportFailure(test_name, level, request, response,
                    "Response didn't have any field in the Response.");
      return;

    case ConformanceResponse::kParseError:
    case ConformanceResponse::kRuntimeError:
    case ConformanceResponse::kSerializeError:
      ReportFailure(test_name, level, request, response,
                    "Failed to parse input or produce output.");
      return;

    case ConformanceResponse::kSkipped:
      ReportSkip(test_name, request, response);
      return;

    default:
      if (!ParseResponse(response, setting, test_message.get())) return;
  }

  MessageDifferencer differencer;
  DefaultFieldComparator field_comparator;
  field_comparator.set_treat_nan_as_equal(true);
  differencer.set_field_comparator(&field_comparator);
  string differences;
  differencer.ReportDifferencesToString(&differences);

  bool check = false;

  if (require_same_wire_format) {
    GOOGLE_DCHECK_EQ(response.result_case(), ConformanceResponse::kProtobufPayload);
    const string& protobuf_payload = response.protobuf_payload();
    check = equivalent_wire_format == protobuf_payload;
    differences = StrCat("Expect: ", ToOctString(equivalent_wire_format),
                               ", but got: ", ToOctString(protobuf_payload));
  } else {
    check = differencer.Compare(*reference_message, *test_message);
  }

  if (check) {
    if (need_report_success) {
      ReportSuccess(test_name);
    }
  } else {
    ReportFailure(test_name, level, request, response,
                  "Output was not equivalent to reference message: %s.",
                  differences.c_str());
  }
}

void ConformanceTestSuite::RunTest(const string& test_name,
                                   const ConformanceRequest& request,
                                   ConformanceResponse* response) {
  if (test_names_.insert(test_name).second == false) {
    GOOGLE_LOG(FATAL) << "Duplicated test name: " << test_name;
  }

  string serialized_request;
  string serialized_response;
  request.SerializeToString(&serialized_request);

  runner_->RunTest(test_name, serialized_request, &serialized_response);

  if (!response->ParseFromString(serialized_response)) {
    response->Clear();
    response->set_runtime_error("response proto could not be parsed.");
  }

  if (verbose_) {
    StringAppendF(&output_,
                  "conformance test: name=%s, request=%s, response=%s\n",
                  test_name.c_str(),
                  request.ShortDebugString().c_str(),
                  response->ShortDebugString().c_str());
  }
}

bool ConformanceTestSuite::CheckSetEmpty(
    const std::set<string>& set_to_check,
    const std::string& write_to_file,
    const std::string& msg) {
  if (set_to_check.empty()) {
    return true;
  } else {
    StringAppendF(&output_, "\n");
    StringAppendF(&output_, "%s\n\n", msg.c_str());
    for (std::set<string>::const_iterator iter = set_to_check.begin();
         iter != set_to_check.end(); ++iter) {
      StringAppendF(&output_, "  %s\n", iter->c_str());
    }
    StringAppendF(&output_, "\n");

    if (!write_to_file.empty()) {
      std::string full_filename;
      const std::string* filename = &write_to_file;
      if (!output_dir_.empty()) {
        full_filename = output_dir_;
        if (*output_dir_.rbegin() != '/') {
          full_filename.push_back('/');
        }
        full_filename += write_to_file;
        filename = &full_filename;
      }
      std::ofstream os(*filename);
      if (os) {
        for (std::set<string>::const_iterator iter = set_to_check.begin();
             iter != set_to_check.end(); ++iter) {
          os << *iter << "\n";
        }
      } else {
        StringAppendF(&output_, "Failed to open file: %s\n",
                      filename->c_str());
      }
    }

    return false;
  }
}

string ConformanceTestSuite::WireFormatToString(
    WireFormat wire_format) {
  switch (wire_format) {
    case conformance::PROTOBUF:
      return "PROTOBUF";
    case conformance::JSON:
      return "JSON";
    case conformance::JSPB:
      return "JSPB";
    case conformance::TEXT_FORMAT:
      return "TEXT_FORMAT";
    case conformance::UNSPECIFIED:
      return "UNSPECIFIED";
    default:
      GOOGLE_LOG(FATAL) << "unknown wire type: " << wire_format;
  }
  return "";
}

void ConformanceTestSuite::AddExpectedFailedTest(const std::string& test_name) {
  expected_to_fail_.insert(test_name);
}

bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
                                    std::string* output, const string& filename,
                                    conformance::FailureSet* failure_list) {
  runner_ = runner;
  successes_ = 0;
  expected_failures_ = 0;
  skipped_.clear();
  test_names_.clear();
  unexpected_failing_tests_.clear();
  unexpected_succeeding_tests_.clear();

  output_ = "\nCONFORMANCE TEST BEGIN ====================================\n\n";

  failure_list_filename_ = filename;
  expected_to_fail_.clear();
  for (const string& failure : failure_list->failure()) {
    AddExpectedFailedTest(failure);
  }
  RunSuiteImpl();

  bool ok = true;
  if (!CheckSetEmpty(expected_to_fail_, "nonexistent_tests.txt",
                     "These tests were listed in the failure list, but they "
                     "don't exist.  Remove them from the failure list by "
                     "running:\n"
                     "  ./update_failure_list.py " + failure_list_filename_ +
                     " --remove nonexistent_tests.txt")) {
    ok = false;
  }
  if (!CheckSetEmpty(unexpected_failing_tests_, "failing_tests.txt",
                     "These tests failed.  If they can't be fixed right now, "
                     "you can add them to the failure list so the overall "
                     "suite can succeed.  Add them to the failure list by "
                     "running:\n"
                     "  ./update_failure_list.py " + failure_list_filename_ +
                     " --add failing_tests.txt")) {
    ok = false;
  }
  if (!CheckSetEmpty(unexpected_succeeding_tests_, "succeeding_tests.txt",
                     "These tests succeeded, even though they were listed in "
                     "the failure list.  Remove them from the failure list "
                     "by running:\n"
                     "  ./update_failure_list.py " + failure_list_filename_ +
                     " --remove succeeding_tests.txt")) {
    ok = false;
  }

  if (verbose_) {
    CheckSetEmpty(skipped_, "",
                  "These tests were skipped (probably because support for some "
                  "features is not implemented)");
  }

  StringAppendF(&output_,
                "CONFORMANCE SUITE %s: %d successes, %zu skipped, "
                "%d expected failures, %zu unexpected failures.\n",
                ok ? "PASSED" : "FAILED", successes_, skipped_.size(),
                expected_failures_, unexpected_failing_tests_.size());
  StringAppendF(&output_, "\n");

  output->assign(output_);

  return ok;
}

}  // namespace protobuf
}  // namespace google