blob: c19a21f3846d89ce86919d35ac6fdabb85e6d26e [file] [log] [blame]
// Copyright 2021 The Pigweed Authors
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.
#pragma once
#include <algorithm>
#include <array>
#include <bit>
#include <cstddef>
#include <cstring>
#include <span>
#include <string_view>
#include "pw_assert/assert.h"
#include "pw_bytes/endian.h"
#include "pw_bytes/span.h"
#include "pw_protobuf/config.h"
#include "pw_protobuf/wire_format.h"
#include "pw_status/status.h"
#include "pw_status/try.h"
#include "pw_stream/memory_stream.h"
#include "pw_stream/stream.h"
#include "pw_varint/varint.h"
namespace pw::protobuf {
// Provides a size estimate to help with sizing buffers passed to
// StreamingEncoder and MemoryEncoder objects.
//
// Args:
// max_message_size: For MemoryEncoder objects, this is the max expected size
// of the final proto. For StreamingEncoder objects, this should be the max
// size of any nested proto submessage that will be built with this encoder
// (recursively accumulating the size from the root submessage). If your
// proto will encode many large submessages, this value should just be the
// size of the largest one.
// max_nested_depth: The max number of nested submessage encoders that are
// expected to be open simultaneously to encode this proto message.
static constexpr size_t MaxScratchBufferSize(size_t max_message_size,
size_t max_nested_depth) {
return max_message_size + max_nested_depth * config::kMaxVarintSize;
}
// Forward declaration. StreamingEncoder and MemoryEncoder are very tightly
// coupled.
class MemoryEncoder;
// A protobuf encoder that encodes serialized proto data to a
// pw::stream::Writer.
class StreamingEncoder {
public:
// The StreamingEncoder will serialize proto data to the pw::stream::Writer
// provided through the constructor. The scratch buffer provided is for
// internal use ONLY and should not be considered valid proto data.
//
// If a StreamingEncoder object will be writing nested proto messages, it must
// provide a scratch buffer large enough to hold the largest submessage some
// additional overhead incurred by the encoder's implementation. It's a good
// idea to be generous when sizing this buffer. MaxScratchBufferSize() can be
// helpful in providing an estimated size for this buffer. The scratch buffer
// must exist for the lifetime of the StreamingEncoder object.
//
// StreamingEncoder objects that do not write nested proto messages can
// provide a zero-length scratch buffer.
constexpr StreamingEncoder(stream::Writer& writer, ByteSpan scratch_buffer)
: writer_(writer),
status_(OkStatus()),
parent_(nullptr),
nested_field_number_(0),
memory_writer_(scratch_buffer) {}
~StreamingEncoder() { Finalize(); }
// Disallow copy/assign to avoid confusion about who owns the buffer.
StreamingEncoder& operator=(const StreamingEncoder& other) = delete;
StreamingEncoder(const StreamingEncoder& other) = delete;
// It's not safe to move an encoder as it could cause another encoder's
// parent_ pointer to become invalid.
StreamingEncoder& operator=(StreamingEncoder&& other) = delete;
// Forwards the conservative write limit of the underlying pw::stream::Writer.
//
// Precondition: Encoder has no active child encoder.
size_t ConservativeWriteLimit() const {
PW_ASSERT(!nested_encoder_open());
return writer_.ConservativeWriteLimit();
}
// Creates a nested encoder with the provided field number. Once this is
// called, the parent encoder is locked and not available for use until the
// nested encoder is finalized (either explicitly or through destruction).
//
// Precondition: Encoder has no active child encoder.
StreamingEncoder GetNestedEncoder(uint32_t field_number);
// Closes the proto encoder. If this encoder is a nested one, the parent is
// unlocked and proto encoding may resume on the parent. This is automatically
// called on object destruction.
//
// Precondition: Encoder has no active child encoder.
//
// Returns:
// OutOfRange: Insufficient space reserved for the submessage. This
// usually means config::kMaxVarintSize was set too small.
Status Finalize();
Status status() const {
if (nested_encoder_open()) {
return Status::Unavailable();
}
return status_;
}
// Writes a proto uint32 key-value pair.
//
// Precondition: Encoder has no active child encoder.
Status WriteUint32(uint32_t field_number, uint32_t value) {
return WriteUint64(field_number, value);
}
// Writes a repeated uint32 using packed encoding.
//
// Precondition: Encoder has no active child encoder.
Status WritePackedUint32(uint32_t field_number,
std::span<const uint32_t> values) {
return WritePackedVarints(field_number, values, VarintEncodeType::kNormal);
}
// Writes a proto uint64 key-value pair.
//
// Precondition: Encoder has no active child encoder.
Status WriteUint64(uint32_t field_number, uint64_t value) {
return WriteVarintField(field_number, value);
}
// Writes a repeated uint64 using packed encoding.
//
// Precondition: Encoder has no active child encoder.
Status WritePackedUint64(uint64_t field_number,
std::span<const uint64_t> values) {
return WritePackedVarints(field_number, values, VarintEncodeType::kNormal);
}
// Writes a proto int32 key-value pair.
//
// Precondition: Encoder has no active child encoder.
Status WriteInt32(uint32_t field_number, int32_t value) {
return WriteUint64(field_number, value);
}
// Writes a repeated int32 using packed encoding.
//
// Precondition: Encoder has no active child encoder.
Status WritePackedInt32(uint32_t field_number,
std::span<const int32_t> values) {
return WritePackedVarints(
field_number,
std::span(reinterpret_cast<const uint32_t*>(values.data()),
values.size()),
VarintEncodeType::kNormal);
}
// Writes a proto int64 key-value pair.
//
// Precondition: Encoder has no active child encoder.
Status WriteInt64(uint32_t field_number, int64_t value) {
return WriteUint64(field_number, value);
}
// Writes a repeated int64 using packed encoding.
//
// Precondition: Encoder has no active child encoder.
Status WritePackedInt64(uint32_t field_number,
std::span<const int64_t> values) {
return WritePackedVarints(
field_number,
std::span(reinterpret_cast<const uint64_t*>(values.data()),
values.size()),
VarintEncodeType::kNormal);
}
// Writes a proto sint32 key-value pair.
//
// Precondition: Encoder has no active child encoder.
Status WriteSint32(uint32_t field_number, int32_t value) {
return WriteUint64(field_number, varint::ZigZagEncode(value));
}
// Writes a repeated sint32 using packed encoding.
//
// Precondition: Encoder has no active child encoder.
Status WritePackedSint32(uint32_t field_number,
std::span<const int32_t> values) {
return WritePackedVarints(
field_number,
std::span(reinterpret_cast<const uint32_t*>(values.data()),
values.size()),
VarintEncodeType::kZigZag);
}
// Writes a proto sint64 key-value pair.
//
// Precondition: Encoder has no active child encoder.
Status WriteSint64(uint32_t field_number, int64_t value) {
return WriteUint64(field_number, varint::ZigZagEncode(value));
}
// Writes a repeated sint64 using packed encoding.
//
// Precondition: Encoder has no active child encoder.
Status WritePackedSint64(uint32_t field_number,
std::span<const int64_t> values) {
return WritePackedVarints(
field_number,
std::span(reinterpret_cast<const uint64_t*>(values.data()),
values.size()),
VarintEncodeType::kZigZag);
}
// Writes a proto bool key-value pair.
//
// Precondition: Encoder has no active child encoder.
Status WriteBool(uint32_t field_number, bool value) {
return WriteUint32(field_number, static_cast<uint32_t>(value));
}
// Writes a proto fixed32 key-value pair.
//
// Precondition: Encoder has no active child encoder.
Status WriteFixed32(uint32_t field_number, uint32_t value) {
std::array<std::byte, sizeof(value)> data =
bytes::CopyInOrder(std::endian::little, value);
return WriteFixed(field_number, data);
}
// Writes a repeated fixed32 field using packed encoding.
//
// Precondition: Encoder has no active child encoder.
Status WritePackedFixed32(uint32_t field_number,
std::span<const uint32_t> values) {
return WritePackedFixed(
field_number, std::as_bytes(values), sizeof(uint32_t));
}
// Writes a proto fixed64 key-value pair.
//
// Precondition: Encoder has no active child encoder.
Status WriteFixed64(uint32_t field_number, uint64_t value) {
std::array<std::byte, sizeof(value)> data =
bytes::CopyInOrder(std::endian::little, value);
return WriteFixed(field_number, data);
}
// Writes a repeated fixed64 field using packed encoding.
//
// Precondition: Encoder has no active child encoder.
Status WritePackedFixed64(uint32_t field_number,
std::span<const uint64_t> values) {
return WritePackedFixed(
field_number, std::as_bytes(values), sizeof(uint64_t));
}
// Writes a proto sfixed32 key-value pair.
//
// Precondition: Encoder has no active child encoder.
Status WriteSfixed32(uint32_t field_number, int32_t value) {
return WriteFixed32(field_number, static_cast<uint32_t>(value));
}
// Writes a repeated sfixed32 field using packed encoding.
//
// Precondition: Encoder has no active child encoder.
Status WritePackedSfixed32(uint32_t field_number,
std::span<const int32_t> values) {
return WritePackedFixed(
field_number, std::as_bytes(values), sizeof(int32_t));
}
// Writes a proto sfixed64 key-value pair.
//
// Precondition: Encoder has no active child encoder.
Status WriteSfixed64(uint32_t field_number, int64_t value) {
return WriteFixed64(field_number, static_cast<uint64_t>(value));
}
// Writes a repeated sfixed64 field using packed encoding.
//
// Precondition: Encoder has no active child encoder.
Status WritePackedSfixed64(uint32_t field_number,
std::span<const int64_t> values) {
return WritePackedFixed(
field_number, std::as_bytes(values), sizeof(int64_t));
}
// Writes a proto float key-value pair.
//
// Precondition: Encoder has no active child encoder.
Status WriteFloat(uint32_t field_number, float value) {
static_assert(sizeof(float) == sizeof(uint32_t),
"Float and uint32_t are not the same size");
uint32_t integral_value;
std::memcpy(&integral_value, &value, sizeof(value));
std::array<std::byte, sizeof(value)> data =
bytes::CopyInOrder(std::endian::little, integral_value);
return WriteFixed(field_number, data);
}
// Writes a repeated float field using packed encoding.
//
// Precondition: Encoder has no active child encoder.
Status WritePackedFloat(uint32_t field_number,
std::span<const float> values) {
return WritePackedFixed(field_number, std::as_bytes(values), sizeof(float));
}
// Writes a proto double key-value pair.
//
// Precondition: Encoder has no active child encoder.
Status WriteDouble(uint32_t field_number, double value) {
static_assert(sizeof(double) == sizeof(uint64_t),
"Double and uint64_t are not the same size");
uint64_t integral_value;
std::memcpy(&integral_value, &value, sizeof(value));
std::array<std::byte, sizeof(value)> data =
bytes::CopyInOrder(std::endian::little, integral_value);
return WriteFixed(field_number, data);
}
// Writes a repeated double field using packed encoding.
//
// Precondition: Encoder has no active child encoder.
Status WritePackedDouble(uint32_t field_number,
std::span<const double> values) {
return WritePackedFixed(
field_number, std::as_bytes(values), sizeof(double));
}
// Writes a proto `bytes` field as a key-value pair. This can also be used to
// write a pre-encoded nested submessage directly without using a nested
// encoder.
//
// Precondition: Encoder has no active child encoder.
Status WriteBytes(uint32_t field_number, ConstByteSpan value) {
return WriteLengthDelimitedField(field_number, value);
}
// Writes a proto string key-value pair.
//
// Precondition: Encoder has no active child encoder.
Status WriteString(uint32_t field_number, std::string_view value) {
return WriteBytes(field_number, std::as_bytes(std::span(value)));
}
// Writes a proto string key-value pair.
//
// Precondition: Encoder has no active child encoder.
Status WriteString(uint32_t field_number, const char* value, size_t len) {
return WriteBytes(field_number, std::as_bytes(std::span(value, len)));
}
// Writes a proto string key-value pair.
// TODO(384): This function is not safe and will be removed as part of the
// transition away from the old protobuf encoder.
//
// Precondition: Encoder has no active child encoder.
Status WriteString(uint32_t field_number, const char* value) {
return WriteBytes(field_number,
std::as_bytes(std::span(std::string_view(value))));
}
protected:
// We need this for codegen.
constexpr StreamingEncoder(StreamingEncoder&& other)
: writer_(&other.writer_ == &other.memory_writer_ ? memory_writer_
: other.writer_),
status_(other.status_),
parent_(other.parent_),
nested_field_number_(other.nested_field_number_),
memory_writer_(std::move(other.memory_writer_)) {
PW_ASSERT(nested_field_number_ == 0);
// Make the nested encoder look like it has an open child to block writes
// for the remainder of the object's life.
other.nested_field_number_ = kFirstReservedNumber;
other.parent_ = nullptr;
}
private:
friend class MemoryEncoder;
enum class VarintEncodeType {
kNormal,
kZigZag,
};
constexpr StreamingEncoder(StreamingEncoder& parent, ByteSpan scratch_buffer)
: writer_(memory_writer_),
status_(scratch_buffer.empty() ? Status::ResourceExhausted()
: OkStatus()),
parent_(&parent),
nested_field_number_(0),
memory_writer_(scratch_buffer) {}
bool nested_encoder_open() const { return nested_field_number_ != 0; }
// Finalization logic for nested encoders that call Finalize(). While
// Finalize() is called on the child encoder, FinalizeNestedMessage() is
// called on the parent encoder.
Status FinalizeNestedMessage(StreamingEncoder& nested);
// Implementation for encoding all varint field types.
Status WriteVarintField(uint32_t field_number, uint64_t value);
// Implementation for encoding all length-delimited field types.
Status WriteLengthDelimitedField(uint32_t field_number, ConstByteSpan data);
// Implementation for encoding all fixed-length integer types.
Status WriteFixed(uint32_t field_number, ConstByteSpan data);
// Encodes a base-128 varint to the buffer.
Status WriteVarint(uint64_t value);
Status WriteZigzagVarint(int64_t value) {
return WriteVarint(varint::ZigZagEncode(value));
}
// Writes a list of varints to the buffer in length-delimited packed encoding.
template <typename T, typename = std::enable_if_t<std::is_integral<T>::value>>
Status WritePackedVarints(uint32_t field_number,
std::span<T> values,
VarintEncodeType encode_type) {
static_assert(std::is_same<T, const uint32_t>::value ||
std::is_same<T, const int32_t>::value ||
std::is_same<T, const uint64_t>::value ||
std::is_same<T, const int64_t>::value,
"Packed varints must be of type uint32_t, int32_t, uint64_t, "
"or int64_t");
size_t payload_size = 0;
for (T val : values) {
if (encode_type == VarintEncodeType::kZigZag) {
int64_t integer =
static_cast<int64_t>(static_cast<std::make_signed_t<T>>(val));
payload_size += varint::EncodedSize(varint::ZigZagEncode(integer));
} else {
uint64_t integer = static_cast<uint64_t>(val);
payload_size += varint::EncodedSize(integer);
}
}
if (!UpdateStatusForWrite(field_number, WireType::kDelimited, payload_size)
.ok()) {
return status_;
}
WriteVarint(MakeKey(field_number, WireType::kDelimited));
WriteVarint(payload_size);
for (T value : values) {
if (encode_type == VarintEncodeType::kZigZag) {
WriteZigzagVarint(static_cast<std::make_signed_t<T>>(value));
} else {
WriteVarint(value);
}
}
return status_;
}
// Writes a list of fixed-size types to the buffer in length-delimited
// packed encoding. Only float, double, uint32_t, int32_t, uint64_t, and
// int64_t are permitted
Status WritePackedFixed(uint32_t field_number,
std::span<const std::byte> values,
size_t elem_size);
// Checks if a write is invalid or will cause the encoder to enter an error
// state, and preemptively sets this encoder's status to that error to block
// the write. Only the first error encountered is tracked.
//
// Precondition: Encoder has no active child encoder.
//
// Returns:
// InvalidArgument: The field number provided was invalid.
// ResourceExhausted: The requested write would have exceeded the
// stream::Writer's conservative write limit.
// Other: If any Write() operations on the stream::Writer caused an error,
// that error will be repeated here.
Status UpdateStatusForWrite(uint32_t field_number,
WireType type,
size_t data_size);
// All proto encode operations are directly written to this writer.
stream::Writer& writer_;
// The current encoder status. This status is only updated to reflect the
// first error encountered. Any further write operations are blocked when the
// encoder enters an error state.
Status status_;
// If this is a nested encoder, this points to the encoder that created it.
// For user-created MemoryEncoders, parent_ points to this object as an
// optimization for the MemoryEncoder and nested encoders to use the same
// underlying buffer.
StreamingEncoder* parent_;
// If an encoder has a child encoder open, this is the field number of that
// submessage. Otherwise, this is 0 to indicate no child encoder is open.
uint32_t nested_field_number_;
// This memory writer is used for staging proto submessages to the
// scratch_buffer.
stream::MemoryWriter memory_writer_;
};
// A protobuf encoder that writes directly to a provided buffer. This will
// eventually replace the original pw::protobuf::Encoder. If you want to encode
// a proto into an in-memory buffer, use this.
//
// Example:
//
// // Writes a proto response to the provided buffer, returning the encode
// // status and number of bytes written.
// StatusWithSize WriteProtoResponse(ByteSpan response) {
// // All proto writes are directly written to the `response` buffer.
// MemoryEncoder encoder(response);
// encoder.WriteUint32(kMagicNumberField, 0x1a1a2b2b);
// encoder.WriteString(kFavoriteFood, "cookies");
// return StatusWithSize(encoder.status(), encoder.size());
// }
//
// Note: Avoid using a MemoryEncoder reference as an argument for a function.
// The StreamingEncoder is more generic.
class MemoryEncoder : public StreamingEncoder {
public:
constexpr MemoryEncoder(ByteSpan dest) : StreamingEncoder(*this, dest) {}
~MemoryEncoder() { Finalize(); }
// Disallow copy/assign to avoid confusion about who owns the buffer.
MemoryEncoder(const MemoryEncoder& other) = delete;
MemoryEncoder& operator=(const MemoryEncoder& other) = delete;
// It's not safe to move an encoder as it could cause another encoder's
// parent_ pointer to become invalid.
MemoryEncoder& operator=(MemoryEncoder&& other) = delete;
const std::byte* data() const { return memory_writer_.data(); }
size_t size() const { return memory_writer_.bytes_written(); }
protected:
// This is needed by codegen.
MemoryEncoder(MemoryEncoder&& other) = default;
};
} // namespace pw::protobuf