pw_stream: Create module
pw_stream provides a generic writer interface that allows modules to
write data without assuming anything about what the data is written to.
A pw::stream::Writer could be backed by RAM, flash memory, or just pipe
the data out over a communication channel.
Change-Id: Ie8f8a66819a2c80a47f7cc75e2c9d1bbe25a668d
diff --git a/BUILD.gn b/BUILD.gn
index 4b78b31..f9c1e7a 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -60,6 +60,7 @@
"$dir_pw_result",
"$dir_pw_span",
"$dir_pw_status",
+ "$dir_pw_stream",
"$dir_pw_string",
"$dir_pw_unit_test",
"$dir_pw_varint",
@@ -97,6 +98,7 @@
"$dir_pw_rpc:tests",
"$dir_pw_span:tests",
"$dir_pw_status:tests",
+ "$dir_pw_stream:tests",
"$dir_pw_string:tests",
"$dir_pw_tokenizer:tests",
"$dir_pw_unit_test:tests",
diff --git a/docs/BUILD.gn b/docs/BUILD.gn
index 703dac2..d89057b 100644
--- a/docs/BUILD.gn
+++ b/docs/BUILD.gn
@@ -87,6 +87,7 @@
"$dir_pw_rpc:docs",
"$dir_pw_span:docs",
"$dir_pw_status:docs",
+ "$dir_pw_stream:docs",
"$dir_pw_string:docs",
"$dir_pw_sys_io:docs",
"$dir_pw_sys_io_baremetal_stm32f429:docs",
diff --git a/modules.gni b/modules.gni
index 244b0e3..9ca6bd8 100644
--- a/modules.gni
+++ b/modules.gni
@@ -48,6 +48,7 @@
dir_pw_rpc = "$dir_pigweed/pw_rpc"
dir_pw_span = "$dir_pigweed/pw_span"
dir_pw_status = "$dir_pigweed/pw_status"
+dir_pw_stream = "$dir_pigweed/pw_stream"
dir_pw_string = "$dir_pigweed/pw_string"
dir_pw_sys_io = "$dir_pigweed/pw_sys_io"
dir_pw_sys_io_baremetal_lm3s6965evb =
diff --git a/pw_stream/BUILD b/pw_stream/BUILD
new file mode 100644
index 0000000..1f3244a
--- /dev/null
+++ b/pw_stream/BUILD
@@ -0,0 +1,59 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+load(
+ "//pw_build:pigweed.bzl",
+ "pw_cc_library",
+ "pw_cc_test",
+)
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"]) # Apache License 2.0
+
+pw_cc_library(
+ name = "pw_stream",
+ hdrs = [
+ "public/pw_stream/buffered_stream.h",
+ "public/pw_stream/memory_stream.h",
+ "public/pw_stream/stream.h",
+ ],
+ srcs = [
+ "memory_stream.cc",
+ "buffered_stream.cc",
+ ],
+ includes = ["public"],
+)
+
+pw_cc_test(
+ name = "memory_stream_test",
+ srcs = [
+ "memory_stream_test.cc",
+ ],
+ deps = [
+ ":pw_stream",
+ "//pw_unit_test",
+ ],
+)
+
+pw_cc_test(
+ name = "buffered_stream_test",
+ srcs = [
+ "buffered_stream_test.cc",
+ ],
+ deps = [
+ ":pw_stream",
+ "//pw_unit_test",
+ ],
+)
\ No newline at end of file
diff --git a/pw_stream/BUILD.gn b/pw_stream/BUILD.gn
new file mode 100644
index 0000000..311eb1b
--- /dev/null
+++ b/pw_stream/BUILD.gn
@@ -0,0 +1,48 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import("$dir_pw_docgen/docs.gni")
+import("$dir_pw_unit_test/test.gni")
+
+config("default_config") {
+ include_dirs = [ "public" ]
+}
+
+source_set("pw_stream") {
+ public_configs = [ ":default_config" ]
+ public = [
+ "public/pw_stream/memory_stream.h",
+ "public/pw_stream/stream.h",
+ ]
+ sources = [ "memory_stream.cc" ]
+ public_deps = [
+ dir_pw_assert,
+ dir_pw_preprocessor,
+ dir_pw_span,
+ dir_pw_status,
+ ]
+}
+
+pw_doc_group("docs") {
+ sources = [ "docs.rst" ]
+}
+
+pw_test_group("tests") {
+ tests = [ ":memory_stream_test" ]
+}
+
+pw_test("memory_stream_test") {
+ sources = [ "memory_stream_test.cc" ]
+ deps = [ ":pw_stream" ]
+}
diff --git a/pw_stream/docs.rst b/pw_stream/docs.rst
new file mode 100644
index 0000000..753a0e3
--- /dev/null
+++ b/pw_stream/docs.rst
@@ -0,0 +1,203 @@
+.. _chapter-stream:
+
+.. default-domain:: cpp
+
+.. highlight:: sh
+
+---------
+pw_stream
+---------
+
+``pw_stream`` provides a foundational interface for streaming data from one part
+of a system to another. In the simplest use cases, this is basically a memcpy
+behind a reusable interface that can be passed around the system. On the other
+hand, the flexibility of this interface means a ``pw_stream`` could terminate is
+something more complex, like a UART stream or flash memory.
+
+Overview
+========
+At the most basic level, ``pw_stream``'s interfaces provide very simple handles
+to enabling streaming data from one location in a system to an endpoint.
+
+Example:
+
+.. code-block:: cpp
+
+ void DumpSensorData(pw::stream::Writer& writer) {
+ static char temp[64];
+ ImuSample imu_sample;
+ imu.GetSample(&info);
+ size_t bytes_written = imu_sample.AsCsv(temp, sizeof(temp));
+ writer.Write(temp, bytes_written);
+ }
+
+In this example, ``DumpSensorData()`` only cares that it has access to a
+``Writer`` that it can use to stream data to using ``Writer::Write()``. The
+``Writer`` itself can be backed by anything that can act as a data "sink."
+
+
+pw::stream::Writer
+------------------
+This is the foundational stream ``Writer`` abstract class. Any class that wishes
+to implement the ``Writer`` interface **must** provide a ``DoWrite()``
+implementation. Note that ``Write()`` itself is **not** virtual, and should not
+be overridden.
+
+Buffering
+^^^^^^^^^
+If any buffering occurs in a ``Writer`` and data must be flushed before it is
+fully committed to the sink, a ``Writer`` may optionally override ``Flush()``.
+Writes to a buffer may optimistically return ``Status::OK``, so depend on the
+return value of ``Flush()`` to ensure any pending data is committed to a sink.
+
+Generally speaking, the scope that instantiates the concrete ``Writer`` class
+should be in charge of calling ``Flush()``, and functions that only have access
+to the Writer interface should avoid calling this function.
+
+pw::stream::MemoryWriter
+------------------------
+The ``MemoryWriter`` class implements the ``Writer`` interface by backing the
+data destination with an **externally-provided** memory buffer.
+``MemoryWriterBuffer`` extends ``MemoryWriter`` to internally provide a memory
+buffer.
+
+Why use pw_stream?
+==================
+
+Standard API
+------------
+``pw_stream`` provides a standard way for classes to express that they have the
+ability to write data. Writing to one sink versus another sink is a matter of
+just passing a reference to the appropriate ``Writer``.
+
+As an example, imagine dumping sensor data. If written against a random HAL
+or one-off class, there's porting work required to write to a different sink
+(imagine writing over UART vs dumping to flash memory). Building a "dumping"
+implementation against the ``Writer`` interface prevents a dependency from
+forming on an artisainal API that would require porting work.
+
+Similarly, after building a ``Writer`` implementation for a Sink that data
+could be dumped to, that same ``Writer`` can be reused for other contexts that
+already write data to the ``pw::stream::Writer`` interface.
+
+Before:
+
+.. code-block:: cpp
+
+ // Not reusable, depends on `Uart`.
+ void DumpSensorData(Uart& uart) {
+ static char temp[64];
+ ImuSample imu_sample;
+ imu.GetSample(&info);
+ size_t bytes_written = imu_sample.AsCsv(temp, sizeof(temp));
+ uart.Transmit(temp, bytes_written, /*timeout_ms=*/ 200);
+ }
+
+After:
+
+.. code-block:: cpp
+
+ // Reusable; no more Uart dependency!
+ void DumpSensorData(Writer& writer) {
+ static char temp[64];
+ ImuSample imu_sample;
+ imu.GetSample(&info);
+ size_t bytes_written = imu_sample.AsCsv(temp, sizeof(temp));
+ writer.Write(temp, bytes_written);
+ }
+
+Reduce intermediate buffers
+---------------------------
+Often functions that write larger blobs of data request a buffer is passed as
+the destination that data should be written to. This *requires* a buffer is
+allocated, even if the data only exists in that buffer for a very short period
+of time before it's written somewhere else.
+
+In situations where data read from somewhere will immediately be written
+somewhere else, a ``Writer`` interface can cut out the middleman buffer.
+
+Before:
+
+.. code-block:: cpp
+
+ // Requires an intermediate buffer to write the data as CSV.
+ void DumpSensorData(Uart* uart) {
+ char temp[64];
+ ImuSample imu_sample;
+ imu.GetSample(&info);
+ size_t bytes_written = imu_sample.AsCsv(temp, sizeof(temp));
+ uart.Transmit(temp, bytes_written, /*timeout_ms=*/ 200);
+ }
+
+After:
+
+.. code-block:: cpp
+
+ // Both DumpSensorData() and RawSample::AsCsv() use a Writer, eliminating the
+ // need for an intermediate buffer.
+ void DumpSensorData(Writer* writer) {
+ RawSample imu_sample;
+ imu.GetSample(&info);
+ imu_sample.AsCsv(writer);
+ }
+
+Prevent buffer overflow
+-----------------------
+When copying data from one buffer to another, there must be checks to ensure the
+copy does not overflow the destination buffer. As this sort of logic is
+duplicated throughout a codebase, there's more opportunities for bound-checking
+bugs to sneak in. ``Writers`` manage this logic internally rather than pushing
+the bounds checking to the code that is moving or writing the data.
+
+Similarly, since only the ``Writer`` has access to any underlying buffers, it's
+harder for functions that share a ``Writer`` to accidentally clobber data
+written by others using the same buffer.
+
+Before:
+
+.. code-block:: cpp
+
+ Status BuildPacket(Id dest, span<const std::byte> payload,
+ span<std::byte> dest) {
+ Header header;
+ if (dest.size_bytes() + payload.size_bytes() < sizeof(Header)) {
+ return Status::RESOURCE_EXHAUSTED;
+ }
+ header.dest = dest;
+ header.src = DeviceId();
+ header.payload_size = payload.size_bytes();
+
+ memcpy(dest.data(), &header, sizeof(header));
+ // Forgetting this line would clobber buffer contents. Also, using
+ // a temporary span instead could leave `dest` to be misused elsewhere in
+ // the function.
+ dest = dest.subspan(sizeof(header));
+ memcpy(dest.data(), payload.data(), payload.size_bytes());
+ }
+
+After:
+
+.. code-block:: cpp
+
+ Status BuildPacket(Id dest, span<const std::byte> payload, Writer& writer) {
+ Header header;
+ header.dest = dest;
+ header.src = DeviceId();
+ header.payload_size = payload.size_bytes();
+
+ writer.Write(header);
+ return writer.Write(payload);
+ }
+
+Why NOT pw_stream?
+==================
+pw_stream provides a virtual interface. This inherently has more overhead than
+a regular function call. In extremely performance-sensitive contexts, a virtual
+interface might not provide enough utility to justify the performance cost.
+
+Dependencies
+============
+ * ``pw_assert`` module
+ * ``pw_preprocessor`` module
+ * ``pw_status`` module
+ * ``pw_span`` module
diff --git a/pw_stream/memory_stream.cc b/pw_stream/memory_stream.cc
new file mode 100644
index 0000000..efe2c97
--- /dev/null
+++ b/pw_stream/memory_stream.cc
@@ -0,0 +1,36 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+#include "pw_stream/memory_stream.h"
+
+#include <cstddef>
+#include <cstring>
+
+#include "pw_status/status_with_size.h"
+
+namespace pw::stream {
+
+Status MemoryWriter::DoWrite(span<const std::byte> data) {
+ size_t bytes_to_write =
+ std::min(data.size_bytes(), dest_.size_bytes() - bytes_written_);
+ std::memcpy(dest_.data() + bytes_written_, data.data(), bytes_to_write);
+ bytes_written_ += bytes_to_write;
+
+ if (bytes_to_write != data.size_bytes()) {
+ return Status::RESOURCE_EXHAUSTED;
+ }
+ return Status::OK;
+}
+
+} // namespace pw::stream
\ No newline at end of file
diff --git a/pw_stream/memory_stream_test.cc b/pw_stream/memory_stream_test.cc
new file mode 100644
index 0000000..ada85ab
--- /dev/null
+++ b/pw_stream/memory_stream_test.cc
@@ -0,0 +1,101 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+#include "pw_stream/memory_stream.h"
+
+#include "gtest/gtest.h"
+#include "pw_preprocessor/compiler.h"
+
+namespace pw::stream {
+namespace {
+
+// Size of the in-memory buffer to use for this test.
+constexpr size_t kSinkBufferSize = 1013;
+
+struct TestStruct {
+ uint8_t day;
+ uint8_t month;
+ uint16_t year;
+};
+
+constexpr TestStruct kExpectedStruct = {.day = 18, .month = 5, .year = 2020};
+
+std::array<std::byte, kSinkBufferSize> memory_buffer;
+
+TEST(MemoryWriter, BytesWritten) {
+ MemoryWriter memory_writer(memory_buffer);
+ EXPECT_EQ(memory_writer.bytes_written(), 0u);
+ Status status =
+ memory_writer.Write(&kExpectedStruct, sizeof(kExpectedStruct));
+ EXPECT_TRUE(status.ok());
+ EXPECT_EQ(memory_writer.bytes_written(), sizeof(kExpectedStruct));
+}
+
+TEST(MemoryWriter, ValidateContents) {
+ MemoryWriter memory_writer(memory_buffer);
+ EXPECT_TRUE(
+ memory_writer.Write(&kExpectedStruct, sizeof(kExpectedStruct)).ok());
+
+ span<const std::byte> written_data = memory_writer.WrittenData();
+ EXPECT_EQ(written_data.size_bytes(), sizeof(kExpectedStruct));
+ TestStruct temp;
+ std::memcpy(&temp, written_data.data(), written_data.size_bytes());
+ EXPECT_EQ(memcmp(&temp, &kExpectedStruct, sizeof(kExpectedStruct)), 0);
+}
+
+TEST(MemoryWriter, MultipleWrites) {
+ constexpr size_t kTempBufferSize = 72;
+ std::byte buffer[kTempBufferSize] = {};
+ size_t counter = 0;
+
+ MemoryWriter memory_writer(memory_buffer);
+
+ do {
+ for (size_t i = 0; i < sizeof(buffer); ++i) {
+ buffer[i] = std::byte(counter++);
+ }
+ } while (memory_writer.Write(span(buffer)) != Status::RESOURCE_EXHAUSTED);
+
+ // Ensure that we counted up to at least the sink buffer size. This can be
+ // more since we write to the sink via in intermediate buffer.
+ EXPECT_GE(counter, kSinkBufferSize);
+
+ EXPECT_EQ(memory_writer.bytes_written(), kSinkBufferSize);
+
+ counter = 0;
+ for (const std::byte& value : memory_writer.WrittenData()) {
+ EXPECT_EQ(value, std::byte(counter++));
+ }
+}
+
+TEST(MemoryWriter, EmptyData) {
+ std::byte buffer[5] = {};
+
+ MemoryWriter memory_writer(memory_buffer);
+ EXPECT_TRUE(memory_writer.Write(buffer, 0).ok());
+ EXPECT_EQ(memory_writer.bytes_written(), 0u);
+}
+
+#if CHECK_TEST_CRASHES
+
+// TODO(amontanez): Ensure that this test triggers an assert.
+TEST(MemoryWriter, NullPointer) {
+ MemoryWriter memory_writer(memory_buffer);
+ memory_writer.Write(nullptr, 21);
+}
+
+#endif // CHECK_TEST_CRASHES
+
+} // namespace
+} // namespace pw::stream
\ No newline at end of file
diff --git a/pw_stream/public/pw_stream/memory_stream.h b/pw_stream/public/pw_stream/memory_stream.h
new file mode 100644
index 0000000..ad9d30d
--- /dev/null
+++ b/pw_stream/public/pw_stream/memory_stream.h
@@ -0,0 +1,56 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+#pragma once
+
+#include <array>
+#include <cstddef>
+
+#include "pw_span/span.h"
+#include "pw_stream/stream.h"
+
+namespace pw::stream {
+
+class MemoryWriter : public Writer {
+ public:
+ MemoryWriter(span<std::byte> dest) : dest_(dest) {}
+
+ size_t bytes_written() const { return bytes_written_; }
+
+ span<const std::byte> WrittenData() const {
+ return dest_.first(bytes_written_);
+ }
+
+ const std::byte* data() const { return dest_.data(); }
+
+ private:
+ // Implementation for writing data to this stream.
+ //
+ // If the in-memory buffer is exhausted in the middle of a write, this will
+ // perform a partial write and Status::RESOURCE_EXHAUSTED will be returned.
+ Status DoWrite(span<const std::byte> data) override;
+
+ span<std::byte> dest_;
+ size_t bytes_written_ = 0;
+};
+
+template <size_t size_bytes>
+class MemoryWriterBuffer : public MemoryWriter {
+ public:
+ MemoryWriterBuffer() : MemoryWriter(buffer_) {}
+
+ private:
+ std::array<std::byte, size_bytes> buffer_;
+};
+
+} // namespace pw::stream
diff --git a/pw_stream/public/pw_stream/stream.h b/pw_stream/public/pw_stream/stream.h
new file mode 100644
index 0000000..6d51191
--- /dev/null
+++ b/pw_stream/public/pw_stream/stream.h
@@ -0,0 +1,58 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+#pragma once
+
+#include <array>
+#include <cstddef>
+
+#include "pw_assert/assert.h"
+#include "pw_span/span.h"
+#include "pw_status/status.h"
+
+namespace pw::stream {
+
+// General-purpose writer interface.
+class Writer {
+ public:
+ // There are no requirements around if or how a `Writer` should call Flush()
+ // at the time of object destruction. In general, do not assume a `Writer`
+ // will automatically call Flush when destroyed.
+ virtual ~Writer() = default;
+
+ // Write data to this stream Writer. If the writer runs out of resources, it
+ // will return Status::RESOURCE_EXHAUSTED. The derived class choses whether
+ // to do partial writes in this case, or abort the entire write operation.
+ //
+ // Derived classes should NOT try to override these public write methods.
+ // Instead, provide an implementation by overriding DoWrite().
+ Status Write(span<const std::byte> data) {
+ PW_DCHECK(data.empty() || data.data() != nullptr);
+ return DoWrite(data);
+ }
+ Status Write(const void* data, size_t size_bytes) {
+ return Write(span(static_cast<const std::byte*>(data), size_bytes));
+ }
+
+ // Flush any buffered data, finalizing all writes.
+ //
+ // Generally speaking, the scope that instantiates the concrete `Writer`
+ // class should be in charge of calling `Flush()`, and functions that only
+ // have access to the Writer interface should avoid calling this function.
+ virtual Status Flush() { return Status::OK; }
+
+ private:
+ virtual Status DoWrite(span<const std::byte> data) = 0;
+};
+
+} // namespace pw::stream
