pw_blob_store: Add write support
Add BlobStore write implementation
Add initial BlobStore tests.
Add missing CMake build files in dependency modules to unbreak CMake
build.
Still missing:
Cleanup validation checking
KVS handling/usage
Some better testing
Change-Id: Ie6b711598a917d7553b5a682d8ef36cdcc137303
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/16040
Commit-Queue: David Rogers <davidrogers@google.com>
Reviewed-by: Ewout van Bekkum <ewout@google.com>
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 18e45c4..7f75c53 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -34,8 +34,11 @@
add_subdirectory(pw_minimal_cpp_stdlib)
add_subdirectory(pw_polyfill)
add_subdirectory(pw_preprocessor)
+add_subdirectory(pw_random)
+add_subdirectory(pw_result)
add_subdirectory(pw_span)
add_subdirectory(pw_status)
+add_subdirectory(pw_stream)
add_subdirectory(pw_string)
add_subdirectory(pw_sys_io)
add_subdirectory(pw_sys_io_stdio)
diff --git a/pw_blob_store/BUILD b/pw_blob_store/BUILD
index e45ee1c..dc8689f 100644
--- a/pw_blob_store/BUILD
+++ b/pw_blob_store/BUILD
@@ -24,6 +24,7 @@
pw_cc_library(
name = "pw_blob_store",
+ srcs = [ "blob_store.cc" ],
hdrs = [
"public/pw_blob_store/blob_store.h",
],
@@ -32,8 +33,23 @@
"//pw_checksum",
"//pw_containers",
"//pw_log",
- "//pw_log:facade",
"//pw_span",
"//pw_status",
],
)
+
+pw_cc_test(
+ name = "blob_store_test",
+ srcs = [
+ "blob_store_test.cc",
+ ],
+ deps = [
+ ":pw_blob_store",
+ "//pw_kvs:crc16",
+ "//pw_kvs:fake_flash",
+ "//pw_kvs:fake_flash_test_key_value_store",
+ "//pw_log",
+ "//pw_random",
+ "//pw_unit_test",
+ ],
+)
diff --git a/pw_blob_store/BUILD.gn b/pw_blob_store/BUILD.gn
index 3022b97..4576ff4 100644
--- a/pw_blob_store/BUILD.gn
+++ b/pw_blob_store/BUILD.gn
@@ -25,11 +25,17 @@
pw_source_set("pw_blob_store") {
public_configs = [ ":default_config" ]
public = [ "public/pw_blob_store/blob_store.h" ]
+ sources = [ "blob_store.cc" ]
public_deps = [
dir_pw_kvs,
dir_pw_status,
dir_pw_stream,
]
+ deps = [
+ dir_pw_assert,
+ dir_pw_checksum,
+ dir_pw_log,
+ ]
}
pw_test_group("tests") {
@@ -37,7 +43,15 @@
}
pw_test("blob_store_test") {
- deps = [ ":pw_blob_store" ]
+ deps = [
+ ":pw_blob_store",
+ "$dir_pw_kvs:crc16",
+ "$dir_pw_kvs:fake_flash",
+ "$dir_pw_kvs:fake_flash_test_key_value_store",
+ dir_pw_log,
+ dir_pw_random,
+ ]
+ sources = [ "blob_store_test.cc" ]
}
pw_doc_group("docs") {
diff --git a/pw_blob_store/CMakeLists.txt b/pw_blob_store/CMakeLists.txt
index 181eae4..a61888e 100644
--- a/pw_blob_store/CMakeLists.txt
+++ b/pw_blob_store/CMakeLists.txt
@@ -15,11 +15,14 @@
pw_auto_add_simple_module(pw_blob_store
PUBLIC_DEPS
pw_containers
+ pw_kvs
pw_span
pw_status
+ pw_stream
PRIVATE_DEPS
pw_assert
pw_checksum
pw_log
+ pw_random
pw_string
)
diff --git a/pw_blob_store/blob_store.cc b/pw_blob_store/blob_store.cc
new file mode 100644
index 0000000..27da48a
--- /dev/null
+++ b/pw_blob_store/blob_store.cc
@@ -0,0 +1,349 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+#include "pw_blob_store/blob_store.h"
+
+#include <algorithm>
+
+#include "pw_log/log.h"
+
+namespace pw::blob_store {
+
+Status BlobStore::Init() {
+ if (initialized_) {
+ return Status::OK;
+ }
+
+ const size_t write_buffer_size_alignment =
+ write_buffer_.size_bytes() % partition_.alignment_bytes();
+ PW_CHECK_UINT_EQ((write_buffer_size_alignment), 0);
+ PW_CHECK_UINT_GE(write_buffer_.size_bytes(), partition_.alignment_bytes());
+ PW_CHECK_UINT_LE(write_buffer_.size_bytes(), partition_.sector_size_bytes());
+
+ bool erased = false;
+ if (partition_.IsErased(&erased).ok() && erased) {
+ flash_erased_ = true;
+ } else {
+ flash_erased_ = false;
+ }
+
+ ResetChecksum();
+
+ // TODO: Add checking for already written valid blob.
+
+ initialized_ = true;
+ return Status::OK;
+}
+
+size_t BlobStore::MaxDataSizeBytes() const { return partition_.size_bytes(); }
+
+Status BlobStore::OpenWrite() {
+ // TODO: should Open auto initialize?
+ if (!initialized_) {
+ return Status::FAILED_PRECONDITION;
+ }
+
+ // Writer can only be opened if there are no other writer or readers already
+ // open.
+ if (writer_open_ || readers_open_ != 0) {
+ return Status::UNAVAILABLE;
+ }
+
+ writer_open_ = true;
+
+ write_address_ = 0;
+ flash_address_ = 0;
+
+ ResetChecksum();
+
+ return Status::OK;
+}
+
+Status BlobStore::OpenRead() {
+ if (!initialized_) {
+ return Status::FAILED_PRECONDITION;
+ }
+
+ // If there is no open writer, then once the reader opens there can not be
+ // one. So check that there is actully valid data.
+ // TODO: Move this validation to Init and CloseWrite
+ if (!writer_open_) {
+ if (!ValidateChecksum().ok()) {
+ PW_LOG_ERROR("Validation check failed, invalidate blob");
+ Invalidate();
+ return Status::FAILED_PRECONDITION;
+ }
+ }
+
+ readers_open_++;
+ return Status::OK;
+}
+
+Status BlobStore::CloseWrite() {
+ PW_DCHECK(writer_open_);
+ Status status = Status::OK;
+
+ // Only need to finish a blob if the blob has any bytes.
+ if (write_address_ > 0) {
+ // Flush bytes in buffer.
+ if (write_address_ != flash_address_) {
+ PW_CHECK_UINT_GE(write_address_, flash_address_);
+ size_t bytes_in_buffer = write_address_ - flash_address_;
+
+ // Zero out the remainder of the buffer.
+ auto zero_span = write_buffer_.subspan(bytes_in_buffer);
+ std::memset(zero_span.data(), 0, zero_span.size_bytes());
+
+ status = CommitToFlash(write_buffer_, bytes_in_buffer);
+ }
+
+ if (status.ok()) {
+ // Buffer should be clear unless there was a write error.
+ PW_DCHECK_UINT_EQ(flash_address_, write_address_);
+ // Should not be completing a blob write when it has completed metadata.
+ PW_DCHECK(!metadata_.complete);
+
+ metadata_ = {
+ .checksum = 0, .data_size_bytes = flash_address_, .complete = true};
+
+ if (checksum_algo_ != nullptr) {
+ ConstByteSpan checksum = checksum_algo_->Finish();
+ std::memcpy(&metadata_.checksum,
+ checksum.data(),
+ std::min(checksum.size(), sizeof(metadata_.checksum)));
+ }
+
+ status = kvs_.Put(MetadataKey(), metadata_);
+ }
+ }
+ writer_open_ = false;
+
+ if (status.ok()) {
+ return ValidateChecksum();
+ }
+ return Status::DATA_LOSS;
+}
+
+Status BlobStore::CloseRead() {
+ PW_CHECK_UINT_GT(readers_open_, 0);
+ readers_open_--;
+ return Status::OK;
+}
+
+Status BlobStore::Write(ConstByteSpan data) {
+ PW_DCHECK(writer_open_);
+
+ if (data.size_bytes() == 0) {
+ return Status::OK;
+ }
+ if (WriteBytesRemaining() == 0) {
+ return Status::OUT_OF_RANGE;
+ }
+ if (WriteBytesRemaining() < data.size_bytes()) {
+ return Status::RESOURCE_EXHAUSTED;
+ }
+
+ if (write_address_ == 0 && !erased()) {
+ Erase();
+ }
+
+ // Write in (up to) 3 steps:
+ // 1) Finish filling write buffer and if full write it to flash.
+ // 2) Write as many whole block-sized chunks as the data has remaining
+ // after 1.
+ // 3) Put any remaining bytes less than flash write size in the write buffer.
+
+ // Step 1) If there is any data in the write buffer, finish filling write
+ // buffer and if full write it to flash.
+ const size_t write_chunk_bytes = write_buffer_.size_bytes();
+ if (!WriteBufferEmpty()) {
+ PW_CHECK_UINT_GE(write_address_, flash_address_);
+ size_t bytes_in_buffer = write_address_ - flash_address_;
+ PW_CHECK_UINT_GE(write_chunk_bytes, bytes_in_buffer);
+
+ size_t buffer_remaining = write_chunk_bytes - bytes_in_buffer;
+
+ // Add bytes up to filling the write buffer.
+ size_t add_bytes = std::min(buffer_remaining, data.size_bytes());
+ std::memcpy(write_buffer_.data() + bytes_in_buffer, data.data(), add_bytes);
+ write_address_ += add_bytes;
+ bytes_in_buffer += add_bytes;
+ data = data.subspan(add_bytes);
+
+ if (bytes_in_buffer != write_chunk_bytes) {
+ // If there was not enough bytes to finish filling the write buffer, there
+ // should not be any bytes left.
+ PW_DCHECK(data.size_bytes() == 0);
+ return Status::OK;
+ }
+
+ // The write buffer is full, flush to flash.
+ Status status = CommitToFlash(write_buffer_);
+ if (!status.ok()) {
+ return status;
+ }
+
+ PW_DCHECK(WriteBufferEmpty());
+ }
+
+ // At this point, if data.size_bytes() > 0, the write buffer should be empty.
+ // This invariant is checked as part of of steps 2 & 3.
+
+ // Step 2) Write as many block-sized chunks as the data has remaining after
+ // step 1.
+ // TODO: Decouple write buffer size from optimal bulk write size.
+ while (data.size_bytes() >= write_chunk_bytes) {
+ PW_DCHECK(WriteBufferEmpty());
+
+ write_address_ += write_chunk_bytes;
+ Status status = CommitToFlash(data.first(write_chunk_bytes));
+ if (!status.ok()) {
+ return status;
+ }
+
+ data = data.subspan(write_chunk_bytes);
+ }
+
+ // step 3) Put any remaining bytes to the buffer. Put the bytes starting at
+ // the begining of the buffer, since it must be empty if there are
+ // still bytes due to step 1 either cleaned out the buffer or didn't
+ // have any more data to write.
+ if (data.size_bytes() > 0) {
+ PW_DCHECK(WriteBufferEmpty());
+ std::memcpy(write_buffer_.data(), data.data(), data.size_bytes());
+ write_address_ += data.size_bytes();
+ }
+
+ return Status::OK;
+}
+
+StatusWithSize BlobStore::Read(size_t offset, ByteSpan dest) {
+ (void)offset;
+ (void)dest;
+ return StatusWithSize::UNIMPLEMENTED;
+}
+
+Result<ByteSpan> BlobStore::GetMemoryMappedBlob() const {
+ if (write_address_ == 0 || !valid_data_) {
+ return Status::FAILED_PRECONDITION;
+ }
+
+ std::byte* mcu_address = partition_.PartitionAddressToMcuAddress(0);
+ if (mcu_address == nullptr) {
+ return Status::UNIMPLEMENTED;
+ }
+ return std::span(mcu_address, ReadableDataBytes());
+}
+
+size_t BlobStore::ReadableDataBytes() const { return flash_address_; }
+
+Status BlobStore::Erase() {
+ PW_DCHECK(writer_open_);
+
+ // Can't erase if any readers are open, since this would erase flash right out
+ // from under them.
+ if (readers_open_ != 0) {
+ return Status::UNAVAILABLE;
+ }
+
+ // If already erased our work here is done.
+ if (flash_erased_) {
+ PW_DCHECK_UINT_EQ(write_address_, 0);
+ PW_DCHECK_UINT_EQ(flash_address_, 0);
+ PW_DCHECK(!valid_data_);
+ return Status::OK;
+ }
+
+ Invalidate();
+
+ Status status = partition_.Erase();
+
+ if (status.ok()) {
+ flash_erased_ = true;
+ }
+ return status;
+}
+
+Status BlobStore::Invalidate() {
+ if (readers_open_ != 0) {
+ return Status::UNAVAILABLE;
+ }
+
+ Status status = kvs_.Delete(MetadataKey());
+ valid_data_ = false;
+ ResetChecksum();
+ write_address_ = 0;
+ flash_address_ = 0;
+
+ return status;
+}
+
+Status BlobStore::ValidateChecksum() {
+ if (!metadata_.complete) {
+ return Status::UNAVAILABLE;
+ }
+
+ if (checksum_algo_ == nullptr) {
+ if (metadata_.checksum != 0) {
+ return Status::DATA_LOSS;
+ }
+
+ return Status::OK;
+ }
+
+ PW_LOG_DEBUG("Validate checksum of 0x%08x in flash for blob of %u bytes",
+ unsigned(metadata_.checksum),
+ unsigned(metadata_.data_size_bytes));
+ auto result = CalculateChecksumFromFlash(metadata_.data_size_bytes);
+ if (!result.ok()) {
+ return result.status();
+ }
+
+ return checksum_algo_->Verify(as_bytes(std::span(&metadata_.checksum, 1)));
+}
+
+Result<BlobStore::ChecksumValue> BlobStore::CalculateChecksumFromFlash(
+ size_t bytes_to_check) {
+ if (checksum_algo_ == nullptr) {
+ return Status::OK;
+ }
+
+ checksum_algo_->Reset();
+
+ kvs::FlashPartition::Address address = 0;
+ const kvs::FlashPartition::Address end = bytes_to_check;
+
+ constexpr size_t kReadBufferSizeBytes = 32;
+ std::array<std::byte, kReadBufferSizeBytes> buffer;
+ while (address < end) {
+ const size_t read_size = std::min(size_t(end - address), buffer.size());
+ StatusWithSize status =
+ partition_.Read(address, std::span(buffer).first(read_size));
+ if (!status.ok()) {
+ return status.status();
+ }
+
+ checksum_algo_->Update(buffer.data(), read_size);
+ address += read_size;
+ }
+
+ ChecksumValue checksum_value;
+ std::span checksum = checksum_algo_->Finish();
+ std::memcpy(&checksum_value,
+ checksum.data(),
+ std::min(checksum.size(), sizeof(checksum_value)));
+ return checksum_value;
+}
+
+} // namespace pw::blob_store
diff --git a/pw_blob_store/blob_store_test.cc b/pw_blob_store/blob_store_test.cc
new file mode 100644
index 0000000..396a54b
--- /dev/null
+++ b/pw_blob_store/blob_store_test.cc
@@ -0,0 +1,135 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+#include "pw_blob_store/blob_store.h"
+
+#include <array>
+#include <cstddef>
+#include <cstring>
+#include <span>
+
+#include "gtest/gtest.h"
+#include "pw_kvs/crc16_checksum.h"
+#include "pw_kvs/fake_flash_memory.h"
+#include "pw_kvs/flash_memory.h"
+#include "pw_kvs/test_key_value_store.h"
+#include "pw_log/log.h"
+#include "pw_random/xor_shift.h"
+
+namespace pw::blob_store {
+namespace {
+
+class BlobStoreTest : public ::testing::Test {
+ protected:
+ BlobStoreTest() : flash_(kFlashAlignment), partition_(&flash_) {}
+
+ void InitFlashTo(std::span<const std::byte> contents) {
+ partition_.Erase();
+ std::memcpy(flash_.buffer().data(), contents.data(), contents.size());
+ }
+
+ void InitBufferToRandom(uint64_t seed) {
+ partition_.Erase();
+ random::XorShiftStarRng64 rng(seed);
+ rng.Get(buffer_);
+ }
+
+ // Fill the source buffer with random pattern based on given seed, written to
+ // BlobStore in specified chunk size.
+ void ChunkWriteTest(size_t chunk_size, uint64_t seed) {
+ InitBufferToRandom(seed);
+
+ constexpr size_t kBufferSize = 256;
+ kvs::ChecksumCrc16 checksum;
+
+ char name[16] = {};
+ snprintf(name, sizeof(name), "Blob%u", unsigned(chunk_size));
+
+ BlobStoreBuffer<kBufferSize> blob(
+ name, &partition_, &checksum, kvs::TestKvs());
+ EXPECT_EQ(Status::OK, blob.Init());
+
+ BlobStore::BlobWriter writer(blob);
+ EXPECT_EQ(Status::OK, writer.Open());
+
+ ByteSpan source = buffer_;
+ while (source.size_bytes() > 0) {
+ const size_t write_size = std::min(source.size_bytes(), chunk_size);
+
+ PW_LOG_DEBUG("Do write of %u bytes, %u bytes remain",
+ unsigned(write_size),
+ unsigned(source.size_bytes()));
+
+ EXPECT_EQ(Status::OK, writer.Write(source.first(write_size)));
+
+ source = source.subspan(write_size);
+ }
+
+ EXPECT_EQ(Status::OK, writer.Close());
+
+ // Use reader to check for valid data.
+ BlobStore::BlobReader reader(blob, 0);
+ EXPECT_EQ(Status::OK, reader.Open());
+ Result<ByteSpan> result = reader.GetMemoryMappedBlob();
+ ASSERT_TRUE(result.ok());
+ VerifyFlash(result.value());
+ EXPECT_EQ(Status::OK, reader.Close());
+ }
+
+ void VerifyFlash(ByteSpan verify_bytes) {
+ // Should be defined as same size.
+ EXPECT_EQ(buffer_.size(), flash_.buffer().size_bytes());
+
+ // Can't allow it to march off the end of buffer_.
+ ASSERT_LE(verify_bytes.size_bytes(), buffer_.size());
+
+ for (size_t i = 0; i < verify_bytes.size_bytes(); i++) {
+ EXPECT_EQ(buffer_[i], verify_bytes[i]);
+ }
+ }
+
+ static constexpr size_t kFlashAlignment = 16;
+ static constexpr size_t kSectorSize = 2048;
+ static constexpr size_t kSectorCount = 2;
+
+ kvs::FakeFlashMemoryBuffer<kSectorSize, kSectorCount> flash_;
+ kvs::FlashPartition partition_;
+ std::array<std::byte, kSectorCount * kSectorSize> buffer_;
+};
+
+TEST_F(BlobStoreTest, Init_Ok) {
+ BlobStoreBuffer<256> blob("Blob_OK", &partition_, nullptr, kvs::TestKvs());
+ EXPECT_EQ(Status::OK, blob.Init());
+}
+
+TEST_F(BlobStoreTest, ChunkWrite1) { ChunkWriteTest(1, 0x8675309); }
+
+TEST_F(BlobStoreTest, ChunkWrite2) { ChunkWriteTest(2, 0x8675); }
+
+TEST_F(BlobStoreTest, ChunkWrite16) { ChunkWriteTest(16, 0x86); }
+
+TEST_F(BlobStoreTest, ChunkWrite64) { ChunkWriteTest(64, 0x9); }
+
+TEST_F(BlobStoreTest, ChunkWrite256) { ChunkWriteTest(256, 0x12345678); }
+
+TEST_F(BlobStoreTest, ChunkWrite512) { ChunkWriteTest(512, 0x42); }
+
+TEST_F(BlobStoreTest, ChunkWrite4096) { ChunkWriteTest(4096, 0x89); }
+
+TEST_F(BlobStoreTest, ChunkWriteSingleFull) {
+ ChunkWriteTest((kSectorCount * kSectorSize), 0x98765);
+}
+
+} // namespace
+} // namespace pw::blob_store
diff --git a/pw_blob_store/public/pw_blob_store/blob_store.h b/pw_blob_store/public/pw_blob_store/blob_store.h
index bffafa1..87433f6 100644
--- a/pw_blob_store/public/pw_blob_store/blob_store.h
+++ b/pw_blob_store/public/pw_blob_store/blob_store.h
@@ -17,6 +17,7 @@
#include "pw_kvs/checksum.h"
#include "pw_kvs/flash_memory.h"
+#include "pw_kvs/key_value_store.h"
#include "pw_status/status.h"
#include "pw_stream/stream.h"
@@ -50,77 +51,128 @@
// already erased, the Write will do any needed erase.
class BlobWriter final : public stream::Writer {
public:
- constexpr BlobWriter(BlobStore& store) : store_(store) {}
+ constexpr BlobWriter(BlobStore& store) : store_(store), open_(false) {}
BlobWriter(const BlobWriter&) = delete;
BlobWriter& operator=(const BlobWriter&) = delete;
- ~BlobWriter() { Close(); }
+ ~BlobWriter() {
+ if (open_) {
+ Close();
+ }
+ }
- // Open a blob for writing/erasing. Returns:
+ // Open a blob for writing/erasing. Can not open when already open. Only one
+ // writer is allowed to be open at a time. Returns:
//
// OK - success.
- // UNAVAILABLE - Unable to open, already open.
- Status Open() { return Status::UNIMPLEMENTED; }
+ // UNAVAILABLE - Unable to open, another writer or reader instance is
+ // already open.
+ Status Open() {
+ PW_DCHECK(!open_);
+ Status status = store_.OpenWrite();
+ if (status.ok()) {
+ open_ = true;
+ }
+ return status;
+ }
// Finalize a blob write. Flush all remaining buffered data to storage and
- // store blob metadata. Returns:
+ // store blob metadata. Close fails in the closed state, do NOT retry Close
+ // on error. An error may or may not result in an invalid blob stored.
+ // Returns:
//
// OK - success.
- // FAILED_PRECONDITION - blob is not open.
- Status Close() { return Status::UNIMPLEMENTED; }
+ // DATA_LOSS - Error writing data or fail to verify written data.
+ Status Close() {
+ PW_DCHECK(open_);
+ open_ = false;
+ return store_.CloseWrite();
+ }
// Erase the partition and reset state for a new blob. Returns:
//
// OK - success.
- // UNAVAILABLE - Unable to erase, already open.
+ // UNAVAILABLE - Unable to erase while reader is open.
// [error status] - flash erase failed.
- Status Erase() { return Status::UNIMPLEMENTED; }
+ Status Erase() {
+ PW_DCHECK(open_);
+ return Status::UNIMPLEMENTED;
+ }
// Discard blob (in-progress or valid stored). Any written bytes are
// considered invalid. Returns:
//
// OK - success.
// FAILED_PRECONDITION - not open.
- Status Discard() { return Status::UNIMPLEMENTED; }
+ Status Discard() {
+ PW_DCHECK(open_);
+ return store_.Invalidate();
+ }
// Probable (not guaranteed) minimum number of bytes at this time that can
// be written. Returns zero if, in the current state, Write would return
// status other than OK. See stream.h for additional details.
size_t ConservativeWriteLimit() const override {
- return store_.MaxDataSizeBytes() - store_.ReadableDataBytes();
+ PW_DCHECK(open_);
+ return store_.WriteBytesRemaining();
+ }
+
+ size_t CurrentSizeBytes() {
+ PW_DCHECK(open_);
+ return store_.write_address_;
}
private:
- Status DoWrite(ConstByteSpan data) override { return store_.Write(data); }
+ Status DoWrite(ConstByteSpan data) override {
+ PW_DCHECK(open_);
+ return store_.Write(data);
+ }
BlobStore& store_;
+ bool open_;
};
// Implement stream::Reader interface for BlobStore.
class BlobReader final : public stream::Reader {
public:
constexpr BlobReader(BlobStore& store, size_t offset)
- : store_(store), offset_(offset) {}
+ : store_(store), open_(false), offset_(offset) {}
BlobReader(const BlobReader&) = delete;
BlobReader& operator=(const BlobReader&) = delete;
- ~BlobReader() { Close(); }
+ ~BlobReader() {
+ if (open_) {
+ Close();
+ }
+ }
- // Open to do a blob read. Currently only a single reader at a time is
- // supported. Returns:
+ // Open to do a blob read. Can not open when already open. Multiple readers
+ // can be open at the same time. Returns:
//
// OK - success.
// UNAVAILABLE - Unable to open, already open.
- Status Open() { return Status::UNIMPLEMENTED; }
+ Status Open() {
+ PW_DCHECK(!open_);
+ Status status = store_.OpenRead();
+ if (status.ok()) {
+ open_ = true;
+ }
+ return status;
+ }
- // Finish reading a blob. Returns:
+ // Finish reading a blob. Close fails in the closed state, do NOT retry
+ // Close on error. Returns:
//
// OK - success.
- // FAILED_PRECONDITION - blob is not open.
- Status Close() { return Status::UNIMPLEMENTED; }
+ Status Close() {
+ PW_DCHECK(open_);
+ open_ = false;
+ return store_.CloseRead();
+ }
// Probable (not guaranteed) minimum number of bytes at this time that can
// be read. Returns zero if, in the current state, Read would return status
// other than OK. See stream.h for additional details.
size_t ConservativeReadLimit() const override {
+ PW_DCHECK(open_);
return store_.ReadableDataBytes() - offset_;
}
@@ -130,27 +182,39 @@
// FAILED_PRECONDITION - Reader not open.
// UNIMPLEMENTED - Memory mapped access not supported for this blob.
Result<ByteSpan> GetMemoryMappedBlob() {
+ PW_DCHECK(open_);
return store_.GetMemoryMappedBlob();
}
private:
StatusWithSize DoRead(ByteSpan dest) override {
+ PW_DCHECK(open_);
return store_.Read(offset_, dest);
}
BlobStore& store_;
+ bool open_;
size_t offset_;
};
- BlobStore(kvs::FlashPartition* partition,
+ BlobStore(std::string_view name,
+ kvs::FlashPartition* partition,
kvs::ChecksumAlgorithm* checksum_algo,
+ kvs::KeyValueStore& kvs,
ByteSpan write_buffer)
- : partition_(*partition),
+ : name_(name),
+ partition_(*partition),
checksum_algo_(checksum_algo),
+ kvs_(kvs),
write_buffer_(write_buffer),
- state_(BlobStore::State::kInvalidData),
+ initialized_(false),
+ valid_data_(false),
+ flash_erased_(false),
+ writer_open_(false),
+ readers_open_(0),
+ metadata_({}),
write_address_(0),
- flush_address_(0) {}
+ flash_address_(0) {}
BlobStore(const BlobStore&) = delete;
BlobStore& operator=(const BlobStore&) = delete;
@@ -165,31 +229,31 @@
size_t MaxDataSizeBytes() const;
private:
- enum class OpenType {
- // Open for doing read operations.
- kRead,
-
- // Open for Write operations.
- kWrite,
- };
+ typedef uint32_t ChecksumValue;
// Is the blob erased and ready to write.
- bool erased() const { return state_ == State::kErased; }
+ bool erased() const { return flash_erased_; }
- bool IsOpen();
-
- // Open to do a blob read or write. Returns:
+ // Open to do a blob write. Returns:
//
// OK - success.
- // UNAVAILABLE - Unable to open, already open.
- Status Open(BlobStore::OpenType type);
+ // UNAVAILABLE - Unable to open writer, another writer or reader instance is
+ // already open.
+ Status OpenWrite();
+
+ // Open to do a blob read. Returns:
+ //
+ // OK - success.
+ // FAILED_PRECONDITION - Unable to open, no valid blob available.
+ Status OpenRead();
// Finalize a blob write. Flush all remaining buffered data to storage and
// store blob metadata. Returns:
//
// OK - success.
// FAILED_PRECONDITION - blob is not open.
- Status Close();
+ Status CloseWrite();
+ Status CloseRead();
// Write/append data to the in-progress blob write. Data is written
// sequentially, with each append added directly after the previous. Data is
@@ -203,6 +267,27 @@
// no more will be written.
Status Write(ConstByteSpan data);
+ // Commit data to flash and update flash_address_ with data bytes written. The
+ // only time data_bytes should be manually specified is for a CloseWrite with
+ // an unaligned-size chunk remaining in the buffer that has been zero padded
+ // to alignment.
+ Status CommitToFlash(ConstByteSpan source, size_t data_bytes = 0) {
+ if (data_bytes == 0) {
+ data_bytes = source.size_bytes();
+ }
+ flash_erased_ = false;
+ valid_data_ = true;
+ StatusWithSize result = partition_.Write(flash_address_, source);
+ flash_address_ += data_bytes;
+ if (checksum_algo_ != nullptr) {
+ checksum_algo_->Update(source.first(data_bytes));
+ }
+
+ return result.status();
+ }
+
+ bool WriteBufferEmpty() { return flash_address_ == write_address_; }
+
// Read valid data. Attempts to read the lesser of output.size_bytes() or
// available bytes worth of data. Returns:
//
@@ -221,22 +306,96 @@
// OK with span - Valid span respresenting the blob data
// FAILED_PRECONDITION - Blob not in a state to read data
// UNIMPLEMENTED - Memory mapped access not supported for this blob.
- Result<ByteSpan> GetMemoryMappedBlob();
+ Result<ByteSpan> GetMemoryMappedBlob() const;
// Current size of blob/readable data, in bytes. For a completed write this is
// the size of the data blob. For all other cases this is zero bytes.
size_t ReadableDataBytes() const;
- Status EraseInternal();
+ size_t WriteBytesRemaining() const {
+ return MaxDataSizeBytes() - write_address_;
+ }
- Status InvalidateInternal();
+ Status Erase();
+ Status Invalidate();
+
+ void ResetChecksum() {
+ if (checksum_algo_ != nullptr) {
+ checksum_algo_->Reset();
+ }
+ }
+
+ Status ValidateChecksum();
+
+ Result<BlobStore::ChecksumValue> CalculateChecksumFromFlash(
+ size_t bytes_to_check);
+
+ const std::string_view MetadataKey() { return name_; }
+
+ // Changes to the metadata format should also get a different key signature to
+ // avoid new code improperly reading old format metadata.
+ struct BlobMetadata {
+ // The checksum of the blob data stored in flash.
+ ChecksumValue checksum;
+
+ // Number of blob data bytes stored in flash.
+ size_t data_size_bytes;
+
+ // This blob is complete (the write was properly closed).
+ bool complete;
+ };
+
+ std::string_view name_;
kvs::FlashPartition& partition_;
kvs::ChecksumAlgorithm* const checksum_algo_;
+ kvs::KeyValueStore& kvs_;
ByteSpan write_buffer_;
- State state_;
+
+ //
+ // Internal state for Blob store
+ //
+ // TODO: Consolidate blob state to a single struct
+
+ // Initialization has been done.
+ bool initialized_;
+
+ // Bytes stored are valid and good.
+ bool valid_data_;
+
+ // Blob partition is currently erased and ready to write a new blob.
+ bool flash_erased_;
+
+ // BlobWriter instance is currently open
+ bool writer_open_;
+
+ // Count of open BlobReader instances
+ size_t readers_open_;
+
+ // Metadata for the blob.
+ BlobMetadata metadata_;
+
+ // Current index for end of overal blob data. Represents current byte size of
+ // blob data since the FlashPartition starts at address 0.
kvs::FlashPartition::Address write_address_;
- kvs::FlashPartition::Address flush_address_;
+
+ // Current index of end of data written to flash. Number of buffered data
+ // bytes is write_address_ - flash_address_.
+ kvs::FlashPartition::Address flash_address_;
+};
+
+// Creates a BlobStore with the buffer of kBufferSizeBytes.
+template <size_t kBufferSizeBytes>
+class BlobStoreBuffer : public BlobStore {
+ public:
+ explicit BlobStoreBuffer(std::string_view name,
+ kvs::FlashPartition* partition,
+ kvs::ChecksumAlgorithm* checksum_algo,
+ kvs::KeyValueStore& kvs)
+ : BlobStore(name, partition, checksum_algo, kvs, buffer_) {}
+
+ private:
+ std::array<std::byte, kBufferSizeBytes> buffer_;
};
} // namespace pw::blob_store
diff --git a/pw_random/CMakeLists.txt b/pw_random/CMakeLists.txt
new file mode 100644
index 0000000..b297f32
--- /dev/null
+++ b/pw_random/CMakeLists.txt
@@ -0,0 +1,20 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+pw_auto_add_simple_module(pw_random
+ PUBLIC_DEPS
+ pw_bytes
+ pw_span
+ pw_status
+)
diff --git a/pw_result/CMakeLists.txt b/pw_result/CMakeLists.txt
new file mode 100644
index 0000000..9bf8f5f
--- /dev/null
+++ b/pw_result/CMakeLists.txt
@@ -0,0 +1,20 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+pw_auto_add_simple_module(pw_result
+ PUBLIC_DEPS
+ pw_status
+ PRIVATE_DEPS
+ pw_assert
+)
diff --git a/pw_stream/CMakeLists.txt b/pw_stream/CMakeLists.txt
new file mode 100644
index 0000000..1d17437
--- /dev/null
+++ b/pw_stream/CMakeLists.txt
@@ -0,0 +1,26 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+pw_auto_add_simple_module(pw_stream
+ PUBLIC_DEPS
+ pw_bytes
+ pw_containers
+ pw_log
+ pw_result
+ pw_span
+ pw_status
+ PRIVATE_DEPS
+ pw_assert
+ pw_string
+)
