pw_kvs: Initial commit of new KVS design
Builds, but lots more work needed.
Fun fact: this code was written collaboratively in a Google Doc.
Change-Id: I8a89c5d0fdc71ec28cf432350e65d17e24a6f25c
diff --git a/pw_kvs/BUILD b/pw_kvs/BUILD
index 53d5579..5433285 100644
--- a/pw_kvs/BUILD
+++ b/pw_kvs/BUILD
@@ -25,16 +25,17 @@
pw_cc_library(
name = "pw_kvs",
srcs = [
- "flash.cc",
+ "checksum.cc",
+ "flash_memory.cc",
"key_value_store.cc",
+ "pw_kvs_private/format.h",
+ "pw_kvs_private/macros.h",
],
hdrs = [
- "public/pw_kvs/assert.h",
- "public/pw_kvs/flash.h",
+ "public/pw_kvs/checksum.h",
"public/pw_kvs/flash_memory.h",
"public/pw_kvs/in_memory_fake_flash.h",
"public/pw_kvs/key_value_store.h",
- "public/pw_kvs/partition_table_entry.h",
],
includes = ["public"],
deps = [
diff --git a/pw_kvs/BUILD.gn b/pw_kvs/BUILD.gn
index 83b1fc4..35e02d2 100644
--- a/pw_kvs/BUILD.gn
+++ b/pw_kvs/BUILD.gn
@@ -22,16 +22,17 @@
source_set("pw_kvs") {
public_configs = [ ":default_config" ]
public = [
- "public/pw_kvs/assert.h",
- "public/pw_kvs/flash.h",
+ "public/pw_kvs/checksum.h",
"public/pw_kvs/flash_memory.h",
"public/pw_kvs/in_memory_fake_flash.h",
"public/pw_kvs/key_value_store.h",
- "public/pw_kvs/partition_table_entry.h",
]
sources = [
- "flash.cc",
+ "checksum.cc",
+ "flash_memory.cc",
"key_value_store.cc",
+ "pw_kvs_private/format.h",
+ "pw_kvs_private/macros.h",
]
sources += public
public_deps = [
@@ -41,6 +42,7 @@
dir_pw_checksum,
dir_pw_log,
]
+ friend = [ ":key_value_store_test" ]
}
pw_test_group("tests") {
diff --git a/pw_kvs/checksum.cc b/pw_kvs/checksum.cc
new file mode 100644
index 0000000..9f14c90
--- /dev/null
+++ b/pw_kvs/checksum.cc
@@ -0,0 +1,35 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+#include "pw_kvs/checksum.h"
+
+#include <cstring>
+
+namespace pw::kvs {
+
+using std::byte;
+
+Status ChecksumAlgorithm::Verify(span<const byte> checksum) const {
+ if (checksum.size() != size_bytes()) {
+ return Status::INVALID_ARGUMENT;
+ }
+
+ if (std::memcmp(state_.data(), checksum.data(), state_.size()) != 0) {
+ return Status::DATA_LOSS;
+ }
+
+ return Status::OK;
+}
+
+} // namespace pw::kvs
diff --git a/pw_kvs/flash.cc b/pw_kvs/flash.cc
deleted file mode 100644
index fbbc604..0000000
--- a/pw_kvs/flash.cc
+++ /dev/null
@@ -1,85 +0,0 @@
-// Copyright 2020 The Pigweed Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not
-// use this file except in compliance with the License. You may obtain a copy of
-// the License at
-//
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-// License for the specific language governing permissions and limitations under
-// the License.
-
-#include "pw_kvs/flash.h"
-
-#include <cstring>
-
-namespace pw::kvs {
-namespace cfg {
-
-// Configure the maximum supported alignment for the flash utility functions.
-constexpr size_t kFlashUtilMaxAlignmentBytes = 16;
-
-} // namespace cfg
-
-Status PaddedWrite(FlashPartition* partition,
- FlashPartition::Address address,
- const void* raw_buffer,
- uint16_t size) {
- const uint8_t* const buffer = static_cast<const uint8_t*>(raw_buffer);
-
- if (address % partition->GetAlignmentBytes() ||
- partition->GetAlignmentBytes() > cfg::kFlashUtilMaxAlignmentBytes) {
- return Status::INVALID_ARGUMENT;
- }
- uint8_t alignment_buffer[cfg::kFlashUtilMaxAlignmentBytes] = {0};
- uint16_t aligned_bytes = size - size % partition->GetAlignmentBytes();
- if (Status status = partition->Write(address, buffer, aligned_bytes);
- !status.ok()) {
- return status;
- }
- uint16_t remaining_bytes = size - aligned_bytes;
- if (remaining_bytes > 0) {
- std::memcpy(alignment_buffer, &buffer[aligned_bytes], remaining_bytes);
- if (Status status = partition->Write(address + aligned_bytes,
- alignment_buffer,
- partition->GetAlignmentBytes());
- !status.ok()) {
- return status;
- }
- }
- return Status::OK;
-}
-
-Status UnalignedRead(FlashPartition* partition,
- void* raw_buffer,
- FlashPartition::Address address,
- uint16_t size) {
- uint8_t* const buffer = static_cast<uint8_t*>(raw_buffer);
-
- if (address % partition->GetAlignmentBytes() ||
- partition->GetAlignmentBytes() > cfg::kFlashUtilMaxAlignmentBytes) {
- return Status::INVALID_ARGUMENT;
- }
- uint16_t aligned_bytes = size - size % partition->GetAlignmentBytes();
- if (Status status = partition->Read(buffer, address, aligned_bytes);
- !status.ok()) {
- return status;
- }
- uint16_t remaining_bytes = size - aligned_bytes;
- if (remaining_bytes > 0) {
- uint8_t alignment_buffer[cfg::kFlashUtilMaxAlignmentBytes];
- if (Status status = partition->Read(alignment_buffer,
- address + aligned_bytes,
- partition->GetAlignmentBytes());
- !status.ok()) {
- return status;
- }
- std::memcpy(&buffer[aligned_bytes], alignment_buffer, remaining_bytes);
- }
- return Status::OK;
-}
-
-} // namespace pw::kvs
diff --git a/pw_kvs/flash_memory.cc b/pw_kvs/flash_memory.cc
new file mode 100644
index 0000000..bd7493a
--- /dev/null
+++ b/pw_kvs/flash_memory.cc
@@ -0,0 +1,149 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+#include "pw_kvs/flash_memory.h"
+
+#include <algorithm>
+#include <cinttypes>
+#include <cstring>
+
+#include "pw_kvs_private/macros.h"
+#include "pw_log/log.h"
+
+namespace pw::kvs {
+
+using std::byte;
+
+Status FlashPartition::Erase(Address address, size_t num_sectors) {
+ if (permission_ == PartitionPermission::kReadOnly) {
+ return Status::PERMISSION_DENIED;
+ }
+
+ TRY(CheckBounds(address, num_sectors * sector_size_bytes()));
+ return flash_.Erase(PartitionToFlashAddress(address), num_sectors);
+}
+
+StatusWithSize FlashPartition::Read(Address address, span<byte> output) {
+ TRY(CheckBounds(address, output.size()));
+ return flash_.Read(PartitionToFlashAddress(address), output);
+}
+
+StatusWithSize FlashPartition::Write(Address address, span<const byte> data) {
+ if (permission_ == PartitionPermission::kReadOnly) {
+ return Status::PERMISSION_DENIED;
+ }
+ TRY(CheckBounds(address, data.size()));
+ return flash_.Write(PartitionToFlashAddress(address), data);
+}
+
+StatusWithSize FlashPartition::Write(
+ const Address start_address, std::initializer_list<span<const byte>> data) {
+ byte buffer[64]; // TODO: Configure this?
+
+ Address address = start_address;
+ auto bytes_written = [&]() { return address - start_address; };
+
+ const size_t write_size = AlignDown(sizeof(buffer), alignment_bytes());
+ size_t bytes_in_buffer = 0;
+
+ for (span<const byte> chunk : data) {
+ while (!chunk.empty()) {
+ const size_t to_copy =
+ std::min(write_size - bytes_in_buffer, chunk.size());
+
+ std::memcpy(&buffer[bytes_in_buffer], chunk.data(), to_copy);
+ chunk = chunk.subspan(to_copy);
+ bytes_in_buffer += to_copy;
+
+ // If the buffer is full, write it out.
+ if (bytes_in_buffer == write_size) {
+ Status status = Write(address, span(buffer, write_size));
+ if (!status.ok()) {
+ return StatusWithSize(status, bytes_written());
+ }
+
+ address += write_size;
+ bytes_in_buffer = 0;
+ }
+ }
+ }
+
+ // If data remains in the buffer, pad it to the alignment size and flush
+ // the remaining data.
+ if (bytes_in_buffer != 0u) {
+ size_t remaining_write_size = AlignUp(bytes_in_buffer, alignment_bytes());
+ std::memset(
+ &buffer[bytes_in_buffer], 0, remaining_write_size - bytes_in_buffer);
+ if (Status status = Write(address, span(buffer, remaining_write_size));
+ !status.ok()) {
+ return StatusWithSize(status, bytes_written());
+ }
+ address += remaining_write_size;
+ }
+
+ return StatusWithSize(bytes_written());
+}
+
+Status FlashPartition::IsRegionErased(Address source_flash_address,
+ size_t length,
+ bool* is_erased) {
+ // Max alignment is artificial to keep the stack usage low for this
+ // function. Using 16 because it's the alignment of encrypted flash.
+ constexpr size_t kMaxAlignment = 16;
+
+ // Relying on Read() to check address and len arguments.
+ if (is_erased == nullptr) {
+ return Status::INVALID_ARGUMENT;
+ }
+ const size_t alignment = alignment_bytes();
+ if (alignment > kMaxAlignment || kMaxAlignment % alignment ||
+ length % alignment) {
+ return Status::INVALID_ARGUMENT;
+ }
+
+ byte buffer[kMaxAlignment];
+ byte erased_pattern_buffer[kMaxAlignment];
+
+ size_t offset = 0;
+ std::memset(erased_pattern_buffer,
+ int(flash_.erased_memory_content()),
+ sizeof(erased_pattern_buffer));
+ *is_erased = false;
+ while (length > 0u) {
+ // Check earlier that length is aligned, no need to round up
+ size_t read_size = std::min(sizeof(buffer), length);
+ TRY(Read(source_flash_address + offset, read_size, buffer).status());
+ if (std::memcmp(buffer, erased_pattern_buffer, read_size)) {
+ // Detected memory chunk is not entirely erased
+ return Status::OK;
+ }
+ offset += read_size;
+ length -= read_size;
+ }
+ *is_erased = true;
+ return Status::OK;
+}
+
+Status FlashPartition::CheckBounds(Address address, size_t length) const {
+ if (address + length > size_bytes()) {
+ PW_LOG_ERROR("Attempted out-of-bound flash memory access (address: %" PRIu32
+ " length: %zu)",
+ address,
+ length);
+ return Status::INVALID_ARGUMENT;
+ }
+ return Status::OK;
+}
+
+} // namespace pw::kvs
diff --git a/pw_kvs/key_value_store.cc b/pw_kvs/key_value_store.cc
index 6eefb84..1801dc3 100644
--- a/pw_kvs/key_value_store.cc
+++ b/pw_kvs/key_value_store.cc
@@ -12,863 +12,319 @@
// License for the specific language governing permissions and limitations under
// the License.
-// KVS is a key-value storage format for flash based memory.
-//
-// Currently it stores key-value sets in chunks aligned with the flash memory.
-// Each chunk contains a header (KvsHeader), which is immediately followed by
-// the key, and then the data blob. To support different alignments both the
-// key length and value length are rounded up to be aligned.
-//
-// Example memory layout of sector with two KVS chunks:
-// [ SECTOR_HEADER - Meta | alignment_padding ]
-// [ SECTOR_HEADER - Cleaning State | alignment_padding ]
-// [First Chunk Header | alignment_padding ]
-// [First Chunk Key | alignment_padding ]
-// [First Chunk Value | alignment_padding ]
-// [Second Chunk Header | alignment_padding ]
-// [Second Chunk Key | alignment_padding ]
-// [Second Chunk Value | alignment_padding ]
-//
-// For efficiency if a key's value is rewritten the new value is just appended
-// to the same sector, a clean of the sector is only needed if there is no more
-// room. Cleaning the sector involves moving the most recent value of each key
-// to another sector and erasing the sector. Erasing data is treated the same
-// as rewriting data, but an erased chunk has the erased flag set, and no data.
-//
-// KVS maintains a data structure in RAM for quick indexing of each key's most
-// recent value, but no caching of data is ever performed. If a write/erase
-// function returns successfully, it is guaranteed that the data has been
-// pushed to flash. The KVS should also be resistant to sudden unplanned power
-// outages and be capable of recovering even mid clean (this is accomplished
-// using a flag which marks the sector before the clean is started).
-
#include "pw_kvs/key_value_store.h"
-#include <algorithm>
#include <cstring>
+#include <type_traits>
-#include "pw_checksum/ccitt_crc16.h"
-#include "pw_kvs/flash.h"
-#include "pw_log/log.h"
+#include "pw_kvs_private/format.h"
+#include "pw_kvs_private/macros.h"
namespace pw::kvs {
using std::byte;
+using std::string_view;
-Status KeyValueStore::Enable() {
- // TODO: LOCK MUTEX
- if (enabled_) {
- return Status::OK;
+namespace {
+
+// constexpr uint32_t SixFiveFiveNineNine(std::string_view string) {
+constexpr uint32_t HashKey(std::string_view string) {
+ uint32_t hash = 0;
+ uint32_t coefficient = 65599u;
+
+ for (char ch : string) {
+ hash += coefficient * unsigned(ch);
+ coefficient *= 65599u;
}
- // Reset parameters.
- std::memset(sector_space_remaining_, 0, sizeof(sector_space_remaining_));
- map_size_ = 0;
-
- // For now alignment is set to use partitions alignment.
- alignment_bytes_ = partition_.GetAlignmentBytes();
- DCHECK(alignment_bytes_ <= kMaxAlignmentBytes);
-
- if (partition_.GetSectorCount() > kSectorCountMax) {
- PW_LOG_WARN(
- "Partition is larger then KVS max sector count, "
- "not all space will be used.");
- }
- // Load map and setup sectors if needed (first word isn't kSectorReadyValue).
- next_sector_clean_order_ = 0;
- for (SectorIndex i = 0; i < SectorCount(); i++) {
- KvsSectorHeaderMeta sector_header_meta;
- // Because underlying flash can be encrypted + erased, trying to readback
- // may give random values. It's important to make sure that data is not
- // erased before trying to see if there is a token match.
- bool is_sector_meta_erased;
- if (Status status = partition_.IsChunkErased(
- SectorIndexToAddress(i),
- RoundUpForAlignment(sizeof(sector_header_meta)),
- &is_sector_meta_erased);
- !status.ok()) {
- return status;
- };
- if (Status status =
- UnalignedRead(&partition_,
- reinterpret_cast<uint8_t*>(§or_header_meta),
- SectorIndexToAddress(i),
- sizeof(sector_header_meta));
- !status.ok()) {
- return status;
- }
-
- constexpr int kVersion3 = 3; // Version 3 only cleans 1 sector at a time.
- constexpr int kVersion2 = 2; // Version 2 is always 1 byte aligned.
- if (is_sector_meta_erased ||
- sector_header_meta.synchronize_token != kSectorReadyValue) {
- // Sector needs to be setup
- if (Status status = ResetSector(i); !status.ok()) {
- return status;
- }
- continue;
- } else if (sector_header_meta.version != kVersion &&
- sector_header_meta.version != kVersion3 && // Allow version 3
- sector_header_meta.version != kVersion2) { // Allow version 2
- PW_LOG_ERROR("Unsupported KVS version in sector: %u",
- static_cast<unsigned>(i));
- return Status::FAILED_PRECONDITION;
- }
- uint32_t sector_header_cleaning_offset =
- RoundUpForAlignment(sizeof(KvsSectorHeaderMeta));
-
- bool clean_not_pending;
- if (Status status = partition_.IsChunkErased(
- SectorIndexToAddress(i) + sector_header_cleaning_offset,
- RoundUpForAlignment(sizeof(KvsSectorHeaderCleaning)),
- &clean_not_pending);
- !status.ok()) {
- return status;
- }
-
- if (!clean_not_pending) {
- // Sector is marked for cleaning, read the sector_clean_order
- if (Status status = UnalignedRead(
- &partition_,
- reinterpret_cast<uint8_t*>(§or_clean_order_[i]),
- SectorIndexToAddress(i) + sector_header_cleaning_offset,
- sizeof(KvsSectorHeaderCleaning::sector_clean_order));
- !status.ok()) {
- return status;
- }
- next_sector_clean_order_ =
- std::max(sector_clean_order_[i] + 1, next_sector_clean_order_);
- } else {
- sector_clean_order_[i] = kSectorCleanNotPending;
- }
-
- // Handle alignment
- if (sector_header_meta.version == kVersion2) {
- sector_header_meta.alignment_bytes = 1;
- }
- if (sector_header_meta.alignment_bytes != alignment_bytes_) {
- // NOTE: For now all sectors must have same alignment.
- PW_LOG_ERROR("Sector %u has unexpected alignment %u != %u",
- unsigned(i),
- unsigned(alignment_bytes_),
- unsigned(sector_header_meta.alignment_bytes));
- return Status::FAILED_PRECONDITION;
- }
-
- // Scan through sector and add key/value pairs to map.
- FlashPartition::Address offset =
- RoundUpForAlignment(sizeof(KvsSectorHeaderMeta)) +
- RoundUpForAlignment(sizeof(KvsSectorHeaderCleaning));
- sector_space_remaining_[i] = partition_.GetSectorSizeBytes() - offset;
- while (offset < partition_.GetSectorSizeBytes() -
- RoundUpForAlignment(sizeof(KvsHeader))) {
- FlashPartition::Address address = SectorIndexToAddress(i) + offset;
-
- // Read header
- KvsHeader header;
- bool is_kvs_header_erased;
- // Because underlying flash can be encrypted + erased, trying to readback
- // may give random values. It's important to make sure that data is not
- // erased before trying to see if there is a token match.
- if (Status status =
- partition_.IsChunkErased(address,
- RoundUpForAlignment(sizeof(header)),
- &is_kvs_header_erased);
- !status.ok()) {
- return status;
- }
- if (Status status = UnalignedRead(&partition_,
- reinterpret_cast<uint8_t*>(&header),
- address,
- sizeof(header));
- !status.ok()) {
- return status;
- }
- if (is_kvs_header_erased || header.synchronize_token != kChunkSyncValue) {
- if (!is_kvs_header_erased) {
- PW_LOG_ERROR("Next sync_token is not clear!");
- // TODO: handle this?
- }
- break; // End of elements in sector
- }
-
- if (header.key_len > kChunkKeyLengthMax) {
- PW_LOG_CRITICAL("Found key with invalid length %u; maximum is %u",
- header.key_len,
- static_cast<unsigned>(kChunkKeyLengthMax));
- return Status::DATA_LOSS;
- }
- static_assert(sizeof(temp_key_buffer_) >= kChunkKeyLengthMax + 1,
- "Key buffer must be at least large enough for a key and "
- "null terminator");
-
- // Read key and add to map
- if (Status status =
- UnalignedRead(&partition_,
- reinterpret_cast<uint8_t*>(&temp_key_buffer_),
- address + RoundUpForAlignment(sizeof(header)),
- header.key_len);
- !status.ok()) {
- return status;
- }
- temp_key_buffer_[header.key_len] = '\0';
- std::string_view key(temp_key_buffer_, header.key_len);
- bool is_erased = header.flags & kFlagsIsErasedMask;
-
- KeyIndex index = FindKeyInMap(key);
- if (index == kListCapacityMax) {
- if (Status status =
- AppendToMap(key, address, header.chunk_len, is_erased);
- !status.ok()) {
- return status;
- }
- } else if (sector_clean_order_[i] >=
- sector_clean_order_[AddressToSectorIndex(
- key_map_[index].address)]) {
- // The value being added is also in another sector (which is marked for
- // cleaning), but the current sector's order is larger and thefore this
- // is a newer version then what is already in the map.
- UpdateMap(index, address, header.chunk_len, is_erased);
- }
-
- // Increment search
- offset += ChunkSize(header.key_len, header.chunk_len);
- }
- sector_space_remaining_[i] =
- clean_not_pending ? partition_.GetSectorSizeBytes() - offset : 0;
- }
-
- if (Status status = EnforceFreeSector(); !status.ok()) {
- PW_LOG_ERROR(
- "%s: Failed to force clean at boot, no free sectors available!",
- status.str());
- }
- enabled_ = true;
- return Status::OK;
+ return hash;
}
-Status KeyValueStore::Get(const std::string_view& key,
- const span<byte>& value,
- uint16_t offset) {
+constexpr size_t EntrySize(string_view key, span<const byte> value) {
+ return sizeof(EntryHeader) + key.size() + value.size();
+}
+
+} // namespace
+
+Status KeyValueStore::Init() {
+ // enabled_ = true;
+
+ return Status::UNIMPLEMENTED;
+}
+
+StatusWithSize KeyValueStore::Get(string_view key,
+ span<byte> value_buffer) const {
+ TRY(InvalidOperation(key));
+
+ const KeyMapEntry* key_map_entry;
+ TRY(FindKeyMapEntry(key, &key_map_entry));
+
+ EntryHeader header;
+ TRY(ReadEntryHeader(*key_map_entry, &header));
+ StatusWithSize result = ReadEntryValue(*key_map_entry, header, value_buffer);
+
+ if (result.ok() && options_.verify_on_read) {
+ return ValidateEntryChecksum(header, key, value_buffer);
+ }
+ return result;
+}
+
+Status KeyValueStore::Put(string_view key, span<const byte> value) {
+ TRY(InvalidOperation(key));
+
+ if (value.size() > (1 << 24)) {
+ // TODO: Reject sizes that are larger than the maximum?
+ }
+
+ if (KeyMapEntry * entry; FindKeyMapEntry(key, &entry).ok()) {
+ return WriteEntryForExistingKey(entry, key, value);
+ }
+ return WriteEntryForNewKey(key, value);
+}
+
+Status KeyValueStore::Delete(string_view key) {
+ TRY(InvalidOperation(key));
+
+ return Status::UNIMPLEMENTED;
+}
+
+const KeyValueStore::Entry& KeyValueStore::Iterator::operator*() {
+ const KeyMapEntry& map_entry = entry_.kvs_.key_map_[index_];
+
+ EntryHeader header;
+ if (entry_.kvs_.ReadEntryHeader(map_entry, &header).ok()) {
+ entry_.kvs_.ReadEntryKey(
+ map_entry, header.key_length(), entry_.key_buffer_.data());
+ }
+
+ return entry_;
+}
+
+Status KeyValueStore::InvalidOperation(string_view key) const {
if (InvalidKey(key)) {
return Status::INVALID_ARGUMENT;
}
-
- // TODO: Support unaligned offset reads.
- if (offset % alignment_bytes_ != 0) {
- PW_LOG_ERROR("Currently unaligned offsets are not supported");
- return Status::INVALID_ARGUMENT;
- }
- // TODO: LOCK MUTEX
- if (!enabled_) {
+ if (!/*enabled_*/ 0) {
return Status::FAILED_PRECONDITION;
}
-
- KeyIndex key_index = FindKeyInMap(key);
- if (key_index == kListCapacityMax || key_map_[key_index].is_erased) {
- return Status::NOT_FOUND;
- }
- KvsHeader header;
- // TODO: Could cache the CRC and avoid reading the header.
- if (Status status = UnalignedRead(&partition_,
- reinterpret_cast<uint8_t*>(&header),
- key_map_[key_index].address,
- sizeof(header));
- !status.ok()) {
- return status;
- }
- if (kChunkSyncValue != header.synchronize_token) {
- return Status::DATA_LOSS;
- }
- if (value.size() + offset > header.chunk_len) {
- PW_LOG_ERROR("Out of bounds read: offset(%u) + size(%u) > data_size(%u)",
- offset,
- unsigned(value.size()),
- header.chunk_len);
- return Status::INVALID_ARGUMENT;
- }
- if (Status status = UnalignedRead(
- &partition_,
- value.data(),
- key_map_[key_index].address + RoundUpForAlignment(sizeof(KvsHeader)) +
- RoundUpForAlignment(header.key_len) + offset,
- value.size());
- !status.ok()) {
- return status;
- }
-
- // Verify CRC only when full packet was read.
- if (offset == 0 && value.size() == header.chunk_len) {
- uint16_t crc = CalculateCrc(key, value);
- if (crc != header.crc) {
- // TODO: Figure out how to print the key's string_view. For now, using %s
- // with a maximum length (8 characters). This still could trigger a small
- // out-of-bounds read.
- PW_LOG_ERROR(
- "KVS CRC does not match for key=%.8s [expected %u, found %u]",
- key.data(),
- header.crc,
- crc);
- return Status::DATA_LOSS;
- }
- }
return Status::OK;
}
-uint16_t KeyValueStore::CalculateCrc(const std::string_view& key,
- const span<const byte>& value) const {
- uint16_t crc = checksum::CcittCrc16(as_bytes(span(key)));
- return checksum::CcittCrc16(value, crc);
-}
+Status KeyValueStore::FindKeyMapEntry(string_view key,
+ const KeyMapEntry** result) const {
+ char key_buffer[kMaxKeyLength];
+ const uint32_t hash = HashKey(key);
-Status KeyValueStore::CalculateCrcFromValueAddress(
- const std::string_view& key,
- FlashPartition::Address value_address,
- uint16_t value_size,
- uint16_t* crc_ret) {
- uint16_t crc = checksum::CcittCrc16(as_bytes(span(key)));
- for (size_t i = 0; i < value_size; i += TempBufferAlignedSize()) {
- auto read_size = std::min(value_size - i, TempBufferAlignedSize());
- if (Status status = UnalignedRead(
- &partition_, temp_buffer_, value_address + i, read_size);
- !status.ok()) {
- return status;
- }
- crc = checksum::CcittCrc16(as_bytes(span(temp_buffer_, read_size)));
- }
- *crc_ret = crc;
- return Status::OK;
-}
+ for (auto& entry : entries()) {
+ if (entry.key_hash == hash) {
+ TRY(ReadEntryKey(entry, key.size(), key_buffer));
-Status KeyValueStore::Put(const std::string_view& key,
- const span<const byte>& value) {
- if (InvalidKey(key)) {
- return Status::INVALID_ARGUMENT;
- }
-
- // TODO: LOCK MUTEX
- if (!enabled_) {
- return Status::FAILED_PRECONDITION;
- }
-
- KeyIndex index = FindKeyInMap(key);
- if (index != kListCapacityMax) { // Key already in map, rewrite value
- return RewriteValue(index, value);
- }
-
- FlashPartition::Address address =
- FindSpace(ChunkSize(key.size(), value.size()));
- if (address == kSectorInvalid) {
- return Status::RESOURCE_EXHAUSTED;
- }
-
- // Check if this would use the last empty sector on KVS with multiple sectors
- if (SectorCount() > 1 && IsInLastFreeSector(address)) {
- // Forcing a full garbage collect to free more sectors.
- if (Status status = FullGarbageCollect(); !status.ok()) {
- return status;
- }
- address = FindSpace(ChunkSize(key.size(), value.size()));
- if (address == kSectorInvalid || IsInLastFreeSector(address)) {
- // Couldn't find space, KVS is full.
- return Status::RESOURCE_EXHAUSTED;
- }
- }
-
- if (Status status = WriteKeyValue(address, key, value); !status.ok()) {
- return status;
- }
- if (Status status = AppendToMap(key, address, value.size()); !status.ok()) {
- return status;
- }
-
- return Status::OK;
-}
-
-// Garbage collection cleans sectors to try to free up space.
-// If clean_pending_sectors is true, it will only clean sectors which are
-// currently pending a clean.
-// If clean_pending_sectors is false, it will only clean sectors which are not
-// currently pending a clean, instead it will mark them for cleaning and attempt
-// a clean.
-// If exit_when_have_free_sector is true, it will exit once a single free sector
-// exists.
-Status KeyValueStore::GarbageCollectImpl(bool clean_pending_sectors,
- bool exit_when_have_free_sector) {
- // Try to clean any pending sectors
- for (SectorIndex sector = 0; sector < SectorCount(); sector++) {
- if (clean_pending_sectors ==
- (sector_clean_order_[sector] != kSectorCleanNotPending)) {
- if (!clean_pending_sectors) {
- if (Status status = MarkSectorForClean(sector); !status.ok()) {
- return status;
- }
- }
- Status status = CleanSector(sector);
- if (!status.ok() && status != Status::RESOURCE_EXHAUSTED) {
- return status;
- }
- if (exit_when_have_free_sector && HasEmptySector()) {
- return Status::OK; // Now have a free sector
- }
- }
- }
- return Status::OK;
-}
-
-Status KeyValueStore::EnforceFreeSector() {
- if (SectorCount() == 1 || HasEmptySector()) {
- return Status::OK;
- }
- PW_LOG_INFO("KVS garbage collecting to get a free sector");
- if (Status status = GarbageCollectImpl(true /*clean_pending_sectors*/,
- true /*exit_when_have_free_sector*/);
- !status.ok()) {
- return status;
- }
- if (HasEmptySector()) {
- return Status::OK;
- }
- PW_LOG_INFO("KVS: trying to clean non-pending sectors for more space");
- if (Status status = GarbageCollectImpl(false /*clean_pending_sectors*/,
- true /*exit_when_have_free_sector*/);
- !status.ok()) {
- return status;
- }
- return HasEmptySector() ? Status::OK : Status::RESOURCE_EXHAUSTED;
-}
-
-Status KeyValueStore::FullGarbageCollect() {
- PW_LOG_INFO("KVS: Full garbage collecting to try to free space");
- if (Status status = GarbageCollectImpl(true /*clean_pending_sectors*/,
- false /*exit_when_have_free_sector*/);
- !status.ok()) {
- return status;
- }
- return GarbageCollectImpl(false /*clean_pending_sectors*/,
- false /*exit_when_have_free_sector*/);
-}
-
-Status KeyValueStore::RewriteValue(KeyIndex index,
- const span<const byte>& value,
- bool is_erased) {
- // Compare values, return if values are the same.
- if (ValueMatches(index, value, is_erased)) {
- return Status::OK;
- }
-
- if (key_map_[index].key_length > kChunkKeyLengthMax) {
- return Status::INTERNAL;
- }
-
- uint32_t space_required = ChunkSize(key_map_[index].key_length, value.size());
- SectorIndex sector = AddressToSectorIndex(key_map_[index].address);
- uint32_t sector_space_remaining = SectorSpaceRemaining(sector);
-
- FlashPartition::Address address = kSectorInvalid;
- if (sector_space_remaining >= space_required) {
- // Space available within sector, append to end
- address = SectorIndexToAddress(sector) + partition_.GetSectorSizeBytes() -
- sector_space_remaining;
- } else {
- // No space in current sector, mark sector for clean and use another sector.
- if (Status status = MarkSectorForClean(sector); !status.ok()) {
- return status;
- }
- address = FindSpace(ChunkSize(key_map_[index].key_length, value.size()));
- }
- if (address == kSectorInvalid) {
- return Status::RESOURCE_EXHAUSTED;
- }
- if (Status status =
- WriteKeyValue(address, key_map_[index].key(), value, is_erased);
- !status.ok()) {
- return status;
- }
- UpdateMap(index, address, value.size(), is_erased);
-
- return EnforceFreeSector();
-}
-
-bool KeyValueStore::ValueMatches(KeyIndex index,
- const span<const byte>& value,
- bool is_erased) {
- // Compare sizes of CRC.
- if (value.size() != key_map_[index].chunk_len) {
- return false;
- }
- KvsHeader header;
- UnalignedRead(&partition_,
- reinterpret_cast<uint8_t*>(&header),
- key_map_[index].address,
- sizeof(header));
- std::string_view key = key_map_[index].key();
- if (InvalidKey(key)) {
- return false;
- }
-
- if ((header.flags & kFlagsIsErasedMask) != is_erased) {
- return false;
- }
- if ((header.flags & kFlagsIsErasedMask) && is_erased) {
- return true;
- }
-
- // Compare checksums.
- if (header.crc != CalculateCrc(key_map_[index].key(), value)) {
- return false;
- }
- FlashPartition::Address address = key_map_[index].address +
- RoundUpForAlignment(sizeof(KvsHeader)) +
- RoundUpForAlignment(key.size());
- // Compare full values.
- for (size_t i = 0; i < key_map_[index].chunk_len;
- i += TempBufferAlignedSize()) {
- auto read_size =
- std::min(key_map_[index].chunk_len - i, TempBufferAlignedSize());
- auto status =
- UnalignedRead(&partition_, temp_buffer_, address + i, read_size);
- if (!status.ok()) {
- PW_LOG_ERROR("%s: Failed to read chunk", status.str());
- return false;
- }
- if (std::memcmp(&value[i], temp_buffer_, read_size) != 0) {
- return false;
- }
- }
- return true;
-}
-
-Status KeyValueStore::Erase(const std::string_view& key) {
- if (InvalidKey(key)) {
- return Status::INVALID_ARGUMENT;
- }
- // TODO: LOCK MUTEX
- if (!enabled_) {
- return Status::FAILED_PRECONDITION;
- }
-
- KeyIndex key_index = FindKeyInMap(key);
- if (key_index == kListCapacityMax || key_map_[key_index].is_erased) {
- return Status::NOT_FOUND;
- }
- return RewriteValue(key_index, span<byte>(), true);
-}
-
-Status KeyValueStore::ResetSector(SectorIndex sector_index) {
- KvsSectorHeaderMeta sector_header = {.synchronize_token = kSectorReadyValue,
- .version = kVersion,
- .alignment_bytes = alignment_bytes_,
- .padding = 0xFFFF};
- bool sector_erased = false;
- partition_.IsChunkErased(SectorIndexToAddress(sector_index),
- partition_.GetSectorSizeBytes(),
- §or_erased);
- auto status = partition_.Erase(SectorIndexToAddress(sector_index), 1);
-
- // If erasure failed, check first to see if it's merely unimplemented
- // (as in sub-sector KVSs).
- if (!status.ok() && !(status == Status::UNIMPLEMENTED && sector_erased)) {
- return status;
- }
-
- if (Status status =
- PaddedWrite(&partition_,
- SectorIndexToAddress(sector_index),
- reinterpret_cast<const uint8_t*>(§or_header),
- sizeof(sector_header));
- !status.ok()) {
- return status;
- }
-
- // Update space remaining
- sector_clean_order_[sector_index] = kSectorCleanNotPending;
- sector_space_remaining_[sector_index] = SectorSpaceAvailableWhenEmpty();
- return Status::OK;
-}
-
-Status KeyValueStore::WriteKeyValue(FlashPartition::Address address,
- const std::string_view& key,
- const span<const byte>& value,
- bool is_erased) {
- if (InvalidKey(key) ||
- value.size() >
- std::numeric_limits<decltype(KvsHeader::chunk_len)>::max()) {
- return Status::INTERNAL;
- }
-
- constexpr uint16_t kFlagDefaultValue = 0;
- KvsHeader header = {
- .synchronize_token = kChunkSyncValue,
- .crc = CalculateCrc(key, value),
- .flags = is_erased ? kFlagsIsErasedMask : kFlagDefaultValue,
- .key_len = static_cast<uint8_t>(key.size()),
- .chunk_len = static_cast<uint16_t>(value.size())};
-
- SectorIndex sector = AddressToSectorIndex(address);
- if (Status status = PaddedWrite(&partition_,
- address,
- reinterpret_cast<uint8_t*>(&header),
- sizeof(header));
- !status.ok()) {
- return status;
- }
- address += RoundUpForAlignment(sizeof(header));
- if (Status status = PaddedWrite(&partition_,
- address,
- reinterpret_cast<const uint8_t*>(key.data()),
- key.size());
- !status.ok()) {
- }
- address += RoundUpForAlignment(key.size());
- if (!value.empty()) {
- Status status =
- PaddedWrite(&partition_, address, value.data(), value.size());
- if (!status.ok()) {
- return status;
- }
- }
- sector_space_remaining_[sector] -= ChunkSize(key.size(), value.size());
- return Status::OK;
-}
-
-Status KeyValueStore::MoveChunk(FlashPartition::Address dest_address,
- FlashPartition::Address src_address,
- uint16_t size) {
- DCHECK_EQ(src_address % partition_.GetAlignmentBytes(), 0);
- DCHECK_EQ(dest_address % partition_.GetAlignmentBytes(), 0);
- DCHECK_EQ(size % partition_.GetAlignmentBytes(), 0);
-
- // Copy data over in chunks to reduce the size of the temporary buffer
- for (size_t i = 0; i < size; i += TempBufferAlignedSize()) {
- size_t move_size = std::min(size - i, TempBufferAlignedSize());
- DCHECK_EQ(move_size % alignment_bytes_, 0);
- if (Status status =
- partition_.Read(temp_buffer_, src_address + i, move_size);
- !status.ok()) {
- return status;
- }
- if (Status status =
- partition_.Write(dest_address + i, temp_buffer_, move_size);
- !status.ok()) {
- return status;
- }
- }
- return Status::OK;
-}
-
-Status KeyValueStore::MarkSectorForClean(SectorIndex sector) {
- if (sector_clean_order_[sector] != kSectorCleanNotPending) {
- return Status::OK; // Sector is already marked for clean
- }
-
- // Flag the sector as clean being active. This ensures we can handle losing
- // power while a clean is active.
- const KvsSectorHeaderCleaning kValue = {next_sector_clean_order_};
- if (Status status =
- PaddedWrite(&partition_,
- SectorIndexToAddress(sector) +
- RoundUpForAlignment(sizeof(KvsSectorHeaderMeta)),
- reinterpret_cast<const uint8_t*>(&kValue),
- sizeof(kValue));
- !status.ok()) {
- return status;
- }
- sector_space_remaining_[sector] = 0;
- sector_clean_order_[sector] = next_sector_clean_order_;
- next_sector_clean_order_++;
- return Status::OK;
-}
-
-Status KeyValueStore::CleanSector(SectorIndex sector) {
- // Iterate through the map, for each valid element which is in this sector:
- // - Find space in another sector which can fit this chunk.
- // - Add to the other sector and update the map.
- for (KeyValueStore::KeyIndex i = 0; i < map_size_; i++) {
- // If this key is an erased chunk don't need to move it.
- while (i < map_size_ &&
- sector == AddressToSectorIndex(key_map_[i].address) &&
- key_map_[i].is_erased) { // Remove this key from the map.
- RemoveFromMap(i);
- // NOTE: i is now a new key, and map_size_ has been decremented.
- }
-
- if (i < map_size_ && sector == AddressToSectorIndex(key_map_[i].address)) {
- FlashPartition::Address address = key_map_[i].address;
- auto size = ChunkSize(key_map_[i].key_length, key_map_[i].chunk_len);
- FlashPartition::Address move_address = FindSpace(size);
- if (move_address == kSectorInvalid) {
- return Status::RESOURCE_EXHAUSTED;
- }
- if (Status status = MoveChunk(move_address, address, size);
- !status.ok()) {
- return status;
- }
- sector_space_remaining_[AddressToSectorIndex(move_address)] -= size;
- key_map_[i].address = move_address; // Update map
- }
- }
- ResetSector(sector);
- return Status::OK;
-}
-
-Status KeyValueStore::CleanOneSector(bool* all_sectors_have_been_cleaned) {
- if (all_sectors_have_been_cleaned == nullptr) {
- return Status::INVALID_ARGUMENT;
- }
- // TODO: LOCK MUTEX
- bool have_cleaned_sector = false;
- for (SectorIndex sector = 0; sector < SectorCount(); sector++) {
- if (sector_clean_order_[sector] != kSectorCleanNotPending) {
- if (have_cleaned_sector) { // only clean 1 sector
- *all_sectors_have_been_cleaned = false;
+ if (key == string_view(key_buffer, key.size())) {
+ *result = &entry;
return Status::OK;
}
- if (Status status = CleanSector(sector); !status.ok()) {
- return status;
- }
- have_cleaned_sector = true;
}
}
- *all_sectors_have_been_cleaned = true;
+ return Status::NOT_FOUND;
+}
+
+Status KeyValueStore::ReadEntryHeader(const KeyMapEntry& entry,
+ EntryHeader* header) const {
+ return partition_.Read(entry.address, sizeof(*header), header).status();
+}
+
+Status KeyValueStore::ReadEntryKey(const KeyMapEntry& entry,
+ size_t key_length,
+ char* key) const {
+ // TODO: This check probably shouldn't be here; this is like
+ // checking that the Cortex M's RAM isn't corrupt. This should be
+ // done at boot time.
+ // ^^ This argument sometimes comes from EntryHeader::key_value_len,
+ // which is read directly from flash. If it's corrupted, we shouldn't try
+ // to read a bunch of extra data.
+ if (key_length == 0u || key_length > kMaxKeyLength) {
+ return Status::DATA_LOSS;
+ }
+ // The key is immediately after the entry header.
+ return partition_.Read(entry.address + sizeof(EntryHeader), key_length, key)
+ .status();
+}
+
+StatusWithSize KeyValueStore::ReadEntryValue(const KeyMapEntry& entry,
+ const EntryHeader& header,
+ span<byte> value) const {
+ const size_t read_size = std::min(header.value_length(), value.size());
+ StatusWithSize result =
+ partition_.Read(entry.address + sizeof(header) + header.key_length(),
+ value.subspan(read_size));
+ TRY(result);
+ if (read_size != header.value_length()) {
+ return StatusWithSize(Status::RESOURCE_EXHAUSTED, read_size);
+ }
+ return StatusWithSize(read_size);
+}
+
+Status KeyValueStore::ValidateEntryChecksum(const EntryHeader& header,
+ string_view key,
+ span<const byte> value) const {
+ CalculateEntryChecksum(header, key, value);
+ return entry_header_format_.checksum->Verify(header.checksum());
+}
+
+Status KeyValueStore::WriteEntryForExistingKey(KeyMapEntry* key_map_entry,
+ string_view key,
+ span<const byte> value) {
+ SectorMapEntry* sector;
+ TRY(FindOrRecoverSectorWithSpace(§or, EntrySize(key, value)));
+ return AppendEntry(sector, key_map_entry, key, value);
+}
+
+Status KeyValueStore::WriteEntryForNewKey(string_view key,
+ span<const byte> value) {
+ if (EntryMapFull()) {
+ // TODO: Log, and also expose "in memory keymap full" in stats dump.
+ return Status::RESOURCE_EXHAUSTED;
+ }
+
+ // Modify the entry at the end of the array, without bumping the map size
+ // so the entry is prepared and written without committing first.
+ KeyMapEntry& entry = key_map_[key_map_size_];
+ entry.key_hash = HashKey(key);
+ entry.key_version = 0; // will be incremented by AppendEntry()
+
+ SectorMapEntry* sector;
+ TRY(FindOrRecoverSectorWithSpace(§or, EntrySize(key, value)));
+ TRY(AppendEntry(sector, &entry, key, value));
+
+ // Only increment key_map_size_ when we are certain the write
+ // succeeded.
+ key_map_size_ += 1;
return Status::OK;
}
-Status KeyValueStore::CleanAllInternal() {
- for (SectorIndex sector = 0; sector < SectorCount(); sector++) {
- if (sector_clean_order_[sector] != kSectorCleanNotPending) {
- if (Status status = CleanSector(sector); !status.ok()) {
- return status;
+// Find either an existing sector with enough space, or an empty sector.
+// Maintains the invariant that there is always at least 1 empty sector.
+KeyValueStore::SectorMapEntry* KeyValueStore::FindSectorWithSpace(size_t size) {
+ int start = (last_written_sector_ + 1) % sector_map_.size();
+ SectorMapEntry* first_empty_sector = nullptr;
+ bool at_least_two_empty_sectors = false;
+
+ for (size_t i = start; i != last_written_sector_;
+ i = (i + 1) % sector_map_.size()) {
+ SectorMapEntry& sector = sector_map_[i];
+ if (!SectorEmpty(sector) && sector.HasSpace(size)) {
+ return §or;
+ }
+
+ if (SectorEmpty(sector)) {
+ if (first_empty_sector == nullptr) {
+ first_empty_sector = §or;
+ } else {
+ at_least_two_empty_sectors = true;
}
}
}
- return Status::OK;
+
+ if (at_least_two_empty_sectors) {
+ return first_empty_sector;
+ }
+ return nullptr;
}
-FlashPartition::Address KeyValueStore::FindSpace(size_t requested_size) const {
- if (requested_size > SectorSpaceAvailableWhenEmpty()) {
- return kSectorInvalid; // This would never fit
+Status KeyValueStore::FindOrRecoverSectorWithSpace(SectorMapEntry** sector,
+ size_t size) {
+ *sector = FindSectorWithSpace(size);
+ if (*sector != nullptr) {
+ return Status::OK;
}
- // Iterate through sectors, find first available sector with enough space.
- SectorIndex first_empty_sector = kSectorInvalid;
- for (SectorIndex i = 0; i < SectorCount(); i++) {
- uint32_t space_remaining = SectorSpaceRemaining(i);
- if (space_remaining == SectorSpaceAvailableWhenEmpty() &&
- first_empty_sector == kSectorInvalid) {
- // Skip the first empty sector to encourage keeping one sector free.
- first_empty_sector = i;
- continue;
- }
- if (space_remaining >= requested_size) {
- return SectorIndexToAddress(i) + partition_.GetSectorSizeBytes() -
- space_remaining;
+ if (options_.partial_gc_on_write) {
+ return GarbageCollectOneSector(sector);
+ }
+ return Status::RESOURCE_EXHAUSTED;
+}
+
+Status KeyValueStore::GarbageCollectOneSector(SectorMapEntry** sector) {
+ // TODO: THIS NEEDS WORK
+ (void)sector;
+ SectorMapEntry* sector_to_gc = FindSectorToGarbageCollect();
+
+ const Address sector_base = SectorBaseAddress(sector_to_gc);
+ const Address sector_end = sector_base + partition_.sector_size_bytes();
+
+ for (auto entry : entries()) {
+ if ((entry.address >= sector_base) && (entry.address < sector_end)) {
+ // RelocateEntry(entry);
}
}
- // Use first empty sector if that is all that is available.
- if (first_empty_sector != kSectorInvalid) {
- return SectorIndexToAddress(first_empty_sector) +
- partition_.GetSectorSizeBytes() - SectorSpaceAvailableWhenEmpty();
- }
- return kSectorInvalid;
-}
-uint32_t KeyValueStore::SectorSpaceRemaining(SectorIndex sector_index) const {
- return sector_space_remaining_[sector_index];
-}
-
-StatusWithSize KeyValueStore::GetValueSize(const std::string_view& key) {
- if (InvalidKey(key)) {
- return StatusWithSize(Status::INVALID_ARGUMENT, 0);
- }
-
- // TODO: LOCK MUTEX
- if (!enabled_) {
- return StatusWithSize(Status::FAILED_PRECONDITION, 0);
- }
-
- uint8_t idx = FindKeyInMap(key);
- if (idx == kListCapacityMax || key_map_[idx].is_erased) {
- return StatusWithSize(Status::NOT_FOUND, 0);
- }
- return StatusWithSize(key_map_[idx].chunk_len);
-}
-
-Status KeyValueStore::AppendToMap(const std::string_view& key,
- FlashPartition::Address address,
- size_t chunk_len,
- bool is_erased) {
- if (map_size_ >= kListCapacityMax) {
- PW_LOG_ERROR("Can't add: reached max supported keys %d", kListCapacityMax);
+ if (sector_to_gc->valid_bytes != 0) {
return Status::INTERNAL;
}
- auto& entry = key_map_[map_size_];
- entry.key_buffer[key.copy(entry.key_buffer, sizeof(KeyMap::key_buffer) - 1)] =
- '\0';
-
- entry.key_length = key.size();
- entry.address = address;
- entry.chunk_len = static_cast<uint16_t>(chunk_len);
- entry.is_erased = is_erased;
- map_size_++;
-
return Status::OK;
}
-KeyValueStore::KeyIndex KeyValueStore::FindKeyInMap(
- const std::string_view& key) const {
- for (KeyIndex i = 0; i < map_size_; i++) {
- if (key == std::string_view(key_map_[i].key())) {
- return i;
- }
+KeyValueStore::SectorMapEntry* KeyValueStore::FindSectorToGarbageCollect() {
+ // TODO: implement me
+ return nullptr;
+}
+
+Status KeyValueStore::AppendEntry(SectorMapEntry* sector,
+ KeyMapEntry* entry,
+ const string_view key,
+ span<const byte> value) {
+ // write header, key, and value
+ const EntryHeader header(entry_header_format_.magic,
+ CalculateEntryChecksum(header, key, value),
+ key.size(),
+ value.size(),
+ entry->key_version + 1);
+
+ // Handles writing multiple concatenated buffers, while breaking up the writes
+ // into alignment-sized blocks.
+ Address address = NextWritableAddress(sector);
+ TRY_ASSIGN(
+ size_t written,
+ partition_.Write(
+ address, {as_bytes(span(&header, 1)), as_bytes(span(key)), value}));
+
+ if (options_.verify_on_write) {
+ TRY(VerifyEntry(sector, entry));
}
- return kListCapacityMax;
+
+ // TODO: UPDATE last_written_sector_ appropriately
+
+ entry->address = address;
+ entry->key_version = header.key_version();
+ sector->valid_bytes += written;
+ sector->tail_free_bytes -= written;
+ return Status::OK;
}
-void KeyValueStore::UpdateMap(KeyIndex index,
- FlashPartition::Address address,
- uint16_t chunk_len,
- bool is_erased) {
- key_map_[index].address = address;
- key_map_[index].chunk_len = chunk_len;
- key_map_[index].is_erased = is_erased;
+Status KeyValueStore::VerifyEntry(SectorMapEntry* sector, KeyMapEntry* entry) {
+ // TODO: Implement me!
+ (void)sector;
+ (void)entry;
+ return Status::UNIMPLEMENTED;
}
-void KeyValueStore::RemoveFromMap(KeyIndex key_index) {
- key_map_[key_index] = key_map_[--map_size_];
+span<const byte> KeyValueStore::CalculateEntryChecksum(
+ const EntryHeader& header,
+ const string_view key,
+ span<const byte> value) const {
+ auto& checksum = *entry_header_format_.checksum;
+
+ checksum.Reset();
+ checksum.Update(header.DataForChecksum());
+ checksum.Update(as_bytes(span(key)));
+ checksum.Update(value.data(), value.size_bytes());
+ return checksum.state();
}
-size_t KeyValueStore::KeyCount() const {
- size_t count = 0;
- for (unsigned i = 0; i < map_size_; i++) {
- count += key_map_[i].is_erased ? 0 : 1;
- }
- return count;
-}
-
-std::string_view KeyValueStore::GetKey(size_t idx) const {
- unsigned count = 0;
- for (unsigned i = 0; i < map_size_; i++) {
- if (!key_map_[i].is_erased) {
- if (idx == count) {
- return key_map_[i].key();
- }
- count++;
- }
- }
- return {};
-}
-
-size_t KeyValueStore::GetValueSize(size_t idx) const {
- size_t count = 0;
- for (unsigned i = 0; i < map_size_; i++) {
- if (!key_map_[i].is_erased) {
- if (idx == count++) {
- return key_map_[i].chunk_len;
- }
- }
- }
- return 0;
+Status KeyValueStore::RelocateEntry(KeyMapEntry* entry) {
+ // TODO: implement me
+ (void)entry;
+ return Status::UNIMPLEMENTED;
}
} // namespace pw::kvs
diff --git a/pw_kvs/key_value_store_test.cc b/pw_kvs/key_value_store_test.cc
index 5d652dd..3b098a5 100644
--- a/pw_kvs/key_value_store_test.cc
+++ b/pw_kvs/key_value_store_test.cc
@@ -16,6 +16,8 @@
#include <array>
#include <cstdio>
+#include <cstring>
+#include <type_traits>
#include "pw_span/span.h"
@@ -23,10 +25,11 @@
#include "gtest/gtest.h"
#include "pw_checksum/ccitt_crc16.h"
-#include "pw_kvs/assert.h"
#include "pw_kvs/flash_memory.h"
+#include "pw_kvs_private/format.h"
+#include "pw_kvs_private/macros.h"
+#include "pw_log/log.h"
#include "pw_status/status.h"
-#include "pw_string/util.h"
#if USE_MEMORY_BUFFER
#include "pw_kvs/in_memory_fake_flash.h"
@@ -37,35 +40,59 @@
using std::byte;
-// Test that the IsSpan trait correctly idenitifies span instantiations.
-static_assert(!internal::IsSpan<int>());
-static_assert(!internal::IsSpan<void>());
-static_assert(!internal::IsSpan<std::array<int, 5>>());
+template <typename... Args>
+constexpr auto ByteArray(Args... args) {
+ return std::array<byte, sizeof...(args)>{static_cast<byte>(args)...};
+}
-static_assert(internal::IsSpan<span<int>>());
-static_assert(internal::IsSpan<span<byte>>());
-static_assert(internal::IsSpan<span<const int*>>());
+// Test that the ConvertsToSpan trait correctly idenitifies types that convert
+// to span.
+static_assert(!ConvertsToSpan<int>());
+static_assert(!ConvertsToSpan<void>());
+static_assert(!ConvertsToSpan<std::byte>());
+static_assert(!ConvertsToSpan<std::byte*>());
+
+static_assert(ConvertsToSpan<std::array<int, 5>>());
+static_assert(ConvertsToSpan<decltype("Hello!")>());
+
+static_assert(ConvertsToSpan<std::string_view>());
+static_assert(ConvertsToSpan<std::string_view&>());
+static_assert(ConvertsToSpan<std::string_view&&>());
+
+static_assert(ConvertsToSpan<const std::string_view>());
+static_assert(ConvertsToSpan<const std::string_view&>());
+static_assert(ConvertsToSpan<const std::string_view&&>());
+
+static_assert(ConvertsToSpan<span<int>>());
+static_assert(ConvertsToSpan<span<byte>>());
+static_assert(ConvertsToSpan<span<const int*>>());
+static_assert(ConvertsToSpan<span<bool>&&>());
+static_assert(ConvertsToSpan<const span<bool>&>());
+static_assert(ConvertsToSpan<span<bool>&&>());
#if USE_MEMORY_BUFFER
// Although it might be useful to test other configurations, some tests require
// at least 3 sectors; therfore it should have this when checked in.
InMemoryFakeFlash<4 * 1024, 4> test_flash(
16); // 4 x 4k sectors, 16 byte alignment
-FlashPartition test_partition(&test_flash, 0, test_flash.GetSectorCount());
+FlashPartition test_partition(&test_flash, 0, test_flash.sector_count());
InMemoryFakeFlash<1024, 60> large_test_flash(8);
FlashPartition large_test_partition(&large_test_flash,
0,
- large_test_flash.GetSectorCount());
+ large_test_flash.sector_count());
#else // TODO: Test with real flash
FlashPartition& test_partition = FlashExternalTestPartition();
#endif // USE_MEMORY_BUFFER
-KeyValueStore kvs(&test_partition);
+// TODO: Need a checksum implementation (e.g. CRC16) to use for tests.
+constexpr EntryHeaderFormat format{.magic = 0xBAD'C0D3, .checksum = nullptr};
+
+KeyValueStore kvs(&test_partition, format);
// Use test fixture for logging support
class KeyValueStoreTest : public ::testing::Test {
protected:
- KeyValueStoreTest() : kvs_local_(&test_partition) {}
+ KeyValueStoreTest() : kvs_local_(&test_partition, format) {}
KeyValueStore kvs_local_;
};
@@ -75,21 +102,20 @@
Status PaddedWrite(FlashPartition* partition,
FlashPartition::Address address,
- const uint8_t* buf,
- uint16_t size) {
- static constexpr uint16_t kMaxAlignmentBytes = 128;
- uint8_t alignment_buffer[kMaxAlignmentBytes] = {0};
- uint16_t aligned_bytes = size - (size % partition->GetAlignmentBytes());
- if (Status status = partition->Write(address, buf, aligned_bytes);
- !status.ok()) {
- return status;
- }
+ const std::byte* buf,
+ size_t size) {
+ static constexpr size_t kMaxAlignmentBytes = 128;
+ byte alignment_buffer[kMaxAlignmentBytes] = {};
+
+ size_t aligned_bytes = size - (size % partition->alignment_bytes());
+ TRY(partition->Write(address, span(buf, aligned_bytes)));
+
uint16_t remaining_bytes = size - aligned_bytes;
if (remaining_bytes > 0) {
std::memcpy(alignment_buffer, &buf[aligned_bytes], remaining_bytes);
- if (Status status = partition->Write(address + aligned_bytes,
- alignment_buffer,
- partition->GetAlignmentBytes());
+ if (Status status = partition->Write(
+ address + aligned_bytes,
+ span(alignment_buffer, partition->alignment_bytes()));
!status.ok()) {
return status;
}
@@ -98,7 +124,10 @@
}
size_t RoundUpForAlignment(size_t size) {
- uint16_t alignment = test_partition.GetAlignmentBytes();
+ // TODO: THIS IS SO PADDEDWRITE APPEARS USED
+ (void)PaddedWrite;
+
+ uint16_t alignment = test_partition.alignment_bytes();
if (size % alignment != 0) {
return size + alignment - size % alignment;
}
@@ -110,10 +139,10 @@
public:
KvsAttributes(size_t key_size, size_t data_size)
: sector_header_meta_size_(
- RoundUpForAlignment(KeyValueStore::kHeaderSize)),
+ RoundUpForAlignment(sizeof(EntryHeader))), // TODO: not correct
sector_header_clean_size_(
- RoundUpForAlignment(KeyValueStore::kHeaderSize)),
- chunk_header_size_(RoundUpForAlignment(KeyValueStore::kHeaderSize)),
+ RoundUpForAlignment(sizeof(EntryHeader))), // TODO: not correct
+ chunk_header_size_(RoundUpForAlignment(sizeof(EntryHeader))),
data_size_(RoundUpForAlignment(data_size)),
key_size_(RoundUpForAlignment(key_size)),
erase_size_(chunk_header_size_ + key_size_),
@@ -148,7 +177,7 @@
const size_t kMaxPutSize =
buffer.size() + kvs_attr.ChunkHeaderSize() + kvs_attr.KeySize();
- CHECK(size_to_fill >= kvs_attr.MinPutSize() + kvs_attr.EraseSize());
+ ASSERT_GE(size_to_fill, kvs_attr.MinPutSize() + kvs_attr.EraseSize());
// Saving enough space to perform erase after loop
size_to_fill -= kvs_attr.EraseSize();
@@ -168,43 +197,56 @@
size_to_fill -= chunk_len;
chunk_len = std::min(size_to_fill, kMaxPutSize);
}
- ASSERT_EQ(Status::OK, kvs.Erase(key));
+ ASSERT_EQ(Status::OK, kvs.Delete(key));
}
uint16_t CalcKvsCrc(const char* key, const void* data, size_t data_len) {
- static constexpr size_t kChunkKeyLengthMax = 15;
- uint16_t crc = checksum::CcittCrc16(
- as_bytes(span(key, string::Length(key, kChunkKeyLengthMax))));
+ uint16_t crc = checksum::CcittCrc16(as_bytes(span(key, std::strlen(key))));
return checksum::CcittCrc16(span(static_cast<const byte*>(data), data_len),
crc);
}
uint16_t CalcTestPartitionCrc() {
- uint8_t buf[16]; // Read as 16 byte chunks
- CHECK_EQ(sizeof(buf) % test_partition.GetAlignmentBytes(), 0);
- CHECK_EQ(test_partition.GetSizeBytes() % sizeof(buf), 0);
+ byte buf[16]; // Read as 16 byte chunks
+ EXPECT_EQ(sizeof(buf) % test_partition.alignment_bytes(), 0u);
+ EXPECT_EQ(test_partition.size_bytes() % sizeof(buf), 0u);
uint16_t crc = checksum::kCcittCrc16DefaultInitialValue;
- for (size_t i = 0; i < test_partition.GetSizeBytes(); i += sizeof(buf)) {
- test_partition.Read(buf, i, sizeof(buf));
+ for (size_t i = 0; i < test_partition.size_bytes(); i += sizeof(buf)) {
+ test_partition.Read(i, buf);
crc = checksum::CcittCrc16(as_bytes(span(buf)), crc);
}
return crc;
}
+
} // namespace
-TEST_F(KeyValueStoreTest, FuzzTest) {
- if (test_partition.GetSectorSizeBytes() < 4 * 1024 ||
- test_partition.GetSectorCount() < 4) {
+TEST_F(KeyValueStoreTest, Iteration_Empty_ByReference) {
+ kvs.Init();
+ for (const KeyValueStore::Entry& entry : kvs) {
+ FAIL(); // The KVS is empty; this shouldn't execute.
+ static_cast<void>(entry);
+ }
+}
+
+TEST_F(KeyValueStoreTest, Iteration_Empty_ByValue) {
+ kvs.Init();
+ for (KeyValueStore::Entry entry : kvs) {
+ FAIL(); // The KVS is empty; this shouldn't execute.
+ static_cast<void>(entry);
+ }
+}
+
+TEST_F(KeyValueStoreTest, DISABLED_FuzzTest) {
+ if (test_partition.sector_size_bytes() < 4 * 1024 ||
+ test_partition.sector_count() < 4) {
PW_LOG_INFO("Sectors too small, skipping test.");
return; // TODO: Test could be generalized
}
// Erase
- ASSERT_EQ(Status::OK,
- test_partition.Erase(0, test_partition.GetSectorCount()));
+ ASSERT_EQ(Status::OK, test_partition.Erase(0, test_partition.sector_count()));
// Reset KVS
- kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
const char* key1 = "Buf1";
const char* key2 = "Buf2";
const size_t kLargestBufSize = 3 * 1024;
@@ -229,22 +271,21 @@
ASSERT_EQ(Status::OK, kvs.Put("some_data", j));
}
// Delete and re-add everything
- ASSERT_EQ(Status::OK, kvs.Erase(key1));
+ ASSERT_EQ(Status::OK, kvs.Delete(key1));
ASSERT_EQ(Status::OK, kvs.Put(key1, span(buf1, size1)));
- ASSERT_EQ(Status::OK, kvs.Erase(key2));
+ ASSERT_EQ(Status::OK, kvs.Delete(key2));
ASSERT_EQ(Status::OK, kvs.Put(key2, span(buf2, size2)));
for (size_t j = 0; j < keys.size(); j++) {
- ASSERT_EQ(Status::OK, kvs.Erase(keys[j]));
+ ASSERT_EQ(Status::OK, kvs.Delete(keys[j]));
ASSERT_EQ(Status::OK, kvs.Put(keys[j], j));
}
// Re-enable and verify
- kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
static byte buf[4 * 1024];
- ASSERT_EQ(Status::OK, kvs.Get(key1, span(buf, size1)));
+ ASSERT_EQ(Status::OK, kvs.Get(key1, span(buf, size1)).status());
ASSERT_EQ(std::memcmp(buf, buf1, size1), 0);
- ASSERT_EQ(Status::OK, kvs.Get(key2, span(buf, size2)));
+ ASSERT_EQ(Status::OK, kvs.Get(key2, span(buf, size2)).status());
ASSERT_EQ(std::memcmp(buf2, buf2, size2), 0);
for (size_t j = 0; j < keys.size(); j++) {
size_t ret = 1000;
@@ -254,14 +295,12 @@
}
}
-TEST_F(KeyValueStoreTest, Basic) {
+TEST_F(KeyValueStoreTest, DISABLED_Basic) {
// Erase
- ASSERT_EQ(Status::OK,
- test_partition.Erase(0, test_partition.GetSectorCount()));
+ ASSERT_EQ(Status::OK, test_partition.Erase(0, test_partition.sector_count()));
// Reset KVS
- kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
// Add some data
uint8_t value1 = 0xDA;
@@ -281,31 +320,30 @@
EXPECT_EQ(test2, value2);
// Erase a key
- EXPECT_EQ(Status::OK, kvs.Erase(keys[0]));
+ EXPECT_EQ(Status::OK, kvs.Delete(keys[0]));
// Verify it was erased
EXPECT_EQ(kvs.Get(keys[0], &test1), Status::NOT_FOUND);
test2 = 0;
ASSERT_EQ(
Status::OK,
- kvs.Get(keys[1], span(reinterpret_cast<byte*>(&test2), sizeof(test2))));
+ kvs.Get(keys[1], span(reinterpret_cast<byte*>(&test2), sizeof(test2)))
+ .status());
EXPECT_EQ(test2, value2);
// Erase other key
- kvs.Erase(keys[1]);
+ kvs.Delete(keys[1]);
// Verify it was erased
- EXPECT_EQ(kvs.KeyCount(), 0u);
+ EXPECT_EQ(kvs.size(), 0u);
}
-TEST_F(KeyValueStoreTest, MaxKeyLength) {
+TEST_F(KeyValueStoreTest, DISABLED_MaxKeyLength) {
// Erase
- ASSERT_EQ(Status::OK,
- test_partition.Erase(0, test_partition.GetSectorCount()));
+ ASSERT_EQ(Status::OK, test_partition.Erase(0, test_partition.sector_count()));
// Reset KVS
- kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
// Add some data
char key[16] = "123456789abcdef"; // key length 15 (without \0)
@@ -318,14 +356,14 @@
EXPECT_EQ(test, value);
// Erase a key
- kvs.Erase(key);
+ kvs.Delete(key);
// Verify it was erased
EXPECT_EQ(kvs.Get(key, &test), Status::NOT_FOUND);
}
-TEST_F(KeyValueStoreTest, LargeBuffers) {
- test_partition.Erase(0, test_partition.GetSectorCount());
+TEST_F(KeyValueStoreTest, DISABLED_LargeBuffers) {
+ test_partition.Erase(0, test_partition.sector_count());
// Note this assumes that no other keys larger then key0
static_assert(sizeof(keys[0]) >= sizeof(keys[1]) &&
@@ -337,26 +375,25 @@
// empty sector also.
const size_t kAllChunkSize = kvs_attr.MinPutSize() * keys.size();
const size_t kAllSectorHeaderSizes =
- kvs_attr.SectorHeaderSize() * (test_partition.GetSectorCount() - 1);
+ kvs_attr.SectorHeaderSize() * (test_partition.sector_count() - 1);
const size_t kMinSize = kAllChunkSize + kAllSectorHeaderSizes;
- const size_t kAvailSectorSpace = test_partition.GetSectorSizeBytes() *
- (test_partition.GetSectorCount() - 1);
+ const size_t kAvailSectorSpace =
+ test_partition.sector_size_bytes() * (test_partition.sector_count() - 1);
if (kAvailSectorSpace < kMinSize) {
PW_LOG_INFO("KVS too small, skipping test.");
return;
}
// Reset KVS
- kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
// Add and verify
for (unsigned add_idx = 0; add_idx < keys.size(); add_idx++) {
std::memset(buffer.data(), add_idx, buffer.size());
ASSERT_EQ(Status::OK, kvs.Put(keys[add_idx], buffer));
- EXPECT_EQ(kvs.KeyCount(), add_idx + 1);
+ EXPECT_EQ(kvs.size(), add_idx + 1);
for (unsigned verify_idx = 0; verify_idx <= add_idx; verify_idx++) {
std::memset(buffer.data(), 0, buffer.size());
- ASSERT_EQ(Status::OK, kvs.Get(keys[verify_idx], buffer));
+ ASSERT_EQ(Status::OK, kvs.Get(keys[verify_idx], buffer).status());
for (unsigned i = 0; i < buffer.size(); i++) {
EXPECT_EQ(static_cast<unsigned>(buffer[i]), verify_idx);
}
@@ -365,14 +402,15 @@
// Erase and verify
for (unsigned erase_idx = 0; erase_idx < keys.size(); erase_idx++) {
- ASSERT_EQ(Status::OK, kvs.Erase(keys[erase_idx]));
- EXPECT_EQ(kvs.KeyCount(), keys.size() - erase_idx - 1);
+ ASSERT_EQ(Status::OK, kvs.Delete(keys[erase_idx]));
+ EXPECT_EQ(kvs.size(), keys.size() - erase_idx - 1);
for (unsigned verify_idx = 0; verify_idx < keys.size(); verify_idx++) {
std::memset(buffer.data(), 0, buffer.size());
if (verify_idx <= erase_idx) {
- ASSERT_EQ(kvs.Get(keys[verify_idx], buffer), Status::NOT_FOUND);
+ ASSERT_EQ(kvs.Get(keys[verify_idx], buffer).status(),
+ Status::NOT_FOUND);
} else {
- ASSERT_EQ(Status::OK, kvs.Get(keys[verify_idx], buffer));
+ ASSERT_EQ(Status::OK, kvs.Get(keys[verify_idx], buffer).status());
for (uint32_t i = 0; i < buffer.size(); i++) {
EXPECT_EQ(buffer[i], static_cast<byte>(verify_idx));
}
@@ -381,8 +419,8 @@
}
}
-TEST_F(KeyValueStoreTest, Enable) {
- test_partition.Erase(0, test_partition.GetSectorCount());
+TEST_F(KeyValueStoreTest, DISABLED_Enable) {
+ test_partition.Erase(0, test_partition.sector_count());
KvsAttributes kvs_attr(std::strlen(keys[0]), buffer.size());
@@ -391,30 +429,29 @@
// empty sector also.
const size_t kAllChunkSize = kvs_attr.MinPutSize() * keys.size();
const size_t kAllSectorHeaderSizes =
- kvs_attr.SectorHeaderSize() * (test_partition.GetSectorCount() - 1);
+ kvs_attr.SectorHeaderSize() * (test_partition.sector_count() - 1);
const size_t kMinSize = kAllChunkSize + kAllSectorHeaderSizes;
- const size_t kAvailSectorSpace = test_partition.GetSectorSizeBytes() *
- (test_partition.GetSectorCount() - 1);
+ const size_t kAvailSectorSpace =
+ test_partition.sector_size_bytes() * (test_partition.sector_count() - 1);
if (kAvailSectorSpace < kMinSize) {
PW_LOG_INFO("KVS too small, skipping test.");
return;
}
// Reset KVS
- kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
// Add some items
for (unsigned add_idx = 0; add_idx < keys.size(); add_idx++) {
std::memset(buffer.data(), add_idx, buffer.size());
ASSERT_EQ(Status::OK, kvs.Put(keys[add_idx], buffer));
- EXPECT_EQ(kvs.KeyCount(), add_idx + 1);
+ EXPECT_EQ(kvs.size(), add_idx + 1);
}
// Enable different KVS which should be able to properly setup the same map
// from what is stored in flash.
- ASSERT_EQ(Status::OK, kvs_local_.Enable());
- EXPECT_EQ(kvs_local_.KeyCount(), keys.size());
+ ASSERT_EQ(Status::OK, kvs_local_.Init());
+ EXPECT_EQ(kvs_local_.size(), keys.size());
// Ensure adding to new KVS works
uint8_t value = 0xDA;
@@ -423,34 +460,32 @@
uint8_t test;
ASSERT_EQ(Status::OK, kvs_local_.Get(key, &test));
EXPECT_EQ(value, test);
- EXPECT_EQ(kvs_local_.KeyCount(), keys.size() + 1);
+ EXPECT_EQ(kvs_local_.size(), keys.size() + 1);
// Verify previous data
for (unsigned verify_idx = 0; verify_idx < keys.size(); verify_idx++) {
std::memset(buffer.data(), 0, buffer.size());
- ASSERT_EQ(Status::OK, kvs_local_.Get(keys[verify_idx], buffer));
+ ASSERT_EQ(Status::OK, kvs_local_.Get(keys[verify_idx], buffer).status());
for (uint32_t i = 0; i < buffer.size(); i++) {
EXPECT_EQ(static_cast<unsigned>(buffer[i]), verify_idx);
}
}
}
-TEST_F(KeyValueStoreTest, MultiSector) {
- test_partition.Erase(0, test_partition.GetSectorCount());
+TEST_F(KeyValueStoreTest, DISABLED_MultiSector) {
+ test_partition.Erase(0, test_partition.sector_count());
// Reset KVS
- kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
// Calculate number of elements to ensure multiple sectors are required.
- uint16_t add_count =
- (test_partition.GetSectorSizeBytes() / buffer.size()) + 1;
+ uint16_t add_count = (test_partition.sector_size_bytes() / buffer.size()) + 1;
- if (kvs.GetMaxKeys() < add_count) {
+ if (kvs.max_size() < add_count) {
PW_LOG_INFO("Sector size too large, skipping test.");
return; // this chip has very large sectors, test won't work
}
- if (test_partition.GetSectorCount() < 3) {
+ if (test_partition.sector_count() < 3) {
PW_LOG_INFO("Not enough sectors, skipping test.");
return; // need at least 3 sectors for multi-sector test
}
@@ -460,13 +495,13 @@
std::memset(buffer.data(), add_idx, buffer.size());
snprintf(key, sizeof(key), "key_%u", add_idx);
ASSERT_EQ(Status::OK, kvs.Put(key, buffer));
- EXPECT_EQ(kvs.KeyCount(), add_idx + 1);
+ EXPECT_EQ(kvs.size(), add_idx + 1);
}
for (unsigned verify_idx = 0; verify_idx < add_count; verify_idx++) {
std::memset(buffer.data(), 0, buffer.size());
snprintf(key, sizeof(key), "key_%u", verify_idx);
- ASSERT_EQ(Status::OK, kvs.Get(key, buffer));
+ ASSERT_EQ(Status::OK, kvs.Get(key, buffer).status());
for (uint32_t i = 0; i < buffer.size(); i++) {
EXPECT_EQ(static_cast<unsigned>(buffer[i]), verify_idx);
}
@@ -475,17 +510,16 @@
// Check erase
for (unsigned erase_idx = 0; erase_idx < add_count; erase_idx++) {
snprintf(key, sizeof(key), "key_%u", erase_idx);
- ASSERT_EQ(Status::OK, kvs.Erase(key));
- EXPECT_EQ(kvs.KeyCount(), add_count - erase_idx - 1);
+ ASSERT_EQ(Status::OK, kvs.Delete(key));
+ EXPECT_EQ(kvs.size(), add_count - erase_idx - 1);
}
}
-TEST_F(KeyValueStoreTest, RewriteValue) {
- test_partition.Erase(0, test_partition.GetSectorCount());
+TEST_F(KeyValueStoreTest, DISABLED_RewriteValue) {
+ test_partition.Erase(0, test_partition.sector_count());
// Reset KVS
- kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
// Write first value
const uint8_t kValue1 = 0xDA;
@@ -495,62 +529,65 @@
// Verify
uint8_t value;
- ASSERT_EQ(Status::OK, kvs.Get(key, as_writable_bytes(span(&value, 1))));
+ ASSERT_EQ(Status::OK,
+ kvs.Get(key, as_writable_bytes(span(&value, 1))).status());
EXPECT_EQ(kValue1, value);
// Write new value for key
ASSERT_EQ(Status::OK, kvs.Put(key, as_bytes(span(&kValue2, 1))));
// Verify
- ASSERT_EQ(Status::OK, kvs.Get(key, as_writable_bytes(span(&value, 1))));
+ ASSERT_EQ(Status::OK,
+ kvs.Get(key, as_writable_bytes(span(&value, 1))).status());
EXPECT_EQ(kValue2, value);
// Verify only 1 element exists
- EXPECT_EQ(kvs.KeyCount(), 1u);
+ EXPECT_EQ(kvs.size(), 1u);
}
+#if 0 // Offset reads are not yet supported
+
TEST_F(KeyValueStoreTest, OffsetRead) {
- test_partition.Erase(0, test_partition.GetSectorCount());
+ test_partition.Erase(0, test_partition.sector_count());
// Reset KVS
- kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
const char* key = "the_key";
constexpr size_t kReadSize = 16; // needs to be a multiple of alignment
constexpr size_t kTestBufferSize = kReadSize * 10;
- CHECK_GT(buffer.size(), kTestBufferSize);
- CHECK_LE(kTestBufferSize, 0xFF);
+ ASSERT_GT(buffer.size(), kTestBufferSize);
+ ASSERT_LE(kTestBufferSize, 0xFF);
// Write the entire buffer
for (size_t i = 0; i < kTestBufferSize; i++) {
buffer[i] = byte(i);
}
ASSERT_EQ(Status::OK, kvs.Put(key, span(buffer.data(), kTestBufferSize)));
- EXPECT_EQ(kvs.KeyCount(), 1u);
+ EXPECT_EQ(kvs.size(), 1u);
// Read in small chunks and verify
for (unsigned i = 0; i < kTestBufferSize / kReadSize; i++) {
std::memset(buffer.data(), 0, buffer.size());
- ASSERT_EQ(Status::OK,
- kvs.Get(key, span(buffer.data(), kReadSize), i * kReadSize));
+ ASSERT_EQ(
+ Status::OK,
+ kvs.Get(key, span(buffer.data(), kReadSize), i * kReadSize).status());
for (unsigned j = 0; j < kReadSize; j++) {
ASSERT_EQ(static_cast<unsigned>(buffer[j]), j + i * kReadSize);
}
}
}
+#endif
-TEST_F(KeyValueStoreTest, MultipleRewrite) {
+TEST_F(KeyValueStoreTest, DISABLED_MultipleRewrite) {
// Write many large buffers to force moving to new sector.
- test_partition.Erase(0, test_partition.GetSectorCount());
+ test_partition.Erase(0, test_partition.sector_count());
// Reset KVS
- kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
// Calculate number of elements to ensure multiple sectors are required.
- unsigned add_count =
- (test_partition.GetSectorSizeBytes() / buffer.size()) + 1;
+ unsigned add_count = (test_partition.sector_size_bytes() / buffer.size()) + 1;
const char* key = "the_key";
constexpr uint8_t kGoodVal = 0x60;
@@ -561,32 +598,31 @@
std::memset(buffer.data(), kGoodVal, buffer.size());
}
ASSERT_EQ(Status::OK, kvs.Put(key, buffer));
- EXPECT_EQ(kvs.KeyCount(), 1u);
+ EXPECT_EQ(kvs.size(), 1u);
}
// Verify
std::memset(buffer.data(), 0, buffer.size());
- ASSERT_EQ(Status::OK, kvs.Get(key, buffer));
+ ASSERT_EQ(Status::OK, kvs.Get(key, buffer).status());
for (uint32_t i = 0; i < buffer.size(); i++) {
ASSERT_EQ(buffer[i], static_cast<byte>(kGoodVal));
}
}
-TEST_F(KeyValueStoreTest, FillSector) {
+TEST_F(KeyValueStoreTest, DISABLED_FillSector) {
// Write key[0], Write/erase Key[2] multiple times to fill sector check
// everything makes sense after.
- test_partition.Erase(0, test_partition.GetSectorCount());
+ test_partition.Erase(0, test_partition.sector_count());
// Reset KVS
- kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
ASSERT_EQ(std::strlen(keys[0]), 8U); // Easier for alignment
ASSERT_EQ(std::strlen(keys[2]), 8U); // Easier for alignment
constexpr size_t kTestDataSize = 8;
KvsAttributes kvs_attr(std::strlen(keys[2]), kTestDataSize);
int bytes_remaining =
- test_partition.GetSectorSizeBytes() - kvs_attr.SectorHeaderSize();
+ test_partition.sector_size_bytes() - kvs_attr.SectorHeaderSize();
constexpr byte kKey0Pattern = byte{0xBA};
std::memset(
@@ -598,10 +634,10 @@
ASSERT_EQ(Status::OK,
kvs.Put(keys[2], span(buffer.data(), kvs_attr.DataSize())));
bytes_remaining -= kvs_attr.MinPutSize();
- EXPECT_EQ(kvs.KeyCount(), 2u);
- ASSERT_EQ(Status::OK, kvs.Erase(keys[2]));
+ EXPECT_EQ(kvs.size(), 2u);
+ ASSERT_EQ(Status::OK, kvs.Delete(keys[2]));
bytes_remaining -= kvs_attr.EraseSize();
- EXPECT_EQ(kvs.KeyCount(), 1u);
+ EXPECT_EQ(kvs.size(), 1u);
// Intentionally adding erase size to trigger sector cleanup
bytes_remaining += kvs_attr.EraseSize();
@@ -609,77 +645,78 @@
// Verify key[0]
std::memset(buffer.data(), 0, kvs_attr.DataSize());
- ASSERT_EQ(Status::OK,
- kvs.Get(keys[0], span(buffer.data(), kvs_attr.DataSize())));
+ ASSERT_EQ(
+ Status::OK,
+ kvs.Get(keys[0], span(buffer.data(), kvs_attr.DataSize())).status());
for (uint32_t i = 0; i < kvs_attr.DataSize(); i++) {
EXPECT_EQ(buffer[i], kKey0Pattern);
}
}
-TEST_F(KeyValueStoreTest, Interleaved) {
- test_partition.Erase(0, test_partition.GetSectorCount());
+TEST_F(KeyValueStoreTest, DISABLED_Interleaved) {
+ test_partition.Erase(0, test_partition.sector_count());
// Reset KVS
- kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
const uint8_t kValue1 = 0xDA;
const uint8_t kValue2 = 0x12;
uint8_t value;
ASSERT_EQ(Status::OK, kvs.Put(keys[0], kValue1));
- EXPECT_EQ(kvs.KeyCount(), 1u);
- ASSERT_EQ(Status::OK, kvs.Erase(keys[0]));
+ EXPECT_EQ(kvs.size(), 1u);
+ ASSERT_EQ(Status::OK, kvs.Delete(keys[0]));
EXPECT_EQ(kvs.Get(keys[0], &value), Status::NOT_FOUND);
ASSERT_EQ(Status::OK, kvs.Put(keys[1], as_bytes(span(&kValue1, 1))));
ASSERT_EQ(Status::OK, kvs.Put(keys[2], kValue2));
- ASSERT_EQ(Status::OK, kvs.Erase(keys[1]));
+ ASSERT_EQ(Status::OK, kvs.Delete(keys[1]));
EXPECT_EQ(Status::OK, kvs.Get(keys[2], &value));
EXPECT_EQ(kValue2, value);
- EXPECT_EQ(kvs.KeyCount(), 1u);
+ EXPECT_EQ(kvs.size(), 1u);
}
-TEST_F(KeyValueStoreTest, BadCrc) {
+TEST_F(KeyValueStoreTest, DISABLED_BadCrc) {
static constexpr uint32_t kTestPattern = 0xBAD0301f;
// clang-format off
// There is a top and bottom because for each because we don't want to write
// the erase 0xFF, especially on encrypted flash.
- static constexpr uint8_t kKvsTestDataAligned1Top[] = {
- 0xCD, 0xAB, 0x03, 0x00, 0x01, 0x00, 0xFF, 0xFF, // Sector Header
- };
- static constexpr uint8_t kKvsTestDataAligned1Bottom[] = {
+ static constexpr auto kKvsTestDataAligned1Top = ByteArray(
+ 0xCD, 0xAB, 0x03, 0x00, 0x01, 0x00, 0xFF, 0xFF // Sector Header
+ );
+ static constexpr auto kKvsTestDataAligned1Bottom = ByteArray(
0xAA, 0x55, 0xBA, 0xDD, 0x00, 0x00, 0x18, 0x00, // header (BAD CRC)
0x54, 0x65, 0x73, 0x74, 0x4B, 0x65, 0x79, 0x31, // Key (keys[0])
0xDA, // Value
0xAA, 0x55, 0xB5, 0x87, 0x00, 0x00, 0x44, 0x00, // Header (GOOD CRC)
0x4B, 0x65, 0x79, 0x32, // Key (keys[1])
- 0x1F, 0x30, 0xD0, 0xBA}; // Value
- static constexpr uint8_t kKvsTestDataAligned2Top[] = {
- 0xCD, 0xAB, 0x03, 0x00, 0x02, 0x00, 0xFF, 0xFF, // Sector Header
- };
- static constexpr uint8_t kKvsTestDataAligned2Bottom[] = {
+ 0x1F, 0x30, 0xD0, 0xBA); // Value
+ static constexpr auto kKvsTestDataAligned2Top = ByteArray(
+ 0xCD, 0xAB, 0x03, 0x00, 0x02, 0x00, 0xFF, 0xFF // Sector Header
+ );
+ static constexpr auto kKvsTestDataAligned2Bottom = ByteArray(
0xAA, 0x55, 0xBA, 0xDD, 0x00, 0x00, 0x18, 0x00, // header (BAD CRC)
0x54, 0x65, 0x73, 0x74, 0x4B, 0x65, 0x79, 0x31, // Key (keys[0])
0xDA, 0x00, // Value + padding
0xAA, 0x55, 0xB5, 0x87, 0x00, 0x00, 0x44, 0x00, // Header (GOOD CRC)
0x4B, 0x65, 0x79, 0x32, // Key (keys[1])
- 0x1F, 0x30, 0xD0, 0xBA}; // Value
- static constexpr uint8_t kKvsTestDataAligned8Top[] = {
- 0xCD, 0xAB, 0x03, 0x00, 0x08, 0x00, 0xFF, 0xFF, // Sector Header
- };
- static constexpr uint8_t kKvsTestDataAligned8Bottom[] = {
+ 0x1F, 0x30, 0xD0, 0xBA // Value
+ );
+ static constexpr auto kKvsTestDataAligned8Top = ByteArray(
+ 0xCD, 0xAB, 0x03, 0x00, 0x08, 0x00, 0xFF, 0xFF // Sector Header
+ );
+ static constexpr auto kKvsTestDataAligned8Bottom = ByteArray(
0xAA, 0x55, 0xBA, 0xDD, 0x00, 0x00, 0x18, 0x00, // header (BAD CRC)
0x54, 0x65, 0x73, 0x74, 0x4B, 0x65, 0x79, 0x31, // Key (keys[0])
0xDA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Value + padding
0xAA, 0x55, 0xB5, 0x87, 0x00, 0x00, 0x44, 0x00, // header (GOOD CRC)
0x4B, 0x65, 0x79, 0x32, 0x00, 0x00, 0x00, 0x00, // Key (keys[1])
- 0x1F, 0x30, 0xD0, 0xBA, 0x00, 0x00, 0x00, 0x00, // Value + padding
- };
- static constexpr uint8_t kKvsTestDataAligned16Top[] = {
+ 0x1F, 0x30, 0xD0, 0xBA, 0x00, 0x00, 0x00, 0x00 // Value + padding
+ );
+ static constexpr auto kKvsTestDataAligned16Top = ByteArray(
0xCD, 0xAB, 0x03, 0x00, 0x10, 0x00, 0xFF, 0xFF, // Sector Header
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Alignment to 16
- };
- static constexpr uint8_t kKvsTestDataAligned16Bottom[] = {
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // Alignment to 16
+ );
+ static constexpr auto kKvsTestDataAligned16Bottom = ByteArray(
0xAA, 0x55, 0xBA, 0xDD, 0x00, 0x00, 0x18, 0x00, // header (BAD CRC)
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Alignment to 16
0x54, 0x65, 0x73, 0x74, 0x4B, 0x65, 0x79, 0x31, // Key (keys[0])
@@ -691,57 +728,56 @@
0x4B, 0x65, 0x79, 0x32, 0x00, 0x00, 0x00, 0x00, // Key (keys[1])
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Alignment to 16
0x1F, 0x30, 0xD0, 0xBA, 0x00, 0x00, 0x00, 0x00, // Value + padding
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Alignment to 16
- };
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // Alignment to 16
+ );
// clang-format on
- ASSERT_EQ(Status::OK,
- test_partition.Erase(0, test_partition.GetSectorCount()));
+ ASSERT_EQ(Status::OK, test_partition.Erase(0, test_partition.sector_count()));
// We don't actually care about the size values provided, since we are only
// using kvs_attr to get Sector Size
KvsAttributes kvs_attr(8, 8);
- if (test_partition.GetAlignmentBytes() == 1) {
+ if (test_partition.alignment_bytes() == 1) {
ASSERT_EQ(Status::OK,
- test_partition.Write(
- 0, kKvsTestDataAligned1Top, sizeof(kKvsTestDataAligned1Top)));
- ASSERT_EQ(Status::OK,
- test_partition.Write(kvs_attr.SectorHeaderSize(),
- kKvsTestDataAligned1Bottom,
- sizeof(kKvsTestDataAligned1Bottom)));
- } else if (test_partition.GetAlignmentBytes() == 2) {
- ASSERT_EQ(Status::OK,
- test_partition.Write(
- 0, kKvsTestDataAligned2Top, sizeof(kKvsTestDataAligned2Top)));
- ASSERT_EQ(Status::OK,
- test_partition.Write(kvs_attr.SectorHeaderSize(),
- kKvsTestDataAligned2Bottom,
- sizeof(kKvsTestDataAligned2Bottom)));
- } else if (test_partition.GetAlignmentBytes() == 8) {
- ASSERT_EQ(Status::OK,
- test_partition.Write(
- 0, kKvsTestDataAligned8Top, sizeof(kKvsTestDataAligned8Top)));
- ASSERT_EQ(Status::OK,
- test_partition.Write(kvs_attr.SectorHeaderSize(),
- kKvsTestDataAligned8Bottom,
- sizeof(kKvsTestDataAligned8Bottom)));
- } else if (test_partition.GetAlignmentBytes() == 16) {
+ test_partition.Write(0, kKvsTestDataAligned1Top).status());
ASSERT_EQ(
Status::OK,
- test_partition.Write(
- 0, kKvsTestDataAligned16Top, sizeof(kKvsTestDataAligned16Top)));
+ test_partition
+ .Write(kvs_attr.SectorHeaderSize(), kKvsTestDataAligned1Bottom)
+ .status());
+ } else if (test_partition.alignment_bytes() == 2) {
ASSERT_EQ(Status::OK,
- test_partition.Write(kvs_attr.SectorHeaderSize(),
- kKvsTestDataAligned16Bottom,
- sizeof(kKvsTestDataAligned16Bottom)));
+ test_partition.Write(0, kKvsTestDataAligned2Top).status());
+ ASSERT_EQ(
+ Status::OK,
+ test_partition
+ .Write(kvs_attr.SectorHeaderSize(), kKvsTestDataAligned2Bottom)
+ .status());
+ } else if (test_partition.alignment_bytes() == 8) {
+ ASSERT_EQ(Status::OK,
+ test_partition.Write(0, kKvsTestDataAligned8Top).status());
+ ASSERT_EQ(
+ Status::OK,
+ test_partition
+ .Write(kvs_attr.SectorHeaderSize(), kKvsTestDataAligned8Bottom)
+ .status());
+ } else if (test_partition.alignment_bytes() == 16) {
+ ASSERT_EQ(Status::OK,
+ test_partition.Write(0, kKvsTestDataAligned16Top).status());
+ ASSERT_EQ(
+ Status::OK,
+ test_partition
+ .Write(kvs_attr.SectorHeaderSize(), kKvsTestDataAligned16Bottom)
+ .status());
} else {
PW_LOG_ERROR("Test only supports 1, 2, 8 and 16 byte alignments.");
ASSERT_EQ(Status::OK, false);
}
- EXPECT_EQ(kvs_local_.Enable(), Status::OK);
- EXPECT_TRUE(kvs_local_.IsEnabled());
+ EXPECT_EQ(kvs_local_.Init(), Status::OK);
+ EXPECT_TRUE(kvs_local_.initialized());
- EXPECT_EQ(kvs_local_.Get(keys[0], span(buffer.data(), 1)), Status::DATA_LOSS);
+ EXPECT_EQ(Status::DATA_LOSS,
+ kvs_local_.Get(keys[0], span(buffer.data(), 1)).status());
// Value with correct CRC should still be available.
uint32_t test2 = 0;
@@ -755,14 +791,13 @@
EXPECT_EQ(kTestPattern, test2);
// Check correct when re-enabled
- kvs_local_.Disable();
- EXPECT_EQ(kvs_local_.Enable(), Status::OK);
+ EXPECT_EQ(kvs_local_.Init(), Status::OK);
test2 = 0;
EXPECT_EQ(Status::OK, kvs_local_.Get(keys[0], &test2));
EXPECT_EQ(kTestPattern, test2);
}
-TEST_F(KeyValueStoreTest, TestVersion2) {
+TEST_F(KeyValueStoreTest, DISABLED_TestVersion2) {
static constexpr uint32_t kTestPattern = 0xBAD0301f;
// Since this test is not run on encypted flash, we can write the clean
// pending flag for just this test.
@@ -773,27 +808,25 @@
0x4B, 0x65, 0x79, 0x32, // Key (keys[1])
0x1F, 0x30, 0xD0, 0xBA}; // Value
- if (test_partition.GetAlignmentBytes() == 1) {
+ if (test_partition.alignment_bytes() == 1) {
// Test only runs on 1 byte alignment partitions
- test_partition.Erase(0, test_partition.GetSectorCount());
- test_partition.Write(0, kKvsTestDataAligned1, sizeof(kKvsTestDataAligned1));
- EXPECT_EQ(Status::OK, kvs_local_.Enable());
+ test_partition.Erase(0, test_partition.sector_count());
+ test_partition.Write(0, as_bytes(span(kKvsTestDataAligned1)));
+ EXPECT_EQ(Status::OK, kvs_local_.Init());
uint32_t test2 = 0;
- ASSERT_EQ(Status::OK,
- kvs_local_.Get(keys[1], as_writable_bytes(span(&test2, 1))));
+ ASSERT_EQ(
+ Status::OK,
+ kvs_local_.Get(keys[1], as_writable_bytes(span(&test2, 1))).status());
EXPECT_EQ(kTestPattern, test2);
}
}
-TEST_F(KeyValueStoreTest, ReEnable) {
- test_partition.Erase(0, test_partition.GetSectorCount());
+TEST_F(KeyValueStoreTest, DISABLED_ReEnable) {
+ test_partition.Erase(0, test_partition.sector_count());
// Reset KVS
- kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
- kvs.Disable();
-
- EXPECT_EQ(Status::OK, kvs_local_.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
+ EXPECT_EQ(Status::OK, kvs_local_.Init());
// Write value
const uint8_t kValue = 0xDA;
ASSERT_EQ(Status::OK, kvs_local_.Put(keys[0], kValue));
@@ -804,33 +837,30 @@
EXPECT_EQ(kValue, value);
}
-TEST_F(KeyValueStoreTest, Erase) {
- test_partition.Erase(0, test_partition.GetSectorCount());
+TEST_F(KeyValueStoreTest, DISABLED_Erase) {
+ test_partition.Erase(0, test_partition.sector_count());
// Reset KVS
- kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
// Write value
const uint8_t kValue = 0xDA;
ASSERT_EQ(Status::OK, kvs.Put(keys[0], kValue));
- ASSERT_EQ(Status::OK, kvs.Erase(keys[0]));
+ ASSERT_EQ(Status::OK, kvs.Delete(keys[0]));
uint8_t value;
ASSERT_EQ(kvs.Get(keys[0], &value), Status::NOT_FOUND);
// Reset KVS, ensure captured at enable
- kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
ASSERT_EQ(kvs.Get(keys[0], &value), Status::NOT_FOUND);
}
-TEST_F(KeyValueStoreTest, TemplatedPutAndGet) {
- test_partition.Erase(0, test_partition.GetSectorCount());
+TEST_F(KeyValueStoreTest, DISABLED_TemplatedPutAndGet) {
+ test_partition.Erase(0, test_partition.sector_count());
// Reset KVS
- kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
// Store a value with the convenience method.
const uint32_t kValue = 0x12345678;
ASSERT_EQ(Status::OK, kvs.Put(keys[0], kValue));
@@ -847,123 +877,114 @@
ASSERT_EQ(small_value, kSmallValue);
}
-TEST_F(KeyValueStoreTest, SameValueRewrite) {
+TEST_F(KeyValueStoreTest, DISABLED_SameValueRewrite) {
static constexpr uint32_t kTestPattern = 0xBAD0301f;
// clang-format off
- static constexpr uint8_t kKvsTestDataAligned1Top[] = {
- 0xCD, 0xAB, 0x02, 0x00, 0x00, 0x00, 0xFF, 0xFF, // Sector Header
- };
- static constexpr uint8_t kKvsTestDataAligned1Bottom[] = {
+ static constexpr auto kKvsTestDataAligned1Top = ByteArray(
+ 0xCD, 0xAB, 0x02, 0x00, 0x00, 0x00, 0xFF, 0xFF // Sector Header
+ );
+ static constexpr auto kKvsTestDataAligned1Bottom = ByteArray(
+ 0xAA, 0x55, 0xB5, 0x87, 0x00, 0x00, 0x44, 0x00, // Header (GOOD CRC)
+ 0x4B, 0x65, 0x79, 0x32, // Key (keys[1])
+ 0x1F, 0x30, 0xD0, 0xBA // Value
+ );
+ static constexpr auto kKvsTestDataAligned2Top = ByteArray(
+ 0xCD, 0xAB, 0x03, 0x00, 0x02, 0x00, 0xFF, 0xFF // Sector Header
+ );
+ static constexpr auto kKvsTestDataAligned2Bottom = ByteArray(
0xAA, 0x55, 0xB5, 0x87, 0x00, 0x00, 0x44, 0x00, // Header (GOOD CRC)
0x4B, 0x65, 0x79, 0x32, // Key (keys[1])
- 0x1F, 0x30, 0xD0, 0xBA}; // Value
- static constexpr uint8_t kKvsTestDataAligned2Top[] = {
- 0xCD, 0xAB, 0x03, 0x00, 0x02, 0x00, 0xFF, 0xFF, // Sector Header
- };
- static constexpr uint8_t kKvsTestDataAligned2Bottom[] = {
- 0xAA, 0x55, 0xB5, 0x87, 0x00, 0x00, 0x44, 0x00, // Header (GOOD CRC)
- 0x4B, 0x65, 0x79, 0x32, // Key (keys[1])
- 0x1F, 0x30, 0xD0, 0xBA}; // Value
- static constexpr uint8_t kKvsTestDataAligned8Top[] = {
- 0xCD, 0xAB, 0x03, 0x00, 0x08, 0x00, 0xFF, 0xFF, // Sector Header
- };
- static constexpr uint8_t kKvsTestDataAligned8Bottom[] = {
+ 0x1F, 0x30, 0xD0, 0xBA // Value
+ );
+ static constexpr auto kKvsTestDataAligned8Top = ByteArray(
+ 0xCD, 0xAB, 0x03, 0x00, 0x08, 0x00, 0xFF, 0xFF // Sector Header
+ );
+ static constexpr auto kKvsTestDataAligned8Bottom = ByteArray(
0xAA, 0x55, 0xB5, 0x87, 0x00, 0x00, 0x44, 0x00, // header (GOOD CRC)
0x4B, 0x65, 0x79, 0x32, 0x00, 0x00, 0x00, 0x00, // Key (keys[1])
- 0x1F, 0x30, 0xD0, 0xBA, 0x00, 0x00, 0x00, 0x00, // Value + padding
- };
- static constexpr uint8_t kKvsTestDataAligned16Top[] = {
+ 0x1F, 0x30, 0xD0, 0xBA, 0x00, 0x00, 0x00, 0x00 // Value + padding
+ );
+ static constexpr auto kKvsTestDataAligned16Top = ByteArray(
0xCD, 0xAB, 0x03, 0x00, 0x10, 0x00, 0xFF, 0xFF, // Sector Header
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Alignment to 16
- };
- static constexpr uint8_t kKvsTestDataAligned16Bottom[] = {
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // Alignment to 16
+ );
+ static constexpr auto kKvsTestDataAligned16Bottom = ByteArray(
0xAA, 0x55, 0xB5, 0x87, 0x00, 0x00, 0x44, 0x00, // header (GOOD CRC)
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Alignment to 16
0x4B, 0x65, 0x79, 0x32, 0x00, 0x00, 0x00, 0x00, // Key (keys[1])
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Alignment to 16
0x1F, 0x30, 0xD0, 0xBA, 0x00, 0x00, 0x00, 0x00, // Value + padding
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Alignment to 16
- };
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // Alignment to 16
+ );
// clang-format on
- ASSERT_EQ(Status::OK,
- test_partition.Erase(0, test_partition.GetSectorCount()));
+ ASSERT_EQ(Status::OK, test_partition.Erase(0, test_partition.sector_count()));
// We don't actually care about the size values provided, since we are only
// using kvs_attr to get Sector Size
KvsAttributes kvs_attr(8, 8);
FlashPartition::Address address = kvs_attr.SectorHeaderSize();
- if (test_partition.GetAlignmentBytes() == 1) {
+ if (test_partition.alignment_bytes() == 1) {
ASSERT_EQ(Status::OK,
- test_partition.Write(
- 0, kKvsTestDataAligned1Top, sizeof(kKvsTestDataAligned1Top)));
- ASSERT_EQ(Status::OK,
- test_partition.Write(address,
- kKvsTestDataAligned1Bottom,
- sizeof(kKvsTestDataAligned1Bottom)));
- address += sizeof(kKvsTestDataAligned1Bottom);
- } else if (test_partition.GetAlignmentBytes() == 2) {
- ASSERT_EQ(Status::OK,
- test_partition.Write(
- 0, kKvsTestDataAligned2Top, sizeof(kKvsTestDataAligned2Top)));
- ASSERT_EQ(Status::OK,
- test_partition.Write(address,
- kKvsTestDataAligned2Bottom,
- sizeof(kKvsTestDataAligned2Bottom)));
- address += sizeof(kKvsTestDataAligned2Bottom);
- } else if (test_partition.GetAlignmentBytes() == 8) {
- ASSERT_EQ(Status::OK,
- test_partition.Write(
- 0, kKvsTestDataAligned8Top, sizeof(kKvsTestDataAligned8Top)));
- ASSERT_EQ(Status::OK,
- test_partition.Write(address,
- kKvsTestDataAligned8Bottom,
- sizeof(kKvsTestDataAligned8Bottom)));
- address += sizeof(kKvsTestDataAligned8Bottom);
- } else if (test_partition.GetAlignmentBytes() == 16) {
+ test_partition.Write(0, kKvsTestDataAligned1Top).status());
ASSERT_EQ(
Status::OK,
- test_partition.Write(
- 0, kKvsTestDataAligned16Top, sizeof(kKvsTestDataAligned16Top)));
+ test_partition.Write(address, kKvsTestDataAligned1Bottom).status());
+ address += sizeof(kKvsTestDataAligned1Bottom);
+ } else if (test_partition.alignment_bytes() == 2) {
ASSERT_EQ(Status::OK,
- test_partition.Write(address,
- kKvsTestDataAligned16Bottom,
- sizeof(kKvsTestDataAligned16Bottom)));
+ test_partition.Write(0, kKvsTestDataAligned2Top).status());
+ ASSERT_EQ(
+ Status::OK,
+ test_partition.Write(address, kKvsTestDataAligned2Bottom).status());
+ address += sizeof(kKvsTestDataAligned2Bottom);
+ } else if (test_partition.alignment_bytes() == 8) {
+ ASSERT_EQ(Status::OK,
+ test_partition.Write(0, kKvsTestDataAligned8Top).status());
+ ASSERT_EQ(
+ Status::OK,
+ test_partition.Write(address, kKvsTestDataAligned8Bottom).status());
+ address += sizeof(kKvsTestDataAligned8Bottom);
+ } else if (test_partition.alignment_bytes() == 16) {
+ ASSERT_EQ(Status::OK,
+ test_partition.Write(0, kKvsTestDataAligned16Top).status());
+ ASSERT_EQ(
+ Status::OK,
+ test_partition.Write(address, kKvsTestDataAligned16Bottom).status());
address += sizeof(kKvsTestDataAligned16Bottom);
} else {
PW_LOG_ERROR("Test only supports 1, 2, 8 and 16 byte alignments.");
ASSERT_EQ(true, false);
}
- ASSERT_EQ(Status::OK, kvs_local_.Enable());
- EXPECT_TRUE(kvs_local_.IsEnabled());
+ ASSERT_EQ(Status::OK, kvs_local_.Init());
+ EXPECT_TRUE(kvs_local_.initialized());
// Put in same key/value pair
ASSERT_EQ(Status::OK, kvs_local_.Put(keys[1], kTestPattern));
bool is_erased = false;
ASSERT_EQ(Status::OK,
- test_partition.IsChunkErased(
- address, test_partition.GetAlignmentBytes(), &is_erased));
+ test_partition.IsRegionErased(
+ address, test_partition.alignment_bytes(), &is_erased));
EXPECT_EQ(is_erased, true);
}
// This test is derived from bug that was discovered. Testing this corner case
// relies on creating a new key-value just under the size that is left over in
// the sector.
-TEST_F(KeyValueStoreTest, FillSector2) {
- if (test_partition.GetSectorCount() < 3) {
+TEST_F(KeyValueStoreTest, DISABLED_FillSector2) {
+ if (test_partition.sector_count() < 3) {
PW_LOG_INFO("Not enough sectors, skipping test.");
return; // need at least 3 sectors
}
// Reset KVS
- kvs.Disable();
- test_partition.Erase(0, test_partition.GetSectorCount());
- ASSERT_EQ(Status::OK, kvs.Enable());
+ test_partition.Erase(0, test_partition.sector_count());
+ ASSERT_EQ(Status::OK, kvs.Init());
// Start of by filling flash sector to near full
constexpr int kHalfBufferSize = buffer.size() / 2;
- const int kSizeToFill = test_partition.GetSectorSizeBytes() - kHalfBufferSize;
+ const int kSizeToFill = test_partition.sector_size_bytes() - kHalfBufferSize;
constexpr size_t kTestDataSize = 8;
KvsAttributes kvs_attr(std::strlen(keys[2]), kTestDataSize);
@@ -972,15 +993,15 @@
// Find out how much space is remaining for new key-value and confirm it
// makes sense.
size_t new_keyvalue_size = 0;
- size_t alignment = test_partition.GetAlignmentBytes();
+ size_t alignment = test_partition.alignment_bytes();
// Starts on second sector since it will try to keep first sector free
FlashPartition::Address read_address =
- 2 * test_partition.GetSectorSizeBytes() - alignment;
+ 2 * test_partition.sector_size_bytes() - alignment;
for (; read_address > 0; read_address -= alignment) {
bool is_erased = false;
ASSERT_EQ(
Status::OK,
- test_partition.IsChunkErased(read_address, alignment, &is_erased));
+ test_partition.IsRegionErased(read_address, alignment, &is_erased));
if (is_erased) {
new_keyvalue_size += alignment;
} else {
@@ -988,7 +1009,7 @@
}
}
- size_t expected_remaining = test_partition.GetSectorSizeBytes() -
+ size_t expected_remaining = test_partition.sector_size_bytes() -
kvs_attr.SectorHeaderSize() - kSizeToFill;
ASSERT_EQ(new_keyvalue_size, expected_remaining);
@@ -1002,69 +1023,69 @@
// In failed corner case, adding new key is deceptively successful. It isn't
// until KVS is disabled and reenabled that issue can be detected.
- kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
// Might as well check that new key-value is what we expect it to be
ASSERT_EQ(Status::OK,
- kvs.Get(kNewKey, span(buffer.data(), new_keyvalue_size)));
+ kvs.Get(kNewKey, span(buffer.data(), new_keyvalue_size)).status());
for (size_t i = 0; i < new_keyvalue_size; i++) {
EXPECT_EQ(buffer[i], kTestPattern);
}
}
-TEST_F(KeyValueStoreTest, GetValueSizeTests) {
+TEST_F(KeyValueStoreTest, DISABLED_GetValueSizeTests) {
constexpr uint16_t kSizeOfValueToFill = 20U;
constexpr uint8_t kKey0Pattern = 0xBA;
// Start off with disabled KVS
- kvs.Disable();
+ // kvs.Disable();
// Try getting value when KVS is disabled, expect failure
- EXPECT_EQ(kvs.GetValueSize(keys[0]).status(), Status::FAILED_PRECONDITION);
+ EXPECT_EQ(kvs.ValueSize(keys[0]).status(), Status::FAILED_PRECONDITION);
// Reset KVS
- test_partition.Erase(0, test_partition.GetSectorCount());
- ASSERT_EQ(Status::OK, kvs.Enable());
+ test_partition.Erase(0, test_partition.sector_count());
+ ASSERT_EQ(Status::OK, kvs.Init());
// Try some case that are expected to fail
- ASSERT_EQ(kvs.GetValueSize(keys[0]).status(), Status::NOT_FOUND);
- ASSERT_EQ(kvs.GetValueSize("").status(), Status::INVALID_ARGUMENT);
+ ASSERT_EQ(kvs.ValueSize(keys[0]).status(), Status::NOT_FOUND);
+ ASSERT_EQ(kvs.ValueSize("").status(), Status::INVALID_ARGUMENT);
// Add key[0] and test we get the right value size for it.
std::memset(buffer.data(), kKey0Pattern, kSizeOfValueToFill);
ASSERT_EQ(Status::OK,
kvs.Put(keys[0], span(buffer.data(), kSizeOfValueToFill)));
- ASSERT_EQ(kSizeOfValueToFill, kvs.GetValueSize(keys[0]).size());
+ ASSERT_EQ(kSizeOfValueToFill, kvs.ValueSize(keys[0]).size());
// Verify after erase key is not found
- ASSERT_EQ(Status::OK, kvs.Erase(keys[0]));
- ASSERT_EQ(kvs.GetValueSize(keys[0]).status(), Status::NOT_FOUND);
+ ASSERT_EQ(Status::OK, kvs.Delete(keys[0]));
+ ASSERT_EQ(kvs.ValueSize(keys[0]).status(), Status::NOT_FOUND);
}
-TEST_F(KeyValueStoreTest, CanFitEntryTests) {
+#if 0 // TODO: not CanFitEntry function yet
+TEST_F(KeyValueStoreTest, DISABLED_CanFitEntryTests) {
// Reset KVS
- kvs.Disable();
- test_partition.Erase(0, test_partition.GetSectorCount());
- ASSERT_EQ(Status::OK, kvs.Enable());
+ test_partition.Erase(0, test_partition.sector_count());
+ ASSERT_EQ(Status::OK, kvs.Init());
// Get exactly the number of bytes that can fit in the space remaining for
// a large value, accounting for alignment.
constexpr uint16_t kTestKeySize = 2;
size_t space_remaining =
- test_partition.GetSectorSizeBytes() //
- - RoundUpForAlignment(KeyValueStore::kHeaderSize) // Sector Header
- - RoundUpForAlignment(KeyValueStore::kHeaderSize) // Cleaning Header
- - RoundUpForAlignment(KeyValueStore::kHeaderSize) // Chunk Header
+ test_partition.sector_size_bytes() //
+ - RoundUpForAlignment(sizeof(EntryHeader)) // TODO: Sector Header
+ - RoundUpForAlignment(sizeof(EntryHeader)) // Cleaning Header
+ - RoundUpForAlignment(sizeof(EntryHeader)) // TODO: Chunk Header
- RoundUpForAlignment(kTestKeySize);
- space_remaining -= test_partition.GetAlignmentBytes() / 2;
+ space_remaining -= test_partition.alignment_bytes() / 2;
space_remaining = RoundUpForAlignment(space_remaining);
EXPECT_TRUE(kvs.CanFitEntry(kTestKeySize, space_remaining));
EXPECT_FALSE(kvs.CanFitEntry(kTestKeySize, space_remaining + 1));
}
+#endif
-TEST_F(KeyValueStoreTest, DifferentValueSameCrc16) {
- test_partition.Erase(0, test_partition.GetSectorCount());
+TEST_F(KeyValueStoreTest, DISABLED_DifferentValueSameCrc16) {
+ test_partition.Erase(0, test_partition.sector_count());
const char kKey[] = "k";
// With the key and our CRC16 algorithm these both have CRC of 0x82AE
// Given they are the same size and same key, the KVS will need to check
@@ -1077,8 +1098,7 @@
CalcKvsCrc(kKey, kValue2, sizeof(kValue2)));
// Reset KVS
- kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
ASSERT_EQ(Status::OK, kvs.Put(kKey, kValue1));
// Now try to rewrite with the similar value.
@@ -1090,31 +1110,32 @@
ASSERT_EQ(std::memcmp(value, kValue2, sizeof(value)), 0);
}
-TEST_F(KeyValueStoreTest, CallingEraseTwice) {
- test_partition.Erase(0, test_partition.GetSectorCount());
+TEST_F(KeyValueStoreTest, DISABLED_CallingEraseTwice) {
+ test_partition.Erase(0, test_partition.sector_count());
// Reset KVS
- kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
const uint8_t kValue = 0xDA;
ASSERT_EQ(Status::OK, kvs.Put(keys[0], kValue));
- ASSERT_EQ(Status::OK, kvs.Erase(keys[0]));
+ ASSERT_EQ(Status::OK, kvs.Delete(keys[0]));
uint16_t crc = CalcTestPartitionCrc();
- EXPECT_EQ(kvs.Erase(keys[0]), Status::NOT_FOUND);
+ EXPECT_EQ(kvs.Delete(keys[0]), Status::NOT_FOUND);
// Verify the flash has not changed
EXPECT_EQ(crc, CalcTestPartitionCrc());
}
void __attribute__((noinline)) StackHeavyPartialClean() {
- CHECK_GE(test_partition.GetSectorCount(), 2);
+#if 0 // TODO: No FlashSubPartition
+
+ ASSERT_GE(test_partition.sector_count(), 2);
FlashSubPartition test_partition_sector1(&test_partition, 0, 1);
FlashSubPartition test_partition_sector2(&test_partition, 1, 1);
KeyValueStore kvs1(&test_partition_sector1);
KeyValueStore kvs2(&test_partition_sector2);
- test_partition.Erase(0, test_partition.GetSectorCount());
+ test_partition.Erase(0, test_partition.sector_count());
ASSERT_EQ(Status::OK, kvs1.Enable());
ASSERT_EQ(Status::OK, kvs2.Enable());
@@ -1127,7 +1148,7 @@
int values2[3] = {200, 201, 202};
ASSERT_EQ(Status::OK, kvs2.Put(keys[0], values2[0]));
ASSERT_EQ(Status::OK, kvs2.Put(keys[1], values2[1]));
- ASSERT_EQ(Status::OK, kvs2.Erase(keys[1]));
+ ASSERT_EQ(Status::OK, kvs2.Delete(keys[1]));
kvs1.Disable();
kvs2.Disable();
@@ -1145,7 +1166,7 @@
// Reset KVS
kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
int value;
ASSERT_EQ(Status::OK, kvs.Get(keys[0], &value));
ASSERT_EQ(values2[0], value);
@@ -1153,7 +1174,7 @@
ASSERT_EQ(Status::OK, kvs.Get(keys[2], &value));
ASSERT_EQ(values1[2], value);
- if (test_partition.GetSectorCount() == 2) {
+ if (test_partition.sector_count() == 2) {
EXPECT_EQ(kvs.PendingCleanCount(), 0u);
// Has forced a clean, mark again for next test
return; // Not enough sectors to test 2 partial cleans.
@@ -1169,7 +1190,7 @@
sizeof(uint64_t)));
// Reset KVS
kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
EXPECT_EQ(kvs.PendingCleanCount(), 2u);
ASSERT_EQ(Status::OK, kvs.Get(keys[0], &value));
ASSERT_EQ(values1[0], value);
@@ -1177,12 +1198,13 @@
ASSERT_EQ(values1[1], value);
ASSERT_EQ(Status::OK, kvs.Get(keys[2], &value));
ASSERT_EQ(values1[2], value);
+#endif
}
-// TODO: temporary
+// TODO: This doesn't do anything, and would be unreliable anyway.
size_t CurrentTaskStackFree() { return -1; }
-TEST_F(KeyValueStoreTest, PartialClean) {
+TEST_F(KeyValueStoreTest, DISABLED_PartialClean) {
if (CurrentTaskStackFree() < sizeof(KeyValueStore) * 2) {
PW_LOG_ERROR("Not enough stack for test, skipping");
return;
@@ -1191,11 +1213,12 @@
}
void __attribute__((noinline)) StackHeavyCleanAll() {
- CHECK_GE(test_partition.GetSectorCount(), 2);
+#if 0 // TODO: no FlashSubPartition
+ ASSERT_GE(test_partition.sector_count(), 2);
FlashSubPartition test_partition_sector1(&test_partition, 0, 1);
KeyValueStore kvs1(&test_partition_sector1);
- test_partition.Erase(0, test_partition.GetSectorCount());
+ test_partition.Erase(0, test_partition.sector_count());
ASSERT_EQ(Status::OK, kvs1.Enable());
@@ -1217,7 +1240,7 @@
// Reset KVS
kvs.Disable();
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
int value;
EXPECT_EQ(kvs.PendingCleanCount(), 1u);
ASSERT_EQ(Status::OK, kvs.CleanAll());
@@ -1228,9 +1251,10 @@
ASSERT_EQ(values1[1], value);
ASSERT_EQ(Status::OK, kvs.Get(keys[2], &value));
ASSERT_EQ(values1[2], value);
+#endif
}
-TEST_F(KeyValueStoreTest, CleanAll) {
+TEST_F(KeyValueStoreTest, DISABLED_CleanAll) {
if (CurrentTaskStackFree() < sizeof(KeyValueStore) * 1) {
PW_LOG_ERROR("Not enough stack for test, skipping");
return;
@@ -1239,14 +1263,15 @@
}
void __attribute__((noinline)) StackHeavyPartialCleanLargeCounts() {
- CHECK_GE(test_partition.GetSectorCount(), 2);
+#if 0
+ ASSERT_GE(test_partition.sector_count(), 2);
FlashSubPartition test_partition_sector1(&test_partition, 0, 1);
FlashSubPartition test_partition_sector2(&test_partition, 1, 1);
KeyValueStore kvs1(&test_partition_sector1);
KeyValueStore kvs2(&test_partition_sector2);
- test_partition.Erase(0, test_partition.GetSectorCount());
+ test_partition.Erase(0, test_partition.sector_count());
ASSERT_EQ(Status::OK, kvs1.Enable());
ASSERT_EQ(Status::OK, kvs2.Enable());
@@ -1259,7 +1284,7 @@
int values2[3] = {200, 201, 202};
ASSERT_EQ(Status::OK, kvs2.Put(keys[0], values2[0]));
ASSERT_EQ(Status::OK, kvs2.Put(keys[1], values2[1]));
- ASSERT_EQ(Status::OK, kvs2.Erase(keys[1]));
+ ASSERT_EQ(Status::OK, kvs2.Delete(keys[1]));
kvs1.Disable();
kvs2.Disable();
@@ -1277,7 +1302,7 @@
sizeof(uint64_t)));
// Reset KVS
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
int value;
ASSERT_EQ(Status::OK, kvs.Get(keys[0], &value));
ASSERT_EQ(values2[0], value);
@@ -1285,7 +1310,7 @@
ASSERT_EQ(Status::OK, kvs.Get(keys[2], &value));
ASSERT_EQ(values1[2], value);
- if (test_partition.GetSectorCount() == 2) {
+ if (test_partition.sector_count() == 2) {
EXPECT_EQ(kvs.PendingCleanCount(), 0u);
// Has forced a clean, mark again for next test
// Has forced a clean, mark again for next test
@@ -1302,7 +1327,7 @@
reinterpret_cast<uint8_t*>(&mark_clean_count),
sizeof(uint64_t)));
// Reset KVS
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
EXPECT_EQ(kvs.PendingCleanCount(), 2u);
ASSERT_EQ(Status::OK, kvs.Get(keys[0], &value));
ASSERT_EQ(values1[0], value);
@@ -1310,9 +1335,10 @@
ASSERT_EQ(values1[1], value);
ASSERT_EQ(Status::OK, kvs.Get(keys[2], &value));
ASSERT_EQ(values1[2], value);
+#endif
}
-TEST_F(KeyValueStoreTest, PartialCleanLargeCounts) {
+TEST_F(KeyValueStoreTest, DISABLED_PartialCleanLargeCounts) {
if (CurrentTaskStackFree() < sizeof(KeyValueStore) * 2) {
PW_LOG_ERROR("Not enough stack for test, skipping");
return;
@@ -1321,7 +1347,8 @@
}
void __attribute__((noinline)) StackHeavyRecoverNoFreeSectors() {
- CHECK_GE(test_partition.GetSectorCount(), 2);
+#if 0 // TODO: no FlashSubPartition
+ ASSERT_GE(test_partition.sector_count(), 2);
FlashSubPartition test_partition_sector1(&test_partition, 0, 1);
FlashSubPartition test_partition_sector2(&test_partition, 1, 1);
FlashSubPartition test_partition_both(&test_partition, 0, 2);
@@ -1330,7 +1357,7 @@
KeyValueStore kvs2(&test_partition_sector2);
KeyValueStore kvs_both(&test_partition_both);
- test_partition.Erase(0, test_partition.GetSectorCount());
+ test_partition.Erase(0, test_partition.sector_count());
ASSERT_EQ(Status::OK, kvs1.Enable());
ASSERT_EQ(Status::OK, kvs2.Enable());
@@ -1352,6 +1379,7 @@
ASSERT_EQ(values[0], value);
ASSERT_EQ(Status::OK, kvs_both.Get(keys[1], &value));
ASSERT_EQ(values[1], value);
+#endif
}
TEST_F(KeyValueStoreTest, RecoverNoFreeSectors) {
@@ -1363,13 +1391,14 @@
}
void __attribute__((noinline)) StackHeavyCleanOneSector() {
- CHECK_GE(test_partition.GetSectorCount(), 2);
+#if 0 // TODO: no FlashSubPartition
+ ASSERT_GE(test_partition.sector_count(), 2);
FlashSubPartition test_partition_sector1(&test_partition, 0, 1);
FlashSubPartition test_partition_sector2(&test_partition, 1, 1);
KeyValueStore kvs1(&test_partition_sector1);
- test_partition.Erase(0, test_partition.GetSectorCount());
+ test_partition.Erase(0, test_partition.sector_count());
ASSERT_EQ(Status::OK, kvs1.Enable());
@@ -1389,7 +1418,7 @@
sizeof(uint64_t)));
// Reset KVS
- ASSERT_EQ(Status::OK, kvs.Enable());
+ ASSERT_EQ(Status::OK, kvs.Init());
EXPECT_EQ(kvs.PendingCleanCount(), 1u);
@@ -1406,9 +1435,10 @@
ASSERT_EQ(values[1], value);
ASSERT_EQ(Status::OK, kvs.Get(keys[2], &value));
ASSERT_EQ(values[2], value);
+#endif
}
-TEST_F(KeyValueStoreTest, CleanOneSector) {
+TEST_F(KeyValueStoreTest, DISABLED_CleanOneSector) {
if (CurrentTaskStackFree() < sizeof(KeyValueStore)) {
PW_LOG_ERROR("Not enough stack for test, skipping");
return;
@@ -1418,31 +1448,30 @@
#if USE_MEMORY_BUFFER
-TEST_F(KeyValueStoreTest, LargePartition) {
+TEST_F(KeyValueStoreTest, DISABLED_LargePartition) {
if (CurrentTaskStackFree() < sizeof(KeyValueStore)) {
PW_LOG_ERROR("Not enough stack for test, skipping");
return;
}
- large_test_partition.Erase(0, large_test_partition.GetSectorCount());
- KeyValueStore large_kvs(&large_test_partition);
+ large_test_partition.Erase(0, large_test_partition.sector_count());
+ KeyValueStore large_kvs(&large_test_partition, format);
// Reset KVS
- large_kvs.Disable();
- ASSERT_EQ(Status::OK, large_kvs.Enable());
+ ASSERT_EQ(Status::OK, large_kvs.Init());
const uint8_t kValue1 = 0xDA;
const uint8_t kValue2 = 0x12;
uint8_t value;
ASSERT_EQ(Status::OK, large_kvs.Put(keys[0], kValue1));
- EXPECT_EQ(large_kvs.KeyCount(), 1u);
- ASSERT_EQ(Status::OK, large_kvs.Erase(keys[0]));
+ EXPECT_EQ(large_kvs.size(), 1u);
+ ASSERT_EQ(Status::OK, large_kvs.Delete(keys[0]));
EXPECT_EQ(large_kvs.Get(keys[0], &value), Status::NOT_FOUND);
ASSERT_EQ(Status::OK, large_kvs.Put(keys[1], kValue1));
ASSERT_EQ(Status::OK, large_kvs.Put(keys[2], kValue2));
- ASSERT_EQ(Status::OK, large_kvs.Erase(keys[1]));
+ ASSERT_EQ(Status::OK, large_kvs.Delete(keys[1]));
EXPECT_EQ(Status::OK, large_kvs.Get(keys[2], &value));
EXPECT_EQ(kValue2, value);
ASSERT_EQ(large_kvs.Get(keys[1], &value), Status::NOT_FOUND);
- EXPECT_EQ(large_kvs.KeyCount(), 1u);
+ EXPECT_EQ(large_kvs.size(), 1u);
}
#endif // USE_MEMORY_BUFFER
diff --git a/pw_kvs/public/pw_kvs/assert.h b/pw_kvs/public/pw_kvs/assert.h
deleted file mode 100644
index cc20a61..0000000
--- a/pw_kvs/public/pw_kvs/assert.h
+++ /dev/null
@@ -1,69 +0,0 @@
-// Copyright 2020 The Pigweed Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not
-// use this file except in compliance with the License. You may obtain a copy of
-// the License at
-//
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-// License for the specific language governing permissions and limitations under
-// the License.
-#pragma once
-
-#include <algorithm>
-#include <type_traits>
-
-// Compares the provided value to nullptr and returns it. This is intended to be
-// used as part of another statement.
-#define CHECK_NOTNULL(value) \
- ::pw::log::CheckNotNull("", __LINE__, #value " != nullptr", value)
-
-// In release builds, DCHECK_NOTNULL simply passes along the value.
-// DCHECK_NOTNULL must not be used as a standalone expression, since the result
-// would be unused on release builds. Use DCHECK_NE instead.
-//#define DCHECK_NOTNULL(value) value
-
-#define DCHECK_NOTNULL(value) \
- ::pw::log::DCheckNotNull("", __LINE__, #value " != nullptr", value)
-
-namespace pw::log {
-
-template <typename T>
-constexpr T CheckNotNull(const char* /* file */,
- unsigned /* line */,
- const char* /* message */,
- T&& value) {
- static_assert(!std::is_null_pointer<T>(),
- "CHECK_NOTNULL statements cannot be passed nullptr");
- if (value == nullptr) {
- // std::exit(1);
- }
- return std::forward<T>(value);
-}
-
-// DCHECK_NOTNULL cannot be used in standalone expressions, so add a
-// [[nodiscard]] attribute to prevent this in debug builds. Standalone
-// DCHECK_NOTNULL statements in release builds trigger an unused-value warning.
-template <typename T>
-[[nodiscard]] constexpr T DCheckNotNull(const char* file,
- unsigned line,
- const char* message,
- T&& value) {
- return CheckNotNull<T>(file, line, message, std::forward<T>(value));
-}
-
-} // namespace pw::log
-
-// Assert stubs
-#define DCHECK CHECK
-#define DCHECK_EQ CHECK_EQ
-
-#define CHECK(...)
-#define CHECK_EQ(...)
-#define CHECK_GE(...)
-#define CHECK_GT(...)
-#define CHECK_LE(...)
-#define CHECK_LT(...)
diff --git a/pw_kvs/public/pw_kvs/checksum.h b/pw_kvs/public/pw_kvs/checksum.h
new file mode 100644
index 0000000..f7b01a1
--- /dev/null
+++ b/pw_kvs/public/pw_kvs/checksum.h
@@ -0,0 +1,57 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+#pragma once
+
+#include <cstddef>
+
+#include "pw_span/span.h"
+#include "pw_status/status.h"
+
+namespace pw::kvs {
+
+class ChecksumAlgorithm {
+ public:
+ // Resets the checksum to its initial state.
+ virtual void Reset() = 0;
+
+ // Updates the checksum with the provided data.
+ virtual Status Update(span<const std::byte> data_to_checksum) = 0;
+
+ // Convnenience wrapper.
+ Status Update(const void* data, size_t size) {
+ return Update(span(static_cast<const std::byte*>(data), size));
+ }
+
+ // Returns the current state of the checksum algorithm.
+ constexpr const span<const std::byte>& state() const { return state_; }
+
+ // Returns the size of the checksum's state.
+ constexpr size_t size_bytes() const { return state_.size(); }
+
+ // Compares a calculated checksum to this checksum's data.
+ Status Verify(span<const std::byte> calculated_checksum) const;
+
+ protected:
+ // Derived class provides a span of its state buffer.
+ constexpr ChecksumAlgorithm(span<const std::byte> state) : state_(state) {}
+
+ // Protected destructor prevents deleting ChecksumAlgorithms from the base
+ // class, so that it is safe to have a non-virtual destructor.
+ ~ChecksumAlgorithm() = default;
+
+ private:
+ span<const std::byte> state_;
+};
+
+} // namespace pw::kvs
diff --git a/pw_kvs/public/pw_kvs/flash.h b/pw_kvs/public/pw_kvs/flash.h
deleted file mode 100644
index 819f8e2..0000000
--- a/pw_kvs/public/pw_kvs/flash.h
+++ /dev/null
@@ -1,33 +0,0 @@
-// Copyright 2020 The Pigweed Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not
-// use this file except in compliance with the License. You may obtain a copy of
-// the License at
-//
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-// License for the specific language governing permissions and limitations under
-// the License.
-#pragma once
-
-#include "pw_kvs/flash_memory.h"
-
-namespace pw::kvs {
-
-// Writes a buffer which is not guaranteed to be aligned, pads remaining
-// bytes with 0.
-Status PaddedWrite(FlashPartition* partition,
- FlashPartition::Address address,
- const void* buffer,
- uint16_t size);
-
-// Read into a buffer when size is not guaranteed to be aligned.
-Status UnalignedRead(FlashPartition* partition,
- void* buffer,
- FlashPartition::Address address,
- uint16_t size);
-
-} // namespace pw::kvs
diff --git a/pw_kvs/public/pw_kvs/flash_memory.h b/pw_kvs/public/pw_kvs/flash_memory.h
index b69eeb2..5495141 100644
--- a/pw_kvs/public/pw_kvs/flash_memory.h
+++ b/pw_kvs/public/pw_kvs/flash_memory.h
@@ -13,32 +13,50 @@
// the License.
#pragma once
-#include <algorithm>
-#include <cinttypes>
-#include <cstring>
+#include <cstddef>
+#include <cstdint>
+#include <initializer_list>
-#include "pw_kvs/assert.h"
-#include "pw_kvs/partition_table_entry.h"
-#include "pw_log/log.h"
+#include "pw_span/span.h"
#include "pw_status/status.h"
+#include "pw_status/status_with_size.h"
+
+namespace pw {
+
+// TODO: These are general-purpose utility functions that should be moved
+// elsewhere.
+constexpr size_t AlignDown(size_t value, size_t alignment) {
+ return (value / alignment) * alignment;
+}
+
+constexpr size_t AlignUp(size_t value, size_t alignment) {
+ return (value + alignment - 1) / alignment * alignment;
+}
+
+} // namespace pw
namespace pw::kvs {
+enum class PartitionPermission : bool {
+ kReadOnly,
+ kReadAndWrite,
+};
+
class FlashMemory {
public:
// The flash address is in the range of: 0 to FlashSize.
typedef uint32_t Address;
- constexpr FlashMemory(uint32_t sector_size,
- uint32_t sector_count,
- uint8_t alignment,
+ constexpr FlashMemory(size_t sector_size,
+ size_t sector_count,
+ size_t alignment,
uint32_t start_address = 0,
uint32_t sector_start = 0,
- uint8_t erased_memory_content = 0xFF)
+ std::byte erased_memory_content = std::byte{0xFF})
: sector_size_(sector_size),
flash_sector_count_(sector_count),
alignment_(alignment),
start_address_(start_address),
- sector_start_(sector_start),
+ start_sector_(sector_start),
erased_memory_content_(erased_memory_content) {}
virtual ~FlashMemory() = default;
@@ -54,43 +72,40 @@
// TIMEOUT, on timeout.
// INVALID_ARGUMENT, if address or sector count is invalid.
// UNKNOWN, on HAL error
- virtual Status Erase(Address flash_address, uint32_t num_sectors) = 0;
+ virtual Status Erase(Address flash_address, size_t num_sectors) = 0;
// Reads bytes from flash into buffer. Blocking call.
// Returns: OK, on success.
// TIMEOUT, on timeout.
// INVALID_ARGUMENT, if address or length is invalid.
// UNKNOWN, on HAL error
- virtual Status Read(uint8_t* destination_ram_address,
- Address source_flash_address,
- uint32_t len) = 0;
+ virtual StatusWithSize Read(Address address, span<std::byte> output) = 0;
// Writes bytes to flash. Blocking call.
// Returns: OK, on success.
// TIMEOUT, on timeout.
// INVALID_ARGUMENT, if address or length is invalid.
// UNKNOWN, on HAL error
- virtual Status Write(Address destination_flash_address,
- const uint8_t* source_ram_address,
- uint32_t len) = 0;
+ virtual StatusWithSize Write(Address destination_flash_address,
+ span<const std::byte> data) = 0;
// Convert an Address to an MCU pointer, this can be used for memory
// mapped reads. Return NULL if the memory is not memory mapped.
- virtual uint8_t* FlashAddressToMcuAddress(Address) const { return nullptr; }
+ virtual std::byte* FlashAddressToMcuAddress(Address) const { return nullptr; }
- // GetStartSector() is useful for FlashMemory instances where the
+ // start_sector() is useful for FlashMemory instances where the
// sector start is not 0. (ex.: cases where there are portions of flash
// that should be handled independently).
- constexpr uint32_t GetStartSector() const { return sector_start_; }
- constexpr uint32_t GetSectorSizeBytes() const { return sector_size_; }
- constexpr uint32_t GetSectorCount() const { return flash_sector_count_; }
- constexpr uint8_t GetAlignmentBytes() const { return alignment_; }
- constexpr uint32_t GetSizeBytes() const {
+ constexpr uint32_t start_sector() const { return start_sector_; }
+ constexpr size_t sector_size_bytes() const { return sector_size_; }
+ constexpr size_t sector_count() const { return flash_sector_count_; }
+ constexpr size_t alignment_bytes() const { return alignment_; }
+ constexpr size_t size_bytes() const {
return sector_size_ * flash_sector_count_;
}
// Address of the start of flash (the address of sector 0)
- constexpr uint32_t GetStartAddress() const { return start_address_; }
- constexpr uint8_t GetErasedMemoryContent() const {
+ constexpr uint32_t start_address() const { return start_address_; }
+ constexpr std::byte erased_memory_content() const {
return erased_memory_content_;
}
@@ -99,14 +114,14 @@
const uint32_t flash_sector_count_;
const uint8_t alignment_;
const uint32_t start_address_;
- const uint32_t sector_start_;
- const uint8_t erased_memory_content_;
+ const uint32_t start_sector_;
+ const std::byte erased_memory_content_;
};
class FlashPartition {
public:
// The flash address is in the range of: 0 to PartitionSize.
- typedef uint32_t Address;
+ using Address = uint32_t;
constexpr FlashPartition(
FlashMemory* flash,
@@ -118,12 +133,17 @@
sector_count_(sector_count),
permission_(permission) {}
- constexpr FlashPartition(FlashMemory* flash, PartitionTableEntry entry)
+#if 0
+ constexpr FlashPartition(
+ FlashMemory* flash,
+ uint32_t start_sector_index,
+ uint32_t end_sector_index,
+ PartitionPermission permission = PartitionPermission::kReadAndWrite)
: flash_(*flash),
- start_sector_index_(entry.partition_start_sector_index),
- sector_count_(entry.partition_end_sector_index -
- entry.partition_start_sector_index + 1),
- permission_(entry.partition_permission) {}
+ start_sector_index_(start_sector_index),
+ sector_count_(end_sector_index - start_sector_index + 1),
+ permission_(permission) {}
+#endif
virtual ~FlashPartition() = default;
@@ -134,32 +154,17 @@
// INVALID_ARGUMENT, if address or sector count is invalid.
// PERMISSION_DENIED, if partition is read only.
// UNKNOWN, on HAL error
- virtual Status Erase(Address address, uint32_t num_sectors) {
- if (permission_ == PartitionPermission::kReadOnly) {
- return Status::PERMISSION_DENIED;
- }
- if (Status status =
- CheckBounds(address, num_sectors * GetSectorSizeBytes());
- !status.ok()) {
- return status;
- }
- return flash_.Erase(PartitionToFlashAddress(address), num_sectors);
- }
+ virtual Status Erase(Address address, size_t num_sectors);
// Reads bytes from flash into buffer. Blocking call.
// Returns: OK, on success.
// TIMEOUT, on timeout.
// INVALID_ARGUMENT, if address or length is invalid.
// UNKNOWN, on HAL error
- virtual Status Read(uint8_t* destination_ram_address,
- Address source_flash_address,
- uint32_t len) {
- if (Status status = CheckBounds(source_flash_address, len); !status.ok()) {
- return status;
- }
- return flash_.Read(destination_ram_address,
- PartitionToFlashAddress(source_flash_address),
- len);
+ virtual StatusWithSize Read(Address address, span<std::byte> output);
+
+ StatusWithSize Read(Address address, size_t length, void* output) {
+ return Read(address, span(static_cast<std::byte*>(output), length));
}
// Writes bytes to flash. Blocking call.
@@ -168,20 +173,10 @@
// INVALID_ARGUMENT, if address or length is invalid.
// PERMISSION_DENIED, if partition is read only.
// UNKNOWN, on HAL error
- virtual Status Write(Address destination_flash_address,
- const uint8_t* source_ram_address,
- uint32_t len) {
- if (permission_ == PartitionPermission::kReadOnly) {
- return Status::PERMISSION_DENIED;
- }
- if (Status status = CheckBounds(destination_flash_address, len);
- !status.ok()) {
- return status;
- }
- return flash_.Write(PartitionToFlashAddress(destination_flash_address),
- source_ram_address,
- len);
- }
+ virtual StatusWithSize Write(Address address, span<const std::byte> data);
+
+ StatusWithSize Write(Address start_address,
+ std::initializer_list<span<const std::byte>> data);
// Check to see if chunk of flash memory is erased. Address and len need to
// be aligned with FlashMemory.
@@ -189,87 +184,40 @@
// TIMEOUT, on timeout.
// INVALID_ARGUMENT, if address or length is invalid.
// UNKNOWN, on HAL error
- virtual Status IsChunkErased(Address source_flash_address,
- uint32_t len,
- bool* is_erased) {
- // Max alignment is artifical to keep the stack usage low for this
- // function. Using 16 because it's the alignment of encrypted flash.
- const uint8_t kMaxAlignment = 16;
- // Relying on Read() to check address and len arguments.
- if (!is_erased) {
- return Status::INVALID_ARGUMENT;
- }
- uint8_t alignment = GetAlignmentBytes();
- if (alignment > kMaxAlignment || kMaxAlignment % alignment ||
- len % alignment) {
- return Status::INVALID_ARGUMENT;
- }
+ // TODO: StatusWithBool
+ virtual Status IsRegionErased(Address source_flash_address,
+ size_t len,
+ bool* is_erased);
- uint8_t buffer[kMaxAlignment];
- uint8_t erased_pattern_buffer[kMaxAlignment];
- size_t offset = 0;
- std::memset(erased_pattern_buffer,
- flash_.GetErasedMemoryContent(),
- sizeof(erased_pattern_buffer));
- *is_erased = false;
- while (len > 0) {
- // Check earlier that len is aligned, no need to round up
- uint16_t read_size = std::min(static_cast<uint32_t>(sizeof(buffer)), len);
- if (Status status =
- Read(buffer, source_flash_address + offset, read_size);
- !status.ok()) {
- return status;
- }
- if (std::memcmp(buffer, erased_pattern_buffer, read_size)) {
- // Detected memory chunk is not entirely erased
- return Status::OK;
- }
- offset += read_size;
- len -= read_size;
- }
- *is_erased = true;
- return Status::OK;
+ constexpr uint32_t sector_size_bytes() const {
+ return flash_.sector_size_bytes();
}
- constexpr uint32_t GetSectorSizeBytes() const {
- return flash_.GetSectorSizeBytes();
+ // Overridden by base classes which store metadata at the start of a sector.
+ virtual uint32_t sector_available_size_bytes() const {
+ return sector_size_bytes();
}
- uint32_t GetSizeBytes() const {
- return GetSectorCount() * GetSectorSizeBytes();
- }
+ size_t size_bytes() const { return sector_count() * sector_size_bytes(); }
- virtual uint8_t GetAlignmentBytes() const {
- return flash_.GetAlignmentBytes();
- }
+ virtual size_t alignment_bytes() const { return flash_.alignment_bytes(); }
- virtual uint32_t GetSectorCount() const { return sector_count_; }
+ virtual size_t sector_count() const { return sector_count_; }
// Convert a FlashMemory::Address to an MCU pointer, this can be used for
// memory mapped reads. Return NULL if the memory is not memory mapped.
- uint8_t* PartitionAddressToMcuAddress(Address address) const {
+ std::byte* PartitionAddressToMcuAddress(Address address) const {
return flash_.FlashAddressToMcuAddress(PartitionToFlashAddress(address));
}
FlashMemory::Address PartitionToFlashAddress(Address address) const {
- return flash_.GetStartAddress() +
- (start_sector_index_ - flash_.GetStartSector()) *
- GetSectorSizeBytes() +
+ return flash_.start_address() +
+ (start_sector_index_ - flash_.start_sector()) * sector_size_bytes() +
address;
}
protected:
- Status CheckBounds(Address address, size_t len) const {
- if (address + len > GetSizeBytes()) {
- PW_LOG_ERROR(
- "Attempted out-of-bound flash memory access (address: %" PRIu32
- " length: %zu)",
- address,
- len);
- return Status::INVALID_ARGUMENT;
- }
- return Status::OK;
- }
+ Status CheckBounds(Address address, size_t len) const;
private:
FlashMemory& flash_;
@@ -278,74 +226,4 @@
const PartitionPermission permission_;
};
-// FlashSubPartition defines a new partition which maps itself as a smaller
-// piece of another partition. This can used when a partition has special
-// behaviours (for example encrypted flash).
-// For example, this will be the first sector of test_partition:
-// FlashSubPartition test_partition_sector1(&test_partition, 0, 1);
-class FlashSubPartition : public FlashPartition {
- public:
- constexpr FlashSubPartition(FlashPartition* parent_partition,
- uint32_t start_sector_index,
- uint32_t sector_count)
- : FlashPartition(*parent_partition),
- partition_(parent_partition),
- start_sector_index_(start_sector_index),
- sector_count_(sector_count) {}
-
- Status Erase(Address address, uint32_t num_sectors) override {
- if (Status status =
- CheckBounds(address, num_sectors * GetSectorSizeBytes());
- !status.ok()) {
- return status;
- }
- return partition_->Erase(ParentAddress(address), num_sectors);
- }
-
- Status Read(uint8_t* destination_ram_address,
- Address source_flash_address,
- uint32_t len) override {
- if (Status status = CheckBounds(source_flash_address, len); !status.ok()) {
- return status;
- }
- return partition_->Read(
- destination_ram_address, ParentAddress(source_flash_address), len);
- }
-
- Status Write(Address destination_flash_address,
- const uint8_t* source_ram_address,
- uint32_t len) override {
- if (Status status = CheckBounds(destination_flash_address, len);
- !status.ok()) {
- return status;
- }
- return partition_->Write(
- ParentAddress(destination_flash_address), source_ram_address, len);
- }
-
- Status IsChunkErased(Address source_flash_address,
- uint32_t len,
- bool* is_erased) override {
- if (Status status = CheckBounds(source_flash_address, len); !status.ok()) {
- return status;
- }
- return partition_->IsChunkErased(
- ParentAddress(source_flash_address), len, is_erased);
- }
-
- uint8_t GetAlignmentBytes() const override {
- return partition_->GetAlignmentBytes();
- }
-
- uint32_t GetSectorCount() const override { return sector_count_; }
-
- private:
- Address ParentAddress(Address address) const {
- return address + start_sector_index_ * partition_->GetSectorSizeBytes();
- }
- FlashPartition* partition_;
- const uint32_t start_sector_index_;
- const uint32_t sector_count_;
-};
-
} // namespace pw::kvs
diff --git a/pw_kvs/public/pw_kvs/in_memory_fake_flash.h b/pw_kvs/public/pw_kvs/in_memory_fake_flash.h
index d2ea67c..901684e 100644
--- a/pw_kvs/public/pw_kvs/in_memory_fake_flash.h
+++ b/pw_kvs/public/pw_kvs/in_memory_fake_flash.h
@@ -14,6 +14,7 @@
#pragma once
#include <array>
+#include <cstring>
#include "pw_kvs/flash_memory.h"
#include "pw_status/status.h"
@@ -38,17 +39,17 @@
// Returns: OK, on success.
// INVALID_ARGUMENT, if address or sector count is invalid.
// UNKNOWN, on HAL error
- Status Erase(Address addr, uint32_t num_sectors) override {
- if (addr % GetSectorSizeBytes() != 0) {
+ Status Erase(Address addr, size_t num_sectors) override {
+ if (addr % sector_size_bytes() != 0) {
return Status::INVALID_ARGUMENT;
}
- if (addr / GetSectorSizeBytes() + num_sectors > GetSectorCount()) {
+ if (addr / sector_size_bytes() + num_sectors > sector_count()) {
return Status::UNKNOWN;
}
- if (addr % GetAlignmentBytes() != 0) {
+ if (addr % alignment_bytes() != 0) {
return Status::INVALID_ARGUMENT;
}
- memset(&buffer_[addr], 0xFF, GetSectorSizeBytes() * num_sectors);
+ std::memset(&buffer_[addr], 0xFF, sector_size_bytes() * num_sectors);
return Status::OK;
}
@@ -56,13 +57,11 @@
// Returns: OK, on success.
// INVALID_ARGUMENT, if address or length is invalid.
// UNKNOWN, on HAL error
- Status Read(uint8_t* dest_ram_addr,
- Address source_flash_addr,
- uint32_t len) override {
- if ((source_flash_addr + len) >= GetSectorCount() * GetSizeBytes()) {
+ StatusWithSize Read(Address address, span<std::byte> output) override {
+ if (address + output.size() >= sector_count() * size_bytes()) {
return Status::INVALID_ARGUMENT;
}
- memcpy(dest_ram_addr, &buffer_[source_flash_addr], len);
+ std::memcpy(output.data(), &buffer_[address], output.size());
return Status::OK;
}
@@ -70,22 +69,20 @@
// Returns: OK, on success.
// INVALID_ARGUMENT, if address or length is invalid.
// UNKNOWN, on HAL error
- Status Write(Address dest_flash_addr,
- const uint8_t* source_ram_addr,
- uint32_t len) override {
- if ((dest_flash_addr + len) >= GetSectorCount() * GetSizeBytes() ||
- dest_flash_addr % GetAlignmentBytes() != 0 ||
- len % GetAlignmentBytes() != 0) {
+ StatusWithSize Write(Address address, span<const std::byte> data) override {
+ if ((address + data.size()) >= sector_count() * size_bytes() ||
+ address % alignment_bytes() != 0 ||
+ data.size() % alignment_bytes() != 0) {
return Status::INVALID_ARGUMENT;
}
// Check in erased state
- for (unsigned i = 0; i < len; i++) {
- if (buffer_[dest_flash_addr + i] != 0xFF) {
+ for (unsigned i = 0; i < data.size(); i++) {
+ if (buffer_[address + i] != 0xFF) {
return Status::UNKNOWN;
}
}
- memcpy(&buffer_[dest_flash_addr], source_ram_addr, len);
- return Status::OK;
+ std::memcpy(&buffer_[address], data.data(), data.size());
+ return StatusWithSize(data.size());
}
private:
diff --git a/pw_kvs/public/pw_kvs/key_value_store.h b/pw_kvs/public/pw_kvs/key_value_store.h
index 2faa512..115bea0 100644
--- a/pw_kvs/public/pw_kvs/key_value_store.h
+++ b/pw_kvs/public/pw_kvs/key_value_store.h
@@ -13,402 +13,301 @@
// the License.
#pragma once
+#include <array>
#include <cstddef>
#include <cstdint>
-#include <limits>
#include <string_view>
-#include <type_traits>
+#include "pw_kvs/checksum.h"
#include "pw_kvs/flash_memory.h"
#include "pw_span/span.h"
#include "pw_status/status.h"
#include "pw_status/status_with_size.h"
namespace pw::kvs {
-namespace cfg {
-
-// KVS requires a temporary buffer for some operations, this config allows
-// tuning the buffer size. This is a trade-off between a value which is large
-// and therefore requires more RAM, or having a value which is small which will
-// result in some operations taking longer, as the operations are broken into
-// smaller chunks.
-// NOTE: This value can not be smaller then the flash alignment, and it will
-// round the size down to be a multiple of the flash alignment for all
-// operations.
-inline constexpr size_t kKvsBufferSize = 64;
-
-// This represents the maximum amount of keys which can be in the KVS at any
-// given time.
-inline constexpr uint8_t kKvsMaxKeyCount = 50;
-
-// This is the maximum amount of sectors the KVS can operate on, an invalid
-// value will cause an error during enable.
-inline constexpr uint32_t kKvsMaxSectorCount = 20;
-
-} // namespace cfg
-
namespace internal {
+template <typename T, typename = decltype(span(std::declval<T>()))>
+constexpr bool ConvertsToSpan(int) {
+ return true;
+}
+
+// If the expression span(T) fails, then the type can't be converted to a span.
+template <typename T>
+constexpr bool ConvertsToSpan(...) {
+ return false;
+}
+
+} // namespace internal
+
// Traits class to detect if the type is a span. std::is_same is insufficient
// because span is a class template. This is used to ensure that the correct
// overload of the Put function is selected.
-template <typename>
-struct IsSpan : std::false_type {};
+template <typename T>
+using ConvertsToSpan =
+ std::bool_constant<internal::ConvertsToSpan<std::remove_reference_t<T>>(0)>;
-template <typename T, size_t kExtent>
-struct IsSpan<span<T, kExtent>> : std::true_type {};
+// Internal-only persistent storage header format.
+struct EntryHeader;
-} // namespace internal
+struct EntryHeaderFormat {
+ uint32_t magic; // unique identifier
+ ChecksumAlgorithm* checksum;
+};
-// This object is very large (can be over 1500 B, depending on configuration)
-// and should not typically be placed on the stack.
+// TODO: Select the appropriate defaults, add descriptions.
+struct Options {
+ bool partial_gc_on_write = true;
+ bool verify_on_read = true;
+ bool verify_on_write = true;
+};
+
class KeyValueStore {
public:
- constexpr KeyValueStore(FlashPartition* partition) : partition_(*partition) {}
+ // TODO: Make these configurable
+ static constexpr size_t kMaxKeyLength = 64;
+ static constexpr size_t kMaxEntries = 64;
+ static constexpr size_t kUsableSectors = 64;
- KeyValueStore(const KeyValueStore&) = delete;
- KeyValueStore& operator=(const KeyValueStore&) = delete;
+ // In the future, will be able to provide additional EntryHeaderFormats for
+ // backwards compatibility.
+ constexpr KeyValueStore(FlashPartition* partition,
+ const EntryHeaderFormat& format,
+ const Options& options = {})
+ : partition_(*partition),
+ entry_header_format_(format),
+ options_(options),
+ key_map_{},
+ key_map_size_(0),
+ sector_map_{},
+ last_written_sector_(0) {}
- // Enable the KVS, scans the sectors of the partition for any current KVS
- // data. Erases and initializes any sectors which are not initialized.
- // Checks the CRC of all data in the KVS; on failure the corrupted data is
- // lost and Enable will return Status::DATA_LOSS, but the KVS will still load
- // other data and will be enabled.
- Status Enable();
+ Status Init();
+ bool initialized() const { return false; } // TODO: Implement this
- bool IsEnabled() const { return enabled_; }
+ StatusWithSize Get(std::string_view key, span<std::byte> value) const;
- void Disable() {
- if (enabled_ == false) {
- return;
- }
- // TODO: LOCK MUTEX
- enabled_ = false;
- }
-
- // Erase a key value element.
- // key is a null terminated c string.
- // Returns OK if erased
- // NOT_FOUND if key was not found.
- Status Erase(const std::string_view& key);
-
- // Copy the first size bytes of key's value into value buffer.
- // key is a null terminated c string. Size is the amount of bytes to read,
- // Optional offset argument supports reading size bytes starting at the
- // specified offset.
- // Returns OK if successful
- // NOT_FOUND if key was not found
- // DATA_LOSS if the value failed crc check
- Status Get(const std::string_view& key,
- const span<std::byte>& value,
- uint16_t offset = 0);
-
- // Interpret the key's value as the given type and return it.
- // Returns OK if successful
- // NOT_FOUND if key was not found
- // DATA_LOSS if the value failed crc check
- // INVALID_ARGUMENT if size of value != sizeof(T)
template <typename T>
Status Get(const std::string_view& key, T* value) {
static_assert(std::is_trivially_copyable<T>(), "KVS values must copyable");
static_assert(!std::is_pointer<T>(), "KVS values cannot be pointers");
- StatusWithSize result = GetValueSize(key);
+ StatusWithSize result = ValueSize(key);
if (!result.ok()) {
return result.status();
}
if (result.size() != sizeof(T)) {
return Status::INVALID_ARGUMENT;
}
- return Get(key, as_writable_bytes(span(value, 1)));
+ return Get(key, as_writable_bytes(span(value, 1))).status();
}
- // Writes the key value store to the partition. If the key already exists it
- // will be deleted before writing the new value.
- //
- // Returns OK if successful
- // RESOURCE_EXHAUSTED if there is not enough available space
- Status Put(const std::string_view& key, const span<const std::byte>& value);
+ Status Put(std::string_view key, span<const std::byte> value);
- // Alternate Put function that takes an object. The object must be trivially
- // copyable and cannot be a pointer or span.
template <typename T,
typename = std::enable_if_t<std::is_trivially_copyable_v<T> &&
!std::is_pointer_v<T> &&
- !internal::IsSpan<T>()>>
+ !ConvertsToSpan<T>::value>>
Status Put(const std::string_view& key, const T& value) {
return Put(key, as_bytes(span(&value, 1)));
}
- // Gets the size of the value stored for provided key.
- // Returns OK if successful
- // INVALID_ARGUMENT if args are invalid.
- // FAILED_PRECONDITION if KVS is not enabled.
- // NOT_FOUND if key was not found.
- StatusWithSize GetValueSize(const std::string_view& key);
+ Status Delete(std::string_view key);
- // Tests if the proposed key value entry can be stored in the KVS.
- bool CanFitEntry(uint16_t key_len, uint16_t value_len) {
- return kSectorInvalid != FindSpace(ChunkSize(key_len, value_len));
+ StatusWithSize ValueSize(std::string_view key) const {
+ (void)key;
+ return Status::UNIMPLEMENTED;
}
- // CleanAll cleans each sector which is currently marked for cleaning.
- // Note: if any data is invalid/corrupt it could be lost.
- Status CleanAll() {
- // TODO: LOCK MUTEX
- return CleanAllInternal();
- }
- size_t PendingCleanCount() {
- // TODO: LOCK MUTEX
- size_t ret = 0;
- for (size_t i = 0; i < SectorCount(); i++) {
- ret += sector_space_remaining_[i] == 0 ? 1 : 0;
+ // Classes and functions to support STL-style iteration.
+ class Iterator;
+
+ class Entry {
+ public:
+ // Guaranteed to be null-terminated
+ std::string_view key() const { return key_buffer_.data(); }
+
+ Status Get(span<std::byte> value_buffer) const {
+ return kvs_.Get(key(), value_buffer).status();
}
- return ret;
- }
- // Clean a single sector and return, if all sectors are clean it will set
- // all_sectors_have_been_cleaned to true and return.
- Status CleanOneSector(bool* all_sectors_have_been_cleaned);
+ template <typename T>
+ Status Get(T* value) const {
+ return kvs_.Get(key(), value);
+ }
- // For debugging, logging, and testing. (Don't use in regular code)
- // Note: a key_index is not valid after an element is erased or updated.
- size_t KeyCount() const;
- std::string_view GetKey(size_t idx) const;
- size_t GetValueSize(size_t idx) const;
- size_t GetMaxKeys() const { return kListCapacityMax; }
- bool HasEmptySector() const { return HasEmptySectorImpl(kSectorInvalid); }
+ StatusWithSize ValueSize() const { return kvs_.ValueSize(key()); }
- static constexpr size_t kHeaderSize = 8; // Sector and KVS Header size
- static constexpr uint16_t MaxValueLength() { return kChunkValueLengthMax; }
+ private:
+ friend class Iterator;
+
+ constexpr Entry(const KeyValueStore& kvs) : kvs_(kvs), key_buffer_{} {}
+
+ const KeyValueStore& kvs_;
+ std::array<char, kMaxKeyLength + 1> key_buffer_; // +1 for null-terminator
+ };
+
+ class Iterator {
+ public:
+ Iterator& operator++() {
+ index_ += 1;
+ return *this;
+ }
+
+ Iterator& operator++(int) { return operator++(); }
+
+ // Reads the entry's key from flash.
+ const Entry& operator*();
+
+ const Entry* operator->() {
+ operator*(); // Read the key into the Entry object.
+ return &entry_;
+ }
+
+ constexpr bool operator==(const Iterator& rhs) const {
+ return index_ == rhs.index_;
+ }
+
+ constexpr bool operator!=(const Iterator& rhs) const {
+ return index_ != rhs.index_;
+ }
+
+ private:
+ friend class KeyValueStore;
+
+ constexpr Iterator(const KeyValueStore& kvs, size_t index)
+ : entry_(kvs), index_(index) {}
+
+ Entry entry_;
+ size_t index_;
+ };
+
+ // Standard aliases for iterator types.
+ using iterator = Iterator;
+ using const_iterator = Iterator;
+
+ Iterator begin() const { return Iterator(*this, 0); }
+ Iterator end() const { return Iterator(*this, empty() ? 0 : size() - 1); }
+
+ // Returns the number of valid entries in the KeyValueStore.
+ size_t size() const { return key_map_size_; }
+
+ static constexpr size_t max_size() { return kMaxKeyLength; }
+
+ size_t empty() const { return size() == 0u; }
private:
- using KeyIndex = uint8_t;
- using SectorIndex = uint32_t;
+ using Address = FlashPartition::Address;
- static constexpr uint16_t kVersion = 1;
- static constexpr KeyIndex kListCapacityMax = cfg::kKvsMaxKeyCount;
- static constexpr SectorIndex kSectorCountMax = cfg::kKvsMaxSectorCount;
-
- // Number of bits for fields in KVSHeader:
- static constexpr int kChunkHeaderKeyFieldNumBits = 4;
- static constexpr int kChunkHeaderChunkFieldNumBits = 12;
-
- static constexpr uint16_t kSectorReadyValue = 0xABCD;
- static constexpr uint16_t kChunkSyncValue = 0x55AA;
-
- static constexpr uint16_t kChunkLengthMax =
- ((1 << kChunkHeaderChunkFieldNumBits) - 1);
- static constexpr uint16_t kChunkKeyLengthMax =
- ((1 << kChunkHeaderKeyFieldNumBits) - 1);
- static constexpr uint16_t kChunkValueLengthMax =
- (kChunkLengthMax - kChunkKeyLengthMax);
-
- static constexpr SectorIndex kSectorInvalid = 0xFFFFFFFFul;
- static constexpr FlashPartition::Address kAddressInvalid = 0xFFFFFFFFul;
- static constexpr uint64_t kSectorCleanNotPending = 0xFFFFFFFFFFFFFFFFull;
-
- static constexpr uint16_t kFlagsIsErasedMask = 0x0001;
- static constexpr uint16_t kMaxAlignmentBytes = 128;
-
- // This packs into 16 bytes.
- struct KvsSectorHeaderMeta {
- uint16_t synchronize_token;
- uint16_t version;
- uint16_t alignment_bytes; // alignment used for each chunk in this sector.
- uint16_t padding; // padding to support uint64_t alignment.
- };
- static_assert(sizeof(KvsSectorHeaderMeta) == kHeaderSize,
- "Invalid KvsSectorHeaderMeta size");
-
- // sector_clean_pending is broken off from KvsSectorHeaderMeta to support
- // larger than sizeof(KvsSectorHeaderMeta) flash write alignments.
- struct KvsSectorHeaderCleaning {
- // When this sector is not marked for cleaning this will be in the erased
- // state. When marked for clean this value indicates the order the sectors
- // were marked for cleaning, and therfore which data is the newest.
- // To remain backwards compatible with v2 and v3 KVS this is 8 bytes, if the
- // alignment is greater than 8 we will check the entire alignment is in the
- // default erased state.
- uint64_t sector_clean_order;
- };
- static_assert(sizeof(KvsSectorHeaderCleaning) == kHeaderSize,
- "Invalid KvsSectorHeaderCleaning size");
-
- // This packs into 8 bytes, to support uint64_t alignment.
- struct KvsHeader {
- uint16_t synchronize_token;
- uint16_t crc; // Crc of the key + data (Ignoring any padding bytes)
- // flags is a Bitmask: bits 15-1 reserved = 0,
- // bit 0: is_erased [0 when not erased, 1 when erased]
- uint16_t flags;
- // On little endian the length fields map to memory as follows:
- // byte array: [ 0x(c0to3 k0to3) 0x(c8to11 c4to7) ]
- // Key length does not include trailing zero. That is not stored.
- uint16_t key_len : kChunkHeaderKeyFieldNumBits;
- // Chunk length is the total length of the chunk before alignment.
- // That way we can work out the length of the value as:
- // (chunk length - key length - size of chunk header).
- uint16_t chunk_len : kChunkHeaderChunkFieldNumBits;
- };
- static_assert(sizeof(KvsHeader) == kHeaderSize, "Invalid KvsHeader size");
-
- struct KeyMap {
- std::string_view key() const { return {key_buffer, key_length}; }
-
- FlashPartition::Address address;
- char key_buffer[kChunkKeyLengthMax + 1]; // +1 for null terminator
- uint8_t key_length;
- bool is_erased;
- uint16_t chunk_len;
-
- static_assert(kChunkKeyLengthMax <=
- std::numeric_limits<decltype(key_length)>::max());
+ struct KeyMapEntry {
+ uint32_t key_hash;
+ uint32_t key_version;
+ Address address; // In partition address.
};
- static constexpr bool InvalidKey(const std::string_view& key) {
- return key.empty() || key.size() > kChunkKeyLengthMax;
- }
+ struct SectorMapEntry {
+ uint16_t tail_free_bytes;
+ uint16_t valid_bytes; // sum of sizes of valid entries
- // NOTE: All public APIs handle the locking, the internal methods assume the
- // lock has already been acquired.
-
- SectorIndex AddressToSectorIndex(FlashPartition::Address address) const {
- return address / partition_.GetSectorSizeBytes();
- }
-
- FlashPartition::Address SectorIndexToAddress(SectorIndex index) const {
- return index * partition_.GetSectorSizeBytes();
- }
-
- // Returns kAddressInvalid if no space is found, otherwise the address.
- FlashPartition::Address FindSpace(size_t requested_size) const;
-
- // Attempts to rewrite a key's value by appending the new value to the same
- // sector. If the sector is full the value is written to another sector, and
- // the sector is marked for cleaning.
- // Returns RESOURCE_EXHAUSTED if no space is available, OK otherwise.
- Status RewriteValue(KeyIndex key_index,
- const span<const std::byte>& value,
- bool is_erased = false);
-
- bool ValueMatches(KeyIndex key_index,
- const span<const std::byte>& value,
- bool is_erased);
-
- // ResetSector erases the sector and writes the sector header.
- Status ResetSector(SectorIndex sector_index);
- Status WriteKeyValue(FlashPartition::Address address,
- const std::string_view& key,
- const span<const std::byte>& value,
- bool is_erased = false);
- uint32_t SectorSpaceRemaining(SectorIndex sector_index) const;
-
- // Returns idx if key is found, otherwise kListCapacityMax.
- KeyIndex FindKeyInMap(const std::string_view& key) const;
- bool IsKeyInMap(const std::string_view& key) const {
- return FindKeyInMap(key) != kListCapacityMax;
- }
-
- void RemoveFromMap(KeyIndex key_index);
- Status AppendToMap(const std::string_view& key,
- FlashPartition::Address address,
- size_t chunk_len,
- bool is_erased = false);
- void UpdateMap(KeyIndex key_index,
- FlashPartition::Address address,
- uint16_t chunk_len,
- bool is_erased = false);
- uint16_t CalculateCrc(const std::string_view& key,
- const span<const std::byte>& value) const;
-
- // Calculates the CRC by reading the value from flash in chunks.
- Status CalculateCrcFromValueAddress(const std::string_view& key,
- FlashPartition::Address value_address,
- uint16_t value_size,
- uint16_t* crc_ret);
-
- uint16_t RoundUpForAlignment(uint16_t size) const {
- if (size % alignment_bytes_ != 0) {
- return size + alignment_bytes_ - size % alignment_bytes_;
+ bool HasSpace(size_t required_space) const {
+ return (tail_free_bytes >= required_space);
}
- return size;
+ };
+
+ Status InvalidOperation(std::string_view key) const;
+
+ static constexpr bool InvalidKey(std::string_view key) {
+ return key.empty() || (key.size() > kMaxKeyLength);
}
- // The buffer is chosen to be no larger then the config value, and
- // be a multiple of the flash alignment.
- size_t TempBufferAlignedSize() const {
- CHECK_GE(sizeof(temp_buffer_), partition_.GetAlignmentBytes());
- return sizeof(temp_buffer_) -
- (sizeof(temp_buffer_) % partition_.GetAlignmentBytes());
+ Status FindKeyMapEntry(std::string_view key,
+ const KeyMapEntry** result) const;
+
+ // Non-const version of FindKeyMapEntry.
+ Status FindKeyMapEntry(std::string_view key, KeyMapEntry** result) {
+ return static_cast<const KeyValueStore&>(*this).FindKeyMapEntry(
+ key, const_cast<const KeyMapEntry**>(result));
}
- // Moves a key/value chunk from one address to another, all fields expected to
- // be aligned.
- Status MoveChunk(FlashPartition::Address dest_address,
- FlashPartition::Address src_address,
- uint16_t size);
+ Status ReadEntryHeader(const KeyMapEntry& entry, EntryHeader* header) const;
+ Status ReadEntryKey(const KeyMapEntry& entry,
+ size_t key_length,
+ char* key) const;
- // Size of a chunk including header, key, value, and alignment padding.
- size_t ChunkSize(size_t key_length, size_t value_length) const {
- return RoundUpForAlignment(sizeof(KvsHeader)) +
- RoundUpForAlignment(key_length) + RoundUpForAlignment(value_length);
+ StatusWithSize ReadEntryValue(const KeyMapEntry& entry,
+ const EntryHeader& header,
+ span<std::byte> value) const;
+
+ Status ValidateEntryChecksum(const EntryHeader& header,
+ std::string_view key,
+ span<const std::byte> value) const;
+
+ Status WriteEntryForExistingKey(KeyMapEntry* key_map_entry,
+ std::string_view key,
+ span<const std::byte> value);
+
+ Status WriteEntryForNewKey(std::string_view key, span<const std::byte> value);
+
+ SectorMapEntry* FindSectorWithSpace(size_t size);
+
+ Status FindOrRecoverSectorWithSpace(SectorMapEntry** sector, size_t size);
+
+ Status GarbageCollectOneSector(SectorMapEntry** sector);
+
+ SectorMapEntry* FindSectorToGarbageCollect();
+
+ Status AppendEntry(SectorMapEntry* sector,
+ KeyMapEntry* entry,
+ std::string_view key,
+ span<const std::byte> value);
+
+ Status VerifyEntry(SectorMapEntry* sector, KeyMapEntry* entry);
+
+ span<const std::byte> CalculateEntryChecksum(
+ const EntryHeader& header,
+ std::string_view key,
+ span<const std::byte> value) const;
+
+ Status RelocateEntry(KeyMapEntry* entry);
+
+ bool SectorEmpty(const SectorMapEntry& sector) const {
+ return (sector.tail_free_bytes == partition_.sector_available_size_bytes());
}
- // Sectors should be cleaned when full, every valid (Most recent, not erased)
- // chunk of data is moved to another sector and the sector is erased.
- // TODO: Clean sectors with lots of invalid data, without a rewrite
- // or erase triggering it.
- Status MarkSectorForClean(SectorIndex sector);
- Status CleanSector(SectorIndex sector);
- Status CleanAllInternal();
- Status GarbageCollectImpl(bool clean_pending_sectors,
- bool exit_when_have_free_sector);
- Status FullGarbageCollect();
- Status EnforceFreeSector();
-
- SectorIndex SectorCount() const {
- return std::min(partition_.GetSectorCount(), kSectorCountMax);
+ size_t RecoverableBytes(const SectorMapEntry& sector) {
+ return partition_.sector_size_bytes() - sector.valid_bytes -
+ sector.tail_free_bytes;
}
- size_t SectorSpaceAvailableWhenEmpty() const {
- return partition_.GetSectorSizeBytes() -
- RoundUpForAlignment(sizeof(KvsSectorHeaderMeta)) -
- RoundUpForAlignment(sizeof(KvsSectorHeaderCleaning));
+ Address SectorBaseAddress(SectorMapEntry* sector) const {
+ return (sector - sector_map_.data()) * partition_.sector_size_bytes();
}
- bool HasEmptySectorImpl(SectorIndex skip_sector) const {
- for (SectorIndex i = 0; i < SectorCount(); i++) {
- if (i != skip_sector &&
- sector_space_remaining_[i] == SectorSpaceAvailableWhenEmpty()) {
- return true;
- }
- }
- return false;
+ Address NextWritableAddress(SectorMapEntry* sector) const {
+ return SectorBaseAddress(sector) + partition_.sector_size_bytes() -
+ sector->tail_free_bytes;
}
- bool IsInLastFreeSector(FlashPartition::Address address) {
- return sector_space_remaining_[AddressToSectorIndex(address)] ==
- SectorSpaceAvailableWhenEmpty() &&
- !HasEmptySectorImpl(AddressToSectorIndex(address));
+ bool EntryMapFull() const { return key_map_size_ == kMaxEntries; }
+
+ span<KeyMapEntry> entries() { return span(key_map_.data(), key_map_size_); }
+
+ span<const KeyMapEntry> entries() const {
+ return span(key_map_.data(), key_map_size_);
}
FlashPartition& partition_;
- // TODO: MUTEX
- bool enabled_ = false;
- uint8_t alignment_bytes_ = 0;
- uint64_t next_sector_clean_order_ = 0;
+ EntryHeaderFormat entry_header_format_;
+ Options options_;
- // Free space available in each sector, set to 0 when clean is pending/active
- uint32_t sector_space_remaining_[kSectorCountMax] = {0};
- uint64_t sector_clean_order_[kSectorCountMax] = {kSectorCleanNotPending};
- KeyMap key_map_[kListCapacityMax] = {};
- KeyIndex map_size_ = 0;
+ // Map is unordered; finding a key requires scanning and
+ // verifying a match by reading the actual entry.
+ std::array<KeyMapEntry, kMaxEntries> key_map_;
+ size_t key_map_size_; // Number of valid entries in entry_map
- // Add +1 for a null terminator. The keys are used as string_views, but the
- // null-terminator provides additional safetly.
- char temp_key_buffer_[kChunkKeyLengthMax + 1u] = {0};
- uint8_t temp_buffer_[cfg::kKvsBufferSize] = {0};
+ // This is dense, so sector_id == indexof(SectorMapEntry) in sector_map
+ std::array<SectorMapEntry, kUsableSectors> sector_map_;
+ size_t last_written_sector_; // TODO: this variable is not used!
};
} // namespace pw::kvs
diff --git a/pw_kvs/public/pw_kvs/partition_table_entry.h b/pw_kvs/public/pw_kvs/partition_table_entry.h
deleted file mode 100644
index 9aa111d..0000000
--- a/pw_kvs/public/pw_kvs/partition_table_entry.h
+++ /dev/null
@@ -1,51 +0,0 @@
-// Copyright 2020 The Pigweed Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not
-// use this file except in compliance with the License. You may obtain a copy of
-// the License at
-//
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-// License for the specific language governing permissions and limitations under
-// the License.
-#pragma once
-
-#include <cstdint>
-
-namespace pw {
-
-namespace partition_table {
-
-enum DataType { kRawData, kKvs };
-
-} // namespace partition_table
-
-enum class PartitionPermission : bool {
- kReadOnly = true,
- kReadAndWrite = false,
-};
-
-struct PartitionTableEntry {
- constexpr PartitionTableEntry(
- const uint32_t start_sector_index,
- const uint32_t end_sector_index,
- const uint8_t partition_identifier,
- partition_table::DataType datatype,
- PartitionPermission permission = PartitionPermission::kReadAndWrite)
- : partition_start_sector_index(start_sector_index),
- partition_end_sector_index(end_sector_index),
- partition_id(partition_identifier),
- data_type(datatype),
- partition_permission(permission) {}
-
- const uint32_t partition_start_sector_index;
- const uint32_t partition_end_sector_index;
- const uint8_t partition_id;
- const partition_table::DataType data_type;
- const PartitionPermission partition_permission;
-};
-
-} // namespace pw
diff --git a/pw_kvs/pw_kvs_private/format.h b/pw_kvs/pw_kvs_private/format.h
new file mode 100644
index 0000000..25484c7
--- /dev/null
+++ b/pw_kvs/pw_kvs_private/format.h
@@ -0,0 +1,70 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+
+#include "pw_span/span.h"
+
+namespace pw::kvs {
+
+// In-flash header format.
+struct EntryHeader {
+ EntryHeader() = default;
+
+ EntryHeader(uint32_t magic,
+ span<const std::byte> checksum,
+ size_t key_length,
+ size_t value_length,
+ uint32_t key_version)
+ : magic_(magic),
+ checksum_(0),
+ key_value_length_(value_length << kValueLengthShift |
+ (key_length & kKeyLengthMask)),
+ key_version_(key_version) {
+ std::memcpy(&checksum_,
+ checksum.data(),
+ std::min(checksum.size(), sizeof(checksum_)));
+ }
+
+ span<const std::byte> DataForChecksum() const {
+ return span(reinterpret_cast<const std::byte*>(this) +
+ offsetof(EntryHeader, key_value_length_),
+ sizeof(*this) - offsetof(EntryHeader, key_value_length_));
+ }
+
+ uint32_t magic() const { return magic_; }
+ span<const std::byte> checksum() const {
+ return as_bytes(span(&checksum_, 1));
+ }
+ size_t key_length() const { return key_value_length_ & kKeyLengthMask; }
+ size_t value_length() const { return key_value_length_ >> kValueLengthShift; }
+ uint32_t key_version() const { return key_version_; }
+
+ private:
+ static constexpr uint32_t kKeyLengthMask = 0b111111;
+ static constexpr uint32_t kValueLengthShift = 8;
+
+ uint32_t magic_;
+ uint32_t checksum_;
+ // 0:5 key_len 8:31 value_len
+ uint32_t key_value_length_;
+ uint32_t key_version_;
+};
+
+static_assert(sizeof(EntryHeader) == 16, "EntryHeader should have no padding");
+
+} // namespace pw::kvs
diff --git a/pw_kvs/pw_kvs_private/macros.h b/pw_kvs/pw_kvs_private/macros.h
new file mode 100644
index 0000000..d27fb2d
--- /dev/null
+++ b/pw_kvs/pw_kvs_private/macros.h
@@ -0,0 +1,40 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+#pragma once
+
+// Macros for cleanly working with Status or StatusWithSize objects.
+#define PW_TRY(expr) _PW_TRY(_PW_TRY_UNIQUE(__LINE__), expr)
+
+#define _PW_TRY(result, expr) \
+ do { \
+ if (auto result = (expr); !result.ok()) { \
+ return result; \
+ } \
+ } while (0)
+
+#define PW_TRY_ASSIGN(assignment_lhs, expression) \
+ _PW_TRY_ASSIGN(_PW_TRY_UNIQUE(__LINE__), assignment_lhs, expression)
+
+#define _PW_TRY_ASSIGN(result, lhs, expr) \
+ auto result = (expr); \
+ if (!result.ok()) { \
+ return result; \
+ } \
+ lhs = std::move(result.size())
+
+#define _PW_TRY_UNIQUE(line) _PW_TRY_UNIQUE_EXPANDED(line)
+#define _PW_TRY_UNIQUE_EXPANDED(line) _pw_try_unique_name_##line
+
+#define TRY PW_TRY
+#define TRY_ASSIGN PW_TRY_ASSIGN
diff --git a/pw_status/public/pw_status/status_with_size.h b/pw_status/public/pw_status/status_with_size.h
index d3957b0..aa9d549 100644
--- a/pw_status/public/pw_status/status_with_size.h
+++ b/pw_status/public/pw_status/status_with_size.h
@@ -54,9 +54,17 @@
explicit constexpr StatusWithSize(Status status, size_t size)
: StatusWithSize(size | (static_cast<size_t>(status) << kStatusShift)) {}
+ // Allow implicit conversions from status.
+ constexpr StatusWithSize(Status status)
+ : StatusWithSize(static_cast<size_t>(status) << kStatusShift) {}
+ constexpr StatusWithSize(Status::Code status)
+ : StatusWithSize(Status(status)) {}
+
constexpr StatusWithSize(const StatusWithSize&) = default;
constexpr StatusWithSize& operator=(const StatusWithSize&) = default;
+ constexpr operator Status() const { return status(); }
+
// Returns the size. The size is always present, even if status() is an error.
constexpr size_t size() const { return size_ & kSizeMask; }
