From 22c3a07677e1743b64182b77e3a34e35307272bb Mon Sep 17 00:00:00 2001 From: Qin Wang Date: Thu, 29 Jun 2023 10:29:46 -0700 Subject: [PATCH] Migrate DataManager to third_party/nearby/internal PiperOrigin-RevId: 544395891 --- Package.swift | 5 + internal/data/BUILD | 58 +++++ internal/data/data_manager.h | 53 +++++ internal/data/data_set.h | 70 ++++++ internal/data/leveldb_data_set.h | 226 +++++++++++++++++++ internal/data/leveldb_data_set_test.cc | 259 ++++++++++++++++++++++ internal/data/leveldb_data_set_test.proto | 22 ++ internal/data/memory_data_set.h | 106 +++++++++ internal/data/memory_data_set_test.cc | 71 ++++++ internal/platform/BUILD | 1 + internal/platform/implementation/g3/BUILD | 1 + internal/test/BUILD | 7 +- internal/test/fake_data_set.h | 131 +++++++++++ internal/test/fake_data_set_test.cc | 107 +++++++++ 14 files changed, 1115 insertions(+), 2 deletions(-) create mode 100644 internal/data/BUILD create mode 100644 internal/data/data_manager.h create mode 100644 internal/data/data_set.h create mode 100644 internal/data/leveldb_data_set.h create mode 100644 internal/data/leveldb_data_set_test.cc create mode 100644 internal/data/leveldb_data_set_test.proto create mode 100644 internal/data/memory_data_set.h create mode 100644 internal/data/memory_data_set_test.cc create mode 100644 internal/test/fake_data_set.h create mode 100644 internal/test/fake_data_set_test.cc diff --git a/Package.swift b/Package.swift index 65b8cb82..6553676a 100644 --- a/Package.swift +++ b/Package.swift @@ -410,6 +410,7 @@ let package = Package( "internal/flags/BUILD", "internal/network/BUILD", "internal/base/BUILD", + "internal/data/BUILD", "internal/test/BUILD", // tests "connections/listeners_test.cc", @@ -476,6 +477,8 @@ let package = Package( "internal/crypto/sha2_unittest.cc", "internal/crypto/signature_verifier_unittest.cc", "internal/crypto/symmetric_key_unittest.cc", + "internal/data/leveldb_data_set_test.cc", + "internal/data/memory_data_set_test.cc", "internal/flags/nearby_flags_test.cc", "internal/proto/analytics/connections_log_test.cc", "internal/platform/feature_flags_test.cc", @@ -539,6 +542,7 @@ let package = Package( "internal/test/fake_timer_test.cc", "internal/test/fake_device_info_test.cc", "internal/test/fake_task_runner_test.cc", + "internal/test/fake_data_set_test.cc", "internal/weave/base_socket_test.cc", "internal/weave/control_packet_write_request_test.cc", "internal/weave/message_write_request_test.cc", @@ -552,6 +556,7 @@ let package = Package( "connections/implementation/proto", "internal/proto", "proto", + "internal/data/leveldb_data_set_test.proto", // webrtc "connections/implementation/webrtc_bwu_handler.cc", "connections/implementation/webrtc_endpoint_channel.cc", diff --git a/internal/data/BUILD b/internal/data/BUILD new file mode 100644 index 00000000..562703c4 --- /dev/null +++ b/internal/data/BUILD @@ -0,0 +1,58 @@ +load("@rules_cc//cc:defs.bzl", "cc_proto_library") + +licenses(["notice"]) + +package(default_visibility = [ + "//visibility:public", +]) + +cc_library( + name = "data_manager", + hdrs = [ + "data_manager.h", + "data_set.h", + "leveldb_data_set.h", + "memory_data_set.h", + ], + deps = [ + "//internal/platform:logging", + "//third_party/leveldb:db", + "//third_party/leveldb:table", + "//third_party/leveldb:util", + "//third_party/protobuf:protobuf_lite", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", + ], +) + +proto_library( + name = "leveldb_data_set_test_proto", + srcs = ["leveldb_data_set_test.proto"], +) + +cc_proto_library( + name = "leveldb_data_set_test_cc_proto", + deps = [":leveldb_data_set_test_proto"], +) + +cc_test( + name = "data_manager_test", + size = "small", + timeout = "short", + srcs = [ + "leveldb_data_set_test.cc", + "memory_data_set_test.cc", + ], + shard_count = 8, + deps = [ + ":data_manager", + ":leveldb_data_set_test_cc_proto", + "//internal/platform/implementation/g3", # fixdeps: keep + "@com_github_protobuf_matchers//protobuf-matchers", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/time", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/internal/data/data_manager.h b/internal/data/data_manager.h new file mode 100644 index 00000000..3f42f456 --- /dev/null +++ b/internal/data/data_manager.h @@ -0,0 +1,53 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_NEARBY_INTERNAL_DATA_DATA_MANAGER_H_ +#define THIRD_PARTY_NEARBY_INTERNAL_DATA_DATA_MANAGER_H_ + +#include + +#include "absl/strings/string_view.h" +#include "internal/data/data_set.h" +#include "internal/data/leveldb_data_set.h" +#include "internal/data/memory_data_set.h" + +namespace nearby { +namespace data { + +class DataManager { + public: + enum class DataStorageType : int { kMemory = 0, kLevelDb = 1 }; + explicit DataManager(DataStorageType data_storage_type) + : data_storage_type_(data_storage_type) {} + ~DataManager() = default; + + template + std::unique_ptr> GetDataSet(absl::string_view path) { + if (data_storage_type_ == DataStorageType::kMemory) { + return std::make_unique>(path); + } else if (data_storage_type_ == DataStorageType::kLevelDb) { + return std::make_unique>(path); + } else { + return nullptr; + } + } + + private: + DataStorageType data_storage_type_; +}; + +} // namespace data +} // namespace nearby + +#endif // THIRD_PARTY_NEARBY_INTERNAL_DATA_DATA_MANAGER_H_ diff --git a/internal/data/data_set.h b/internal/data/data_set.h new file mode 100644 index 00000000..c87492a4 --- /dev/null +++ b/internal/data/data_set.h @@ -0,0 +1,70 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_NEARBY_INTERNAL_DATA_DATA_SET_H_ +#define THIRD_PARTY_NEARBY_INTERNAL_DATA_DATA_SET_H_ + +#include +#include +#include +#include +#include + +namespace nearby { +namespace data { + +enum class InitStatus { + kOK = 0, + kNotInitialized = 1, + kError = 2, + kCorrupt = 3, + kInvalidOperation = 4, + kMaxValue = kInvalidOperation +}; + +template +class DataSet { + public: + using KeyEntryVector = std::vector>; + + virtual ~DataSet() = default; + + // Asynchronously initializes the object, which must have been created by the + // DataManager::GetDataSet function. |callback| will be invoked on the + // calling thread when complete. + virtual void Initialize(std::function callback) = 0; + + // Asynchronously loads all entries from the database and invokes |callback| + // when complete. + virtual void LoadEntries( + std::function>)> callback) = 0; + + // Asynchronously saves |entries_to_save| and deletes entries from + // |keys_to_remove| from the database. |callback| will be invoked on the + // calling thread when complete. |entries_to_save| and |keys_to_remove| must + // be non-null. + virtual void UpdateEntries( + std::unique_ptr entries_to_save, + std::unique_ptr> keys_to_remove, + std::function callback) = 0; + + // Asynchronously destroys the database. Use this call only if the database + // needs to be destroyed for this particular profile. + virtual void Destroy(std::function callback) = 0; +}; + +} // namespace data +} // namespace nearby + +#endif // THIRD_PARTY_NEARBY_INTERNAL_DATA_DATA_SET_H_ diff --git a/internal/data/leveldb_data_set.h b/internal/data/leveldb_data_set.h new file mode 100644 index 00000000..c55a57bc --- /dev/null +++ b/internal/data/leveldb_data_set.h @@ -0,0 +1,226 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_NEARBY_INTERNAL_DATA_LEVELDB_DATA_SET_H_ +#define THIRD_PARTY_NEARBY_INTERNAL_DATA_LEVELDB_DATA_SET_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "third_party/leveldb/include/db.h" +#include "third_party/leveldb/include/iterator.h" +#include "third_party/leveldb/include/options.h" +#include "third_party/leveldb/include/slice.h" +#include "third_party/leveldb/include/status.h" +#include "internal/data/data_set.h" +#include "internal/platform/logging.h" +#include "third_party/protobuf/message_lite.h" + +namespace nearby { +namespace data { +// DataSet implementation using leveldb as its persistent storage. Values are +// serialized and stored in leveldb databases. +template ::value, + bool> = true> +class LeveldbDataSet : public DataSet { + public: + using KeyEntryVector = std::vector>; + + explicit LeveldbDataSet(absl::string_view path) : path_(path) {} + ~LeveldbDataSet() override = default; + + void Initialize(std::function callback) override; + void LoadEntries(std::function>)> + callback) override; + void LoadEntriesWithKeys( + std::function< + void(bool, std::unique_ptr>>)> + callback); + void UpdateEntries(std::unique_ptr entries_to_save, + std::unique_ptr> keys_to_remove, + std::function callback) override; + void Destroy(std::function callback) override; + + private: + void Serialize(T const& value, std::string& str); + void Deserialize(absl::string_view str, T& value); + + private: + std::string path_; + std::unique_ptr db_ = nullptr; + InitStatus status_ = InitStatus::kNotInitialized; +}; + +template ::value, bool> + isMessageLite> +void LeveldbDataSet::Initialize( + std::function callback) { + leveldb::Options options; + options.create_if_missing = true; + + leveldb::DB* db; + leveldb::Status status = leveldb::DB::Open(options, path_, &db); + db_ = std::unique_ptr(db); + + if (status.ok()) { + status_ = InitStatus::kOK; + NEARBY_LOGS(INFO) << "Database is initialized successfully.."; + } else if (status.IsCorruption() || status.IsIOError()) { + status_ = InitStatus::kCorrupt; + NEARBY_LOGS(INFO) << "Database is corrupt."; + + } else { + status_ = InitStatus::kError; + NEARBY_LOGS(INFO) << "Failed to initialize database due to unknown error."; + } + std::move(callback)(status_); +} + +template ::value, bool> + isMessageLite> +void LeveldbDataSet::LoadEntries( + std::function>)> callback) { + auto result = std::make_unique>(); + if (status_ != InitStatus::kOK) { + std::move(callback)(false, std::move(result)); + return; + } + + std::unique_ptr it( + db_->NewIterator(leveldb::ReadOptions())); + + for (it->SeekToFirst(); it->Valid(); it->Next()) { + T value; + Deserialize(it->value().ToString(), value); + result->push_back(value); + } + + if (it->status().ok()) { + NEARBY_LOGS(INFO) << "Loaded " << result->size() + << " entries from database."; + std::move(callback)(true, std::move(result)); + } else { + NEARBY_LOGS(INFO) << "Failed to load entries from database."; + result->clear(); + std::move(callback)(false, std::move(result)); + } +} + +template ::value, bool> + isMessageLite> +void LeveldbDataSet::LoadEntriesWithKeys( + std::function>>)> + callback) { + auto result = std::make_unique>>(); + if (status_ != InitStatus::kOK) { + std::move(callback)(false, std::move(result)); + return; + } + + std::unique_ptr it( + db_->NewIterator(leveldb::ReadOptions())); + + for (it->SeekToFirst(); it->Valid(); it->Next()) { + T value; + Deserialize(it->value().ToString(), value); + result->push_back({it->key().ToString(), value}); + } + + if (it->status().ok()) { + NEARBY_LOGS(INFO) << "Loaded " << result->size() + << " entries from database."; + std::move(callback)(true, std::move(result)); + } else { + NEARBY_LOGS(INFO) << "Failed to load entries from database."; + result->clear(); + std::move(callback)(false, std::move(result)); + } +} + +template ::value, bool> + isMessageLite> +void LeveldbDataSet::UpdateEntries( + std::unique_ptr entries_to_save, + std::unique_ptr> keys_to_remove, + std::function callback) { + NEARBY_LOGS(INFO) << "UpdateEntries is called."; + if (status_ != InitStatus::kOK) { + std::move(callback)(false); + return; + } + + if (entries_to_save != nullptr) { + for (const auto& [key, value] : *entries_to_save) { + std::string str; + Serialize(value, str); + db_->Put(leveldb::WriteOptions(), key, leveldb::Slice(str)); + } + } + + if (keys_to_remove != nullptr) { + for (const auto& it : *keys_to_remove) { + db_->Delete(leveldb::WriteOptions(), it); + } + } + + std::move(callback)(true); +} + +template ::value, bool> + isMessageLite> +void LeveldbDataSet::Destroy( + std::function callback) { + NEARBY_LOGS(INFO) << "Destroy is called."; + db_.reset(); + leveldb::DestroyDB(path_, leveldb::Options()); + std::move(callback)(true); +} + +// Functions for serializing/deserializing data values to/from strings. Strings +// are used as a convenient container that manages its memory. They don't need +// to be human-readable. + +template ::value, bool> + isMessageLite> +void LeveldbDataSet::Serialize(T const& value, + std::string& str) { + value.SerializeToString(&str); +} + +template ::value, bool> + isMessageLite> +void LeveldbDataSet::Deserialize(absl::string_view str, + T& value) { + value.ParseFromString(str); +} + +} // namespace data +} // namespace nearby + +#endif // THIRD_PARTY_NEARBY_INTERNAL_DATA_LEVELDB_DATA_SET_H_ diff --git a/internal/data/leveldb_data_set_test.cc b/internal/data/leveldb_data_set_test.cc new file mode 100644 index 00000000..c5f2113b --- /dev/null +++ b/internal/data/leveldb_data_set_test.cc @@ -0,0 +1,259 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "internal/data/leveldb_data_set.h" + +#include + +#include // NOLINT(build/c++17) +#include +#include +#include +#include +#include +#include +#include + +#include "gmock/gmock.h" +#include "protobuf-matchers/protocol-buffer-matchers.h" +#include "gtest/gtest.h" +#include "absl/container/flat_hash_map.h" +#include "absl/synchronization/notification.h" +#include "absl/time/time.h" +#include "internal/data/data_set.h" +#include "internal/data/leveldb_data_set_test.proto.h" + +namespace nearby { +namespace data { +namespace { + +using ::testing::SizeIs; + +// Generate a unique directory under temp directory for leveldb storage +std::filesystem::path GenerateLeveldbPath() { + auto temp_directory_path = std::filesystem::temp_directory_path(); + std::random_device dev; + std::mt19937 prng(dev()); + std::uniform_int_distribution rand(0); + std::filesystem::path path; + do { + std::stringstream leveldb_directory; + leveldb_directory << std::hex << "nearby_db_" << rand(prng); + path = temp_directory_path / leveldb_directory.str(); + } while (std::filesystem::exists(path)); + return path; +} + +// Helper functions to synchronize LeveldbDataSet function calls for testing +template +std::unique_ptr> CreateDataSet( + const std::filesystem::path& path) { + return std::make_unique>(path.string()); +} + +template +InitStatus InitializeAndWait(std::unique_ptr>& dataset) { + InitStatus status; + absl::Notification notification; + dataset->Initialize([¬ification, &status](InitStatus s) { + status = s; + notification.Notify(); + }); + notification.WaitForNotificationWithTimeout(absl::Seconds(5)); + return status; +} + +template +bool UpdateEntriesAndWait( + std::unique_ptr>& dataset, + std::unique_ptr::KeyEntryVector> entries_to_save, + std::unique_ptr> keys_to_remove) { + bool result = false; + absl::Notification notification; + dataset->UpdateEntries(std::move(entries_to_save), std::move(keys_to_remove), + [&result, ¬ification](bool res) { + result = res; + notification.Notify(); + }); + notification.WaitForNotificationWithTimeout(absl::Seconds(5)); + return result; +} + +template +std::unique_ptr> LoadEntriesAndWait( + std::unique_ptr>& dataset) { + auto result = std::make_unique>(); + absl::Notification notification; + dataset->LoadEntries( + [&result, ¬ification](bool, std::unique_ptr> res) { + for (const auto& it : *res) { + result->push_back(it); + } + notification.Notify(); + }); + notification.WaitForNotificationWithTimeout(absl::Seconds(5)); + return result; +} + +template +absl::flat_hash_map LoadEntriesWithKeysAndWait( + std::unique_ptr>& dataset) { + auto result = std::make_unique>>(); + absl::Notification notification; + dataset->LoadEntriesWithKeys( + [&result, ¬ification]( + bool, std::unique_ptr>> res) { + for (const auto& it : *res) { + result->push_back(it); + } + notification.Notify(); + }); + notification.WaitForNotificationWithTimeout(absl::Seconds(5)); + + absl::flat_hash_map entry_map; + for (const auto& it : *result) { + entry_map[it.first] = it.second; + } + return entry_map; +} + +template +void WipeCleanAndWait(std::unique_ptr>& dataset, + std::filesystem::path path) { + absl::Notification notification; + dataset->Destroy([¬ification](bool) { notification.Notify(); }); + notification.WaitForNotificationWithTimeout(absl::Seconds(5)); + // Call the destructor before removing leveldb storage directory + dataset.reset(); + std::filesystem::remove_all(path); +} + +DiceRoll GenerateDiceRoll(int value) { + DiceRoll result; + result.set_value(value); + if (value == 12) { + result.set_nickname("boxcars"); + } + + if (value == 2) { + result.set_nickname("snake eyes"); + } + + return result; +} + +TEST(LeveldbDataSet, UpdateEntriesDiceRoll) { + std::filesystem::path path = GenerateLeveldbPath(); + std::unique_ptr> diceroll_set = + CreateDataSet(path); + + InitStatus status = InitializeAndWait(diceroll_set); + ASSERT_EQ(status, InitStatus::kOK); + + DiceRoll diceroll1 = GenerateDiceRoll(2); + DiceRoll diceroll2 = GenerateDiceRoll(12); + + auto entries = LeveldbDataSet::KeyEntryVector( + {{"id1", diceroll1}, {"id2", diceroll2}}); + auto data = + std::make_unique::KeyEntryVector>(entries); + + bool result = UpdateEntriesAndWait(diceroll_set, std::move(data), nullptr); + WipeCleanAndWait(diceroll_set, path); + + EXPECT_TRUE(result); +} + +TEST(LeveldbDataSet, LoadEntriesDiceRoll) { + std::filesystem::path path = GenerateLeveldbPath(); + std::unique_ptr> diceroll_set = + CreateDataSet(path); + + InitializeAndWait(diceroll_set); + + DiceRoll diceroll1 = GenerateDiceRoll(2); + DiceRoll diceroll2 = GenerateDiceRoll(12); + + auto entries = LeveldbDataSet::KeyEntryVector( + {{"id1", diceroll1}, {"id2", diceroll2}}); + auto data = + std::make_unique::KeyEntryVector>(entries); + UpdateEntriesAndWait(diceroll_set, std::move(data), nullptr); + + auto result = LoadEntriesAndWait(diceroll_set); + WipeCleanAndWait(diceroll_set, path); + + EXPECT_THAT(*result, SizeIs(2)); + + // EqualsProto is only available internally + // EXPECT_THAT((*result)[0], protobuf_matchers::EqualsProto( + // "value: 2 nickname: 'snake eyes'")); + // EXPECT_THAT((*result)[1], + // protobuf_matchers::EqualsProto("value: 12 nickname: + // 'boxcars'")); + + EXPECT_EQ((*result)[0].value(), 2); + EXPECT_EQ((*result)[0].nickname(), "snake eyes"); + + EXPECT_EQ((*result)[1].value(), 12); + EXPECT_EQ((*result)[1].nickname(), "boxcars"); +} + +TEST(LeveldbDataSet, RemoveEntriesDiceRoll) { + std::filesystem::path path = GenerateLeveldbPath(); + std::unique_ptr> diceroll_set = + CreateDataSet(path); + + InitializeAndWait(diceroll_set); + + DiceRoll diceroll1 = GenerateDiceRoll(2); + DiceRoll diceroll2 = GenerateDiceRoll(12); + DiceRoll diceroll3 = GenerateDiceRoll(5); + DiceRoll diceroll4 = GenerateDiceRoll(7); + + auto entries_to_add1 = LeveldbDataSet::KeyEntryVector( + {{"id1", diceroll1}, {"id2", diceroll2}}); + auto data_to_add1 = + std::make_unique::KeyEntryVector>( + entries_to_add1); + UpdateEntriesAndWait(diceroll_set, std::move(data_to_add1), nullptr); + + auto entries_to_add2 = LeveldbDataSet::KeyEntryVector( + {{"id3", diceroll3}, {"id4", diceroll4}}); + auto data_to_add2 = + std::make_unique::KeyEntryVector>( + entries_to_add2); + + auto keys_to_remove = std::make_unique>( + std::vector({"id1"})); + + UpdateEntriesAndWait(diceroll_set, std::move(data_to_add2), + std::move(keys_to_remove)); + + auto result = LoadEntriesWithKeysAndWait(diceroll_set); + WipeCleanAndWait(diceroll_set, path); + + EXPECT_THAT(result, SizeIs(3)); + + EXPECT_EQ(result["id2"].value(), diceroll2.value()); + EXPECT_EQ(result["id2"].nickname(), diceroll2.nickname()); + EXPECT_EQ(result["id3"].value(), diceroll3.value()); + EXPECT_EQ(result["id3"].nickname(), diceroll3.nickname()); + EXPECT_EQ(result["id4"].value(), diceroll4.value()); + EXPECT_EQ(result["id4"].nickname(), diceroll4.nickname()); +} + +} // namespace +} // namespace data +} // namespace nearby diff --git a/internal/data/leveldb_data_set_test.proto b/internal/data/leveldb_data_set_test.proto new file mode 100644 index 00000000..c1655b0c --- /dev/null +++ b/internal/data/leveldb_data_set_test.proto @@ -0,0 +1,22 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package nearby.data; + +message DiceRoll { + optional int32 value = 1; // value of this roll, e.g. 2..12 + optional string nickname = 2; // string nickname, e.g. "snake eyes" +} diff --git a/internal/data/memory_data_set.h b/internal/data/memory_data_set.h new file mode 100644 index 00000000..48074573 --- /dev/null +++ b/internal/data/memory_data_set.h @@ -0,0 +1,106 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_NEARBY_INTERNAL_DATA_MEMORY_DATA_SET_H_ +#define THIRD_PARTY_NEARBY_INTERNAL_DATA_MEMORY_DATA_SET_H_ + +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/strings/string_view.h" +#include "absl/synchronization/mutex.h" +#include "internal/data/data_set.h" + +namespace nearby { +namespace data { + +template +class MemoryDataSet : public DataSet { + public: + using KeyEntryVector = std::vector>; + + explicit MemoryDataSet(absl::string_view path) : path_(path) {} + ~MemoryDataSet() override = default; + + void Initialize(std::function callback) override; + void LoadEntries(std::function>)> + callback) override; + void UpdateEntries(std::unique_ptr entries_to_save, + std::unique_ptr> keys_to_remove, + std::function callback) override; + void Destroy(std::function callback) override; + + private: + std::string path_; + + absl::Mutex mutex_; + absl::flat_hash_map entries_; +}; + +template +void MemoryDataSet::Initialize(std::function callback) { + std::move(callback)(InitStatus::kOK); +} + +template +void MemoryDataSet::LoadEntries( + std::function>)> callback) { + auto result = std::make_unique>(); + auto it = entries_.begin(); + while (it != entries_.end()) { + result->push_back(it->second); + ++it; + } + + std::move(callback)(true, std::move(result)); +} + +template +void MemoryDataSet::UpdateEntries( + std::unique_ptr entries_to_save, + std::unique_ptr> keys_to_remove, + std::function callback) { + if (entries_to_save != nullptr) { + auto it = entries_to_save->begin(); + while (it != entries_to_save->end()) { + entries_.emplace(it->first, it->second); + ++it; + } + } + + if (keys_to_remove != nullptr) { + auto it = keys_to_remove->begin(); + while (it != keys_to_remove->end()) { + entries_.erase(*it); + ++it; + } + } + + std::move(callback)(true); +} + +template +void MemoryDataSet::Destroy(std::function callback) { + entries_.clear(); + std::move(callback)(true); +} + +} // namespace data +} // namespace nearby + +#endif // THIRD_PARTY_NEARBY_INTERNAL_DATA_MEMORY_DATA_SET_H_ diff --git a/internal/data/memory_data_set_test.cc b/internal/data/memory_data_set_test.cc new file mode 100644 index 00000000..e5e3a13d --- /dev/null +++ b/internal/data/memory_data_set_test.cc @@ -0,0 +1,71 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "internal/data/memory_data_set.h" + +#include +#include +#include +#include +#include + +#include "gmock/gmock.h" +#include "protobuf-matchers/protocol-buffer-matchers.h" +#include "gtest/gtest.h" + +namespace nearby { +namespace data { +namespace { + +TEST(MemoryDataSet, TestUpdateEntries) { + bool result = false; + MemoryDataSet string_set{""}; + + auto temp = MemoryDataSet::KeyEntryVector( + {{"id1", "string1"}, {"id2", "string2"}}); + + auto data = + std::make_unique::KeyEntryVector>(temp); + string_set.UpdateEntries(std::move(data), nullptr, + [&result](bool res) { result = res; }); + EXPECT_TRUE(result); +} + +TEST(MemoryDataSet, TestLoadEntries) { + std::vector result = {}; + MemoryDataSet string_set{""}; + + auto temp = MemoryDataSet::KeyEntryVector( + {{"id1", "string1"}, {"id2", "string2"}}); + auto data = + std::make_unique::KeyEntryVector>(temp); + + string_set.UpdateEntries(std::move(data), nullptr, [](bool ans) {}); + string_set.LoadEntries( + [&result](bool ans, std::unique_ptr> res) { + auto it = res->begin(); + while (it != res->end()) { + result.push_back(*it); + ++it; + } + }); + + EXPECT_THAT(result, testing::SizeIs(2)); + std::sort(result.begin(), result.end()); + EXPECT_EQ(result, std::vector({"string1", "string2"})); +} + +} // namespace +} // namespace data +} // namespace nearby diff --git a/internal/platform/BUILD b/internal/platform/BUILD index b703e51a..1b95292a 100644 --- a/internal/platform/BUILD +++ b/internal/platform/BUILD @@ -103,6 +103,7 @@ cc_library( "//fastpair:__subpackages__", "//internal/auth:__subpackages__", "//internal/auth/credential_store:__subpackages__", + "//internal/data:__subpackages__", "//internal/interop:__pkg__", "//internal/network:__subpackages__", "//internal/platform:__subpackages__", diff --git a/internal/platform/implementation/g3/BUILD b/internal/platform/implementation/g3/BUILD index 6ae19670..bef09ade 100644 --- a/internal/platform/implementation/g3/BUILD +++ b/internal/platform/implementation/g3/BUILD @@ -142,6 +142,7 @@ cc_library( "//fastpair:__subpackages__", "//internal/account:__subpackages__", "//internal/auth:__subpackages__", + "//internal/data:__subpackages__", "//internal/flags:__subpackages__", "//internal/network:__subpackages__", "//internal/platform:__subpackages__", diff --git a/internal/test/BUILD b/internal/test/BUILD index dcf73dfc..dbfbc938 100644 --- a/internal/test/BUILD +++ b/internal/test/BUILD @@ -25,6 +25,7 @@ cc_library( ], hdrs = [ "fake_clock.h", + "fake_data_set.h", "fake_device_info.h", "fake_single_thread_executor.h", "fake_task_runner.h", @@ -37,6 +38,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ "//internal/base:bluetooth_address", + "//internal/data:data_manager", "//internal/platform:comm", "//internal/platform:types", "//internal/platform/implementation:types", @@ -55,6 +57,7 @@ cc_test( timeout = "short", srcs = [ "fake_clock_test.cc", + "fake_data_set_test.cc", "fake_device_info_test.cc", "fake_task_runner_test.cc", "fake_timer_test.cc", @@ -65,9 +68,9 @@ cc_test( shard_count = 8, deps = [ ":test", - "//internal/platform:types", + "//internal/data:data_manager", "//internal/platform/implementation:types", - "//internal/platform/implementation/g3", + "//internal/platform/implementation/g3", # fixdeps: keep "@com_github_protobuf_matchers//protobuf-matchers", "@com_google_absl//absl/synchronization", "@com_google_absl//absl/time", diff --git a/internal/test/fake_data_set.h b/internal/test/fake_data_set.h new file mode 100644 index 00000000..7393abda --- /dev/null +++ b/internal/test/fake_data_set.h @@ -0,0 +1,131 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_NEARBY_INTERNAL_TEST_FAKE_DATA_SET_H_ +#define THIRD_PARTY_NEARBY_INTERNAL_TEST_FAKE_DATA_SET_H_ + +#include +#include +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "internal/data/data_set.h" + +namespace nearby { +namespace data { + +template +class FakeDataSet : public DataSet { + public: + using KeyEntryVector = std::vector>; + + explicit FakeDataSet(const absl::flat_hash_map& entries_map) + : entries_map_(entries_map) {} + + void Initialize(std::function callback) override { + init_callback_ = std::move(callback); + } + + void LoadEntries(std::function>)> + callback) override { + load_callback_ = std::move(callback); + } + + void UpdateEntries(std::unique_ptr entries_to_save, + std::unique_ptr> keys_to_remove, + std::function callback) override { + entries_to_save_ = std::move(entries_to_save); + keys_to_remove_ = std::move(keys_to_remove); + update_callback_ = std::move(callback); + } + + void Destroy(std::function callback) override { + destroy_callback_ = std::move(callback); + } + + // Mocked methods + void InitStatusCallback(InitStatus status) { + if (init_callback_ != nullptr) { + init_callback_(status); + } + } + + void LoadCallback(bool success) { + if (load_callback_ != nullptr) { + auto entries = std::make_unique>(); + for (auto it = entries_map_.begin(); it != entries_map_.end(); ++it) { + entries->push_back(it->second); + } + load_callback_(success, std::move(entries)); + } + } + + void UpdateCallback(bool success) { + if (success) { + if (entries_to_save_ != nullptr) { + for (auto it = entries_to_save_->begin(); it != entries_to_save_->end(); + ++it) { + auto entry = entries_map_.find(it->first); + if (entry == entries_map_.end()) { + entries_map_.emplace(it->first, it->second); + } else { + entry->second = it->second; + } + } + } + + if (keys_to_remove_ != nullptr) { + for (auto it = keys_to_remove_->begin(); it != keys_to_remove_->end(); + ++it) { + entries_map_.erase(*it); + } + } + } + + entries_to_save_ = nullptr; + keys_to_remove_ = nullptr; + if (update_callback_ != nullptr) { + update_callback_(success); + } + } + + void DestroyCallback(bool success) { + if (success) { + entries_map_.clear(); + } + + if (destroy_callback_ != nullptr) { + destroy_callback_(success); + } + } + + absl::flat_hash_map& entries_map() { return entries_map_; } + + private: + absl::flat_hash_map entries_map_ = nullptr; + std::function init_callback_ = nullptr; + std::function>)> load_callback_ = + nullptr; + std::unique_ptr entries_to_save_ = nullptr; + std::unique_ptr> keys_to_remove_ = nullptr; + std::function update_callback_ = nullptr; + std::function destroy_callback_ = nullptr; +}; + +} // namespace data +} // namespace nearby + +#endif // THIRD_PARTY_NEARBY_INTERNAL_TEST_FAKE_DATA_SET_H_ diff --git a/internal/test/fake_data_set_test.cc b/internal/test/fake_data_set_test.cc new file mode 100644 index 00000000..fb7da29d --- /dev/null +++ b/internal/test/fake_data_set_test.cc @@ -0,0 +1,107 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "internal/test/fake_data_set.h" + +#include +#include +#include +#include +#include + +#include "gmock/gmock.h" +#include "protobuf-matchers/protocol-buffer-matchers.h" +#include "gtest/gtest.h" +#include "internal/data/data_set.h" + +namespace nearby { +namespace data { +namespace { + +TEST(FakeDataSet, TestInitialize) { + InitStatus result = InitStatus::kNotInitialized; + FakeDataSet string_set({}); + + string_set.Initialize([&result](InitStatus res) { result = res; }); + string_set.InitStatusCallback(InitStatus::kOK); + EXPECT_EQ(result, InitStatus::kOK); +} + +TEST(FakeDataSet, TestUpdateEntries) { + bool result = false; + FakeDataSet string_set({}); + + auto temp = FakeDataSet::KeyEntryVector( + {{"id1", "string1"}, {"id2", "string2"}}); + + auto data = std::make_unique::KeyEntryVector>(temp); + + string_set.UpdateEntries(std::move(data), nullptr, + [&result](bool res) { result = res; }); + + string_set.UpdateCallback(true); + EXPECT_TRUE(result); + data = std::make_unique::KeyEntryVector>(temp); + string_set.UpdateEntries(std::move(data), nullptr, + [&result](bool res) { result = res; }); + string_set.UpdateCallback(false); + EXPECT_FALSE(result); +} + +TEST(FakeDataSet, TestLoadEntries) { + std::vector result = {}; + FakeDataSet string_set({}); + + auto temp = FakeDataSet::KeyEntryVector( + {{"id1", "string1"}, {"id2", "string2"}}); + auto data = std::make_unique::KeyEntryVector>(temp); + + string_set.UpdateEntries(std::move(data), nullptr, [](bool ans) {}); + string_set.UpdateCallback(true); + string_set.LoadEntries( + [&result](bool ans, std::unique_ptr> res) { + auto it = res->begin(); + while (it != res->end()) { + result.push_back(*it); + ++it; + } + }); + string_set.LoadCallback(true); + EXPECT_THAT(result, testing::SizeIs(2)); + std::sort(result.begin(), result.end()); + EXPECT_EQ(result, std::vector({"string1", "string2"})); +} + +TEST(MockDataSet, TestDestroy) { + bool result; + std::vector data = {}; + FakeDataSet string_set({{"id1", "string1"}, {"id2", "string2"}}); + string_set.Destroy([&result](bool res) { result = res; }); + string_set.DestroyCallback(true); + EXPECT_TRUE(result); + string_set.LoadEntries( + [&data](bool ans, std::unique_ptr> res) { + auto it = res->begin(); + while (it != res->end()) { + data.push_back(*it); + ++it; + } + }); + string_set.LoadCallback(true); + EXPECT_THAT(data, ::testing::SizeIs(0)); +} + +} // namespace +} // namespace data +} // namespace nearby