From cc069f25b31be9be30a8831910df983c74c82b21 Mon Sep 17 00:00:00 2001 From: anand76 Date: Fri, 15 Dec 2023 11:34:08 -0800 Subject: [PATCH] Add some compressed and tiered secondary cache stats (#12150) Summary: Add statistics for more visibility. Pull Request resolved: https://github.com/facebook/rocksdb/pull/12150 Reviewed By: akankshamahajan15 Differential Revision: D52184633 Pulled By: anand1976 fbshipit-source-id: 9969e05d65223811cd12627102b020bb6d229352 --- cache/compressed_secondary_cache.cc | 4 ++- cache/compressed_secondary_cache.h | 2 +- cache/compressed_secondary_cache_test.cc | 26 ++++++++++---------- cache/lru_cache_test.cc | 3 ++- cache/secondary_cache_adapter.cc | 11 +++++---- cache/tiered_secondary_cache.cc | 18 ++++++++++---- cache/tiered_secondary_cache.h | 3 ++- cache/tiered_secondary_cache_test.cc | 2 +- db/blob/blob_source_test.cc | 4 +-- include/rocksdb/secondary_cache.h | 6 ++--- include/rocksdb/statistics.h | 6 +++++ java/rocksjni/portal.h | 20 +++++++++++++++ monitoring/statistics.cc | 8 ++++++ options/customizable_test.cc | 3 ++- utilities/fault_injection_secondary_cache.cc | 8 +++--- utilities/fault_injection_secondary_cache.h | 2 +- 16 files changed, 88 insertions(+), 38 deletions(-) diff --git a/cache/compressed_secondary_cache.cc b/cache/compressed_secondary_cache.cc index b29670b7730..6c19e7921f1 100644 --- a/cache/compressed_secondary_cache.cc +++ b/cache/compressed_secondary_cache.cc @@ -31,7 +31,7 @@ CompressedSecondaryCache::~CompressedSecondaryCache() {} std::unique_ptr CompressedSecondaryCache::Lookup( const Slice& key, const Cache::CacheItemHelper* helper, Cache::CreateContext* create_context, bool /*wait*/, bool advise_erase, - bool& kept_in_sec_cache) { + Statistics* stats, bool& kept_in_sec_cache) { assert(helper); // This is a minor optimization. Its ok to skip it in TSAN in order to // avoid a false positive. @@ -51,6 +51,7 @@ std::unique_ptr CompressedSecondaryCache::Lookup( void* handle_value = cache_->Value(lru_handle); if (handle_value == nullptr) { cache_->Release(lru_handle, /*erase_if_last_ref=*/false); + RecordTick(stats, COMPRESSED_SECONDARY_CACHE_DUMMY_HITS); return nullptr; } @@ -137,6 +138,7 @@ std::unique_ptr CompressedSecondaryCache::Lookup( cache_->Release(lru_handle, /*erase_if_last_ref=*/false); } handle.reset(new CompressedSecondaryCacheResultHandle(value, charge)); + RecordTick(stats, COMPRESSED_SECONDARY_CACHE_HITS); return handle; } diff --git a/cache/compressed_secondary_cache.h b/cache/compressed_secondary_cache.h index 32e6fd0df9b..90e134fcf51 100644 --- a/cache/compressed_secondary_cache.h +++ b/cache/compressed_secondary_cache.h @@ -86,7 +86,7 @@ class CompressedSecondaryCache : public SecondaryCache { std::unique_ptr Lookup( const Slice& key, const Cache::CacheItemHelper* helper, Cache::CreateContext* create_context, bool /*wait*/, bool advise_erase, - bool& kept_in_sec_cache) override; + Statistics* stats, bool& kept_in_sec_cache) override; bool SupportForceErase() const override { return true; } diff --git a/cache/compressed_secondary_cache_test.cc b/cache/compressed_secondary_cache_test.cc index 79f40868a7e..ac1786f160c 100644 --- a/cache/compressed_secondary_cache_test.cc +++ b/cache/compressed_secondary_cache_test.cc @@ -44,7 +44,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test, // Lookup an non-existent key. std::unique_ptr handle0 = sec_cache->Lookup(key0, GetHelper(), this, true, /*advise_erase=*/true, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_EQ(handle0, nullptr); Random rnd(301); @@ -59,7 +59,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test, std::unique_ptr handle1_1 = sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/false, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_EQ(handle1_1, nullptr); // Insert and Lookup the item k1 for the second time and advise erasing it. @@ -68,7 +68,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test, std::unique_ptr handle1_2 = sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/true, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_NE(handle1_2, nullptr); ASSERT_FALSE(kept_in_sec_cache); if (sec_cache_is_compressed) { @@ -89,7 +89,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test, // Lookup the item k1 again. std::unique_ptr handle1_3 = sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/true, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_EQ(handle1_3, nullptr); // Insert and Lookup the item k2. @@ -99,7 +99,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test, ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 2); std::unique_ptr handle2_1 = sec_cache->Lookup(key2, GetHelper(), this, true, /*advise_erase=*/false, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_EQ(handle2_1, nullptr); ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper(), false)); @@ -115,7 +115,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test, } std::unique_ptr handle2_2 = sec_cache->Lookup(key2, GetHelper(), this, true, /*advise_erase=*/false, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_NE(handle2_2, nullptr); std::unique_ptr val2 = std::unique_ptr(static_cast(handle2_2->Value())); @@ -196,14 +196,14 @@ class CompressedSecondaryCacheTestBase : public testing::Test, bool kept_in_sec_cache{false}; std::unique_ptr handle1 = sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/false, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_EQ(handle1, nullptr); // Insert k2 and k1 is evicted. ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper(), false)); std::unique_ptr handle2 = sec_cache->Lookup(key2, GetHelper(), this, true, /*advise_erase=*/false, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_NE(handle2, nullptr); std::unique_ptr val2 = std::unique_ptr(static_cast(handle2->Value())); @@ -215,14 +215,14 @@ class CompressedSecondaryCacheTestBase : public testing::Test, std::unique_ptr handle1_1 = sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/false, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_EQ(handle1_1, nullptr); // Create Fails. SetFailCreate(true); std::unique_ptr handle2_1 = sec_cache->Lookup(key2, GetHelper(), this, true, /*advise_erase=*/true, - kept_in_sec_cache); + /*stats=*/nullptr, kept_in_sec_cache); ASSERT_EQ(handle2_1, nullptr); // Save Fails. @@ -912,9 +912,9 @@ TEST_P(CompressedSecondaryCacheTestWithCompressionParam, EntryRoles) { ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 1U); bool kept_in_sec_cache{true}; - std::unique_ptr handle = - sec_cache->Lookup(ith_key, GetHelper(role), this, true, - /*advise_erase=*/true, kept_in_sec_cache); + std::unique_ptr handle = sec_cache->Lookup( + ith_key, GetHelper(role), this, true, + /*advise_erase=*/true, /*stats=*/nullptr, kept_in_sec_cache); ASSERT_NE(handle, nullptr); // Lookup returns the right data diff --git a/cache/lru_cache_test.cc b/cache/lru_cache_test.cc index 91b1d02c110..9f70a54cf8b 100644 --- a/cache/lru_cache_test.cc +++ b/cache/lru_cache_test.cc @@ -1091,7 +1091,8 @@ class TestSecondaryCache : public SecondaryCache { std::unique_ptr Lookup( const Slice& key, const Cache::CacheItemHelper* helper, Cache::CreateContext* create_context, bool /*wait*/, - bool /*advise_erase*/, bool& kept_in_sec_cache) override { + bool /*advise_erase*/, Statistics* /*stats*/, + bool& kept_in_sec_cache) override { std::string key_str = key.ToString(); TEST_SYNC_POINT_CALLBACK("TestSecondaryCache::Lookup", &key_str); diff --git a/cache/secondary_cache_adapter.cc b/cache/secondary_cache_adapter.cc index dce18390ef4..6261b8ce6e7 100644 --- a/cache/secondary_cache_adapter.cc +++ b/cache/secondary_cache_adapter.cc @@ -294,7 +294,8 @@ Cache::Handle* CacheWithSecondaryAdapter::Lookup(const Slice& key, bool kept_in_sec_cache = false; std::unique_ptr secondary_handle = secondary_cache_->Lookup(key, helper, create_context, /*wait*/ true, - found_dummy_entry, /*out*/ kept_in_sec_cache); + found_dummy_entry, stats, + /*out*/ kept_in_sec_cache); if (secondary_handle) { result = Promote(std::move(secondary_handle), key, helper, priority, stats, found_dummy_entry, kept_in_sec_cache); @@ -348,10 +349,10 @@ void CacheWithSecondaryAdapter::StartAsyncLookupOnMySecondary( assert(async_handle.result_handle == nullptr); std::unique_ptr secondary_handle = - secondary_cache_->Lookup(async_handle.key, async_handle.helper, - async_handle.create_context, /*wait*/ false, - async_handle.found_dummy_entry, - /*out*/ async_handle.kept_in_sec_cache); + secondary_cache_->Lookup( + async_handle.key, async_handle.helper, async_handle.create_context, + /*wait*/ false, async_handle.found_dummy_entry, async_handle.stats, + /*out*/ async_handle.kept_in_sec_cache); if (secondary_handle) { // TODO with stacked secondaries: Check & process if already ready? async_handle.pending_handle = secondary_handle.release(); diff --git a/cache/tiered_secondary_cache.cc b/cache/tiered_secondary_cache.cc index 1a1201a4d15..f7d5dd91d68 100644 --- a/cache/tiered_secondary_cache.cc +++ b/cache/tiered_secondary_cache.cc @@ -5,6 +5,8 @@ #include "cache/tiered_secondary_cache.h" +#include "monitoring/statistics_impl.h" + namespace ROCKSDB_NAMESPACE { // Creation callback for use in the lookup path. It calls the upper layer @@ -29,6 +31,9 @@ Status TieredSecondaryCache::MaybeInsertAndCreate( // TODO: Don't hardcode the source context->comp_sec_cache->InsertSaved(*context->key, data, type, source) .PermitUncheckedError(); + RecordTick(context->stats, COMPRESSED_SECONDARY_CACHE_PROMOTIONS); + } else { + RecordTick(context->stats, COMPRESSED_SECONDARY_CACHE_PROMOTION_SKIPS); } // Primary cache will accept the object, so call its helper to create // the object @@ -43,10 +48,10 @@ Status TieredSecondaryCache::MaybeInsertAndCreate( std::unique_ptr TieredSecondaryCache::Lookup( const Slice& key, const Cache::CacheItemHelper* helper, Cache::CreateContext* create_context, bool wait, bool advise_erase, - bool& kept_in_sec_cache) { + Statistics* stats, bool& kept_in_sec_cache) { bool dummy = false; std::unique_ptr result = - target()->Lookup(key, helper, create_context, wait, advise_erase, + target()->Lookup(key, helper, create_context, wait, advise_erase, stats, /*kept_in_sec_cache=*/dummy); // We never want the item to spill back into the secondary cache kept_in_sec_cache = true; @@ -66,9 +71,10 @@ std::unique_ptr TieredSecondaryCache::Lookup( ctx.helper = helper; ctx.inner_ctx = create_context; ctx.comp_sec_cache = target(); + ctx.stats = stats; return nvm_sec_cache_->Lookup(key, outer_helper, &ctx, wait, advise_erase, - kept_in_sec_cache); + stats, kept_in_sec_cache); } // If wait is false, i.e its an async lookup, we have to allocate a result @@ -80,8 +86,10 @@ std::unique_ptr TieredSecondaryCache::Lookup( handle->ctx()->helper = helper; handle->ctx()->inner_ctx = create_context; handle->ctx()->comp_sec_cache = target(); - handle->SetInnerHandle(nvm_sec_cache_->Lookup( - key, outer_helper, handle->ctx(), wait, advise_erase, kept_in_sec_cache)); + handle->ctx()->stats = stats; + handle->SetInnerHandle( + nvm_sec_cache_->Lookup(key, outer_helper, handle->ctx(), wait, + advise_erase, stats, kept_in_sec_cache)); if (!handle->inner_handle()) { handle.reset(); } else { diff --git a/cache/tiered_secondary_cache.h b/cache/tiered_secondary_cache.h index 46e3eb08431..80542ba4973 100644 --- a/cache/tiered_secondary_cache.h +++ b/cache/tiered_secondary_cache.h @@ -59,7 +59,7 @@ class TieredSecondaryCache : public SecondaryCacheWrapper { virtual std::unique_ptr Lookup( const Slice& key, const Cache::CacheItemHelper* helper, Cache::CreateContext* create_context, bool wait, bool advise_erase, - bool& kept_in_sec_cache) override; + Statistics* stats, bool& kept_in_sec_cache) override; virtual void WaitAll( std::vector handles) override; @@ -72,6 +72,7 @@ class TieredSecondaryCache : public SecondaryCacheWrapper { Cache::CreateContext* inner_ctx; std::shared_ptr inner_handle; SecondaryCache* comp_sec_cache; + Statistics* stats; }; class ResultHandle : public SecondaryCacheResultHandle { diff --git a/cache/tiered_secondary_cache_test.cc b/cache/tiered_secondary_cache_test.cc index d641254df70..28a393325ed 100644 --- a/cache/tiered_secondary_cache_test.cc +++ b/cache/tiered_secondary_cache_test.cc @@ -62,7 +62,7 @@ class TestSecondaryCache : public SecondaryCache { std::unique_ptr Lookup( const Slice& key, const Cache::CacheItemHelper* helper, Cache::CreateContext* create_context, bool wait, bool /*advise_erase*/, - bool& kept_in_sec_cache) override { + Statistics* /*stats*/, bool& kept_in_sec_cache) override { std::string key_str = key.ToString(); TEST_SYNC_POINT_CALLBACK("TestSecondaryCache::Lookup", &key_str); diff --git a/db/blob/blob_source_test.cc b/db/blob/blob_source_test.cc index c0e1aba6ec0..258d2da5e17 100644 --- a/db/blob/blob_source_test.cc +++ b/db/blob/blob_source_test.cc @@ -1220,7 +1220,7 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) { auto sec_handle0 = secondary_cache->Lookup( key0, BlobSource::SharedCacheInterface::GetFullHelper(), /*context*/ nullptr, true, - /*advise_erase=*/true, kept_in_sec_cache); + /*advise_erase=*/true, /*stats=*/nullptr, kept_in_sec_cache); ASSERT_FALSE(kept_in_sec_cache); ASSERT_NE(sec_handle0, nullptr); ASSERT_TRUE(sec_handle0->IsReady()); @@ -1248,7 +1248,7 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) { auto sec_handle1 = secondary_cache->Lookup( key1, BlobSource::SharedCacheInterface::GetFullHelper(), /*context*/ nullptr, true, - /*advise_erase=*/true, kept_in_sec_cache); + /*advise_erase=*/true, /*stats=*/nullptr, kept_in_sec_cache); ASSERT_FALSE(kept_in_sec_cache); ASSERT_EQ(sec_handle1, nullptr); diff --git a/include/rocksdb/secondary_cache.h b/include/rocksdb/secondary_cache.h index 49792ca67a5..b0419b12172 100644 --- a/include/rocksdb/secondary_cache.h +++ b/include/rocksdb/secondary_cache.h @@ -114,7 +114,7 @@ class SecondaryCache : public Customizable { virtual std::unique_ptr Lookup( const Slice& key, const Cache::CacheItemHelper* helper, Cache::CreateContext* create_context, bool wait, bool advise_erase, - bool& kept_in_sec_cache) = 0; + Statistics* stats, bool& kept_in_sec_cache) = 0; // Indicate whether a handle can be erased in this secondary cache. [[nodiscard]] virtual bool SupportForceErase() const = 0; @@ -176,9 +176,9 @@ class SecondaryCacheWrapper : public SecondaryCache { virtual std::unique_ptr Lookup( const Slice& key, const Cache::CacheItemHelper* helper, Cache::CreateContext* create_context, bool wait, bool advise_erase, - bool& kept_in_sec_cache) override { + Statistics* stats, bool& kept_in_sec_cache) override { return target()->Lookup(key, helper, create_context, wait, advise_erase, - kept_in_sec_cache); + stats, kept_in_sec_cache); } virtual bool SupportForceErase() const override { diff --git a/include/rocksdb/statistics.h b/include/rocksdb/statistics.h index ecddf5c7a94..9aab337124f 100644 --- a/include/rocksdb/statistics.h +++ b/include/rocksdb/statistics.h @@ -531,6 +531,12 @@ enum Tickers : uint32_t { // Number of FS reads avoided due to scan prefetching PREFETCH_HITS, + // Compressed secondary cache related stats + COMPRESSED_SECONDARY_CACHE_DUMMY_HITS, + COMPRESSED_SECONDARY_CACHE_HITS, + COMPRESSED_SECONDARY_CACHE_PROMOTIONS, + COMPRESSED_SECONDARY_CACHE_PROMOTION_SKIPS, + TICKER_ENUM_MAX }; diff --git a/java/rocksjni/portal.h b/java/rocksjni/portal.h index 840956dae9b..45d0c184c88 100644 --- a/java/rocksjni/portal.h +++ b/java/rocksjni/portal.h @@ -5175,6 +5175,15 @@ class TickerTypeJni { return -0x41; case ROCKSDB_NAMESPACE::Tickers::PREFETCH_HITS: return -0x42; + case ROCKSDB_NAMESPACE::Tickers::COMPRESSED_SECONDARY_CACHE_DUMMY_HITS: + return -0x43; + case ROCKSDB_NAMESPACE::Tickers::COMPRESSED_SECONDARY_CACHE_HITS: + return -0x44; + case ROCKSDB_NAMESPACE::Tickers::COMPRESSED_SECONDARY_CACHE_PROMOTIONS: + return -0x45; + case ROCKSDB_NAMESPACE::Tickers:: + COMPRESSED_SECONDARY_CACHE_PROMOTION_SKIPS: + return -0x46; case ROCKSDB_NAMESPACE::Tickers::TICKER_ENUM_MAX: // 0x5F was the max value in the initial copy of tickers to Java. // Since these values are exposed directly to Java clients, we keep @@ -5550,6 +5559,17 @@ class TickerTypeJni { return ROCKSDB_NAMESPACE::Tickers::PREFETCH_BYTES_USEFUL; case -0x42: return ROCKSDB_NAMESPACE::Tickers::PREFETCH_HITS; + case -0x43: + return ROCKSDB_NAMESPACE::Tickers:: + COMPRESSED_SECONDARY_CACHE_DUMMY_HITS; + case -0x44: + return ROCKSDB_NAMESPACE::Tickers::COMPRESSED_SECONDARY_CACHE_HITS; + case -0x45: + return ROCKSDB_NAMESPACE::Tickers:: + COMPRESSED_SECONDARY_CACHE_PROMOTIONS; + case -0x46: + return ROCKSDB_NAMESPACE::Tickers:: + COMPRESSED_SECONDARY_CACHE_PROMOTION_SKIPS; case 0x5F: // 0x5F was the max value in the initial copy of tickers to Java. // Since these values are exposed directly to Java clients, we keep diff --git a/monitoring/statistics.cc b/monitoring/statistics.cc index d0a2a71a84d..cc679ec0a94 100644 --- a/monitoring/statistics.cc +++ b/monitoring/statistics.cc @@ -264,6 +264,14 @@ const std::vector> TickersNameMap = { {PREFETCH_BYTES, "rocksdb.prefetch.bytes"}, {PREFETCH_BYTES_USEFUL, "rocksdb.prefetch.bytes.useful"}, {PREFETCH_HITS, "rocksdb.prefetch.hits"}, + {COMPRESSED_SECONDARY_CACHE_DUMMY_HITS, + "rocksdb.compressed.secondary.cache.dummy.hits"}, + {COMPRESSED_SECONDARY_CACHE_HITS, + "rocksdb.compressed.secondary.cache.hits"}, + {COMPRESSED_SECONDARY_CACHE_PROMOTIONS, + "rocksdb.compressed.secondary.cache.promotions"}, + {COMPRESSED_SECONDARY_CACHE_PROMOTION_SKIPS, + "rocksdb.compressed.secondary.cache.promotion.skips"}, }; const std::vector> HistogramsNameMap = { diff --git a/options/customizable_test.cc b/options/customizable_test.cc index f597246563f..696f1b25edf 100644 --- a/options/customizable_test.cc +++ b/options/customizable_test.cc @@ -1241,7 +1241,8 @@ class TestSecondaryCache : public SecondaryCache { std::unique_ptr Lookup( const Slice& /*key*/, const Cache::CacheItemHelper* /*helper*/, Cache::CreateContext* /*create_context*/, bool /*wait*/, - bool /*advise_erase*/, bool& kept_in_sec_cache) override { + bool /*advise_erase*/, Statistics* /*stats*/, + bool& kept_in_sec_cache) override { kept_in_sec_cache = true; return nullptr; } diff --git a/utilities/fault_injection_secondary_cache.cc b/utilities/fault_injection_secondary_cache.cc index c2ea12535bc..fa93e8244d8 100644 --- a/utilities/fault_injection_secondary_cache.cc +++ b/utilities/fault_injection_secondary_cache.cc @@ -92,6 +92,7 @@ FaultInjectionSecondaryCache::Lookup(const Slice& key, const Cache::CacheItemHelper* helper, Cache::CreateContext* create_context, bool wait, bool advise_erase, + Statistics* stats, bool& kept_in_sec_cache) { ErrorContext* ctx = GetErrorContext(); if (base_is_compressed_sec_cache_) { @@ -99,11 +100,12 @@ FaultInjectionSecondaryCache::Lookup(const Slice& key, return nullptr; } else { return base_->Lookup(key, helper, create_context, wait, advise_erase, - kept_in_sec_cache); + stats, kept_in_sec_cache); } } else { - std::unique_ptr hdl = base_->Lookup( - key, helper, create_context, wait, advise_erase, kept_in_sec_cache); + std::unique_ptr hdl = + base_->Lookup(key, helper, create_context, wait, advise_erase, stats, + kept_in_sec_cache); if (wait && ctx->rand.OneIn(prob_)) { hdl.reset(); } diff --git a/utilities/fault_injection_secondary_cache.h b/utilities/fault_injection_secondary_cache.h index dd73ac15630..502478d4060 100644 --- a/utilities/fault_injection_secondary_cache.h +++ b/utilities/fault_injection_secondary_cache.h @@ -43,7 +43,7 @@ class FaultInjectionSecondaryCache : public SecondaryCache { std::unique_ptr Lookup( const Slice& key, const Cache::CacheItemHelper* helper, Cache::CreateContext* create_context, bool wait, bool advise_erase, - bool& kept_in_sec_cache) override; + Statistics* stats, bool& kept_in_sec_cache) override; bool SupportForceErase() const override { return base_->SupportForceErase(); }