Group :: System/Base
RPM: ceph
Navigation
Main Changelog Spec Patches Sources Download Gear Bugs and FR Repocop
Patch: ceph-16.2.11.patch
Download
Download
CMakeLists.txt | 2 ++
cmake/modules/Finduring.cmake | 2 +-
cmake/modules/Findzstd.cmake | 25 +++++++++++++++++
src/common/options.cc | 2 +-
src/compressor/zstd/CMakeLists.txt | 25 +----------------
src/compressor/zstd/ZstdCompressor.h | 2 +-
src/include/buffer.h | 4 ++-
src/include/rados/librados.h | 6 ----
src/init-ceph.in | 1 -
src/kv/rocksdb_cache/BinnedLRUCache.cc | 28 ++++++++++++++++---
src/kv/rocksdb_cache/BinnedLRUCache.h | 18 ++++++++----
src/kv/rocksdb_cache/ShardedCache.cc | 29 ++++++++++++++++++--
src/kv/rocksdb_cache/ShardedCache.h | 50 ++++++++++++++++++++++------------
13 files changed, 130 insertions(+), 64 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 78cae841bb6..ab25ce8743e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -334,6 +334,8 @@ if(WITH_LZ4)
set(HAVE_LZ4 ${LZ4_FOUND})endif(WITH_LZ4)
+find_package(zstd REQUIRED)
+
option(WITH_CEPH_DEBUG_MUTEX "Use debug ceph::mutex with lockdep" OFF)
#if allocator is set on command line make sure it matches below strings
diff --git a/cmake/modules/Finduring.cmake b/cmake/modules/Finduring.cmake
index 10c8de42550..8e11c273a2d 100644
--- a/cmake/modules/Finduring.cmake
+++ b/cmake/modules/Finduring.cmake
@@ -5,7 +5,7 @@
# uring_FOUND - True if uring found.
find_path(URING_INCLUDE_DIR liburing.h)
-find_library(URING_LIBRARIES liburing.a liburing)
+find_library(URING_LIBRARIES liburing.so liburing)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(uring DEFAULT_MSG URING_LIBRARIES URING_INCLUDE_DIR)
diff --git a/cmake/modules/Findzstd.cmake b/cmake/modules/Findzstd.cmake
new file mode 100644
index 00000000000..696bb7603e2
--- /dev/null
+++ b/cmake/modules/Findzstd.cmake
@@ -0,0 +1,25 @@
+# Try to find liblz4
+#
+# Once done, this will define
+#
+# ZSTD_FOUND
+# ZSTD_INCLUDE_DIR
+# ZSTD_LIBRARY
+
+find_path(ZSTD_INCLUDE_DIR zstd.h)
+
+find_library(ZSTD_LIBRARY zstd)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(zstd
+ DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+mark_as_advanced(ZSTD_INCLUDE_DIR ZSTD_LIBRARY)
+
+if(ZSTD_FOUND AND NOT (TARGET zstd::zstd))
+ add_library(zstd::zstd UNKNOWN IMPORTED)
+ set_target_properties(zstd::zstd PROPERTIES
+ INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_INCLUDE_DIR}"+ IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+ IMPORTED_LOCATION "${ZSTD_LIBRARY}")+endif()
diff --git a/src/common/options.cc b/src/common/options.cc
index b0239e24aa4..bff4154decd 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -5543,7 +5543,7 @@ std::vector<Option> get_global_options() { .add_see_also("mgr_module_path"), Option("mgr_initial_modules", Option::TYPE_STR, Option::LEVEL_BASIC)- .set_default("restful iostat nfs")+ .set_default("iostat").set_flag(Option::FLAG_NO_MON_UPDATE)
.set_flag(Option::FLAG_CLUSTER_CREATE)
.add_service("mon")diff --git a/src/compressor/zstd/CMakeLists.txt b/src/compressor/zstd/CMakeLists.txt
index b53b0d94448..2b60137d7e2 100644
--- a/src/compressor/zstd/CMakeLists.txt
+++ b/src/compressor/zstd/CMakeLists.txt
@@ -1,34 +1,11 @@
# zstd
-# libzstd - build it statically
-set(ZSTD_C_FLAGS "-fPIC -Wno-unused-variable -O3")
-
-include(ExternalProject)
-ExternalProject_Add(zstd_ext
- SOURCE_DIR ${CMAKE_SOURCE_DIR}/src/zstd/build/cmake- CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}- -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}- -DCMAKE_C_FLAGS=${ZSTD_C_FLAGS}- -DCMAKE_AR=${CMAKE_AR}- -DCMAKE_POSITION_INDEPENDENT_CODE=${ENABLE_SHARED}- -G${CMAKE_GENERATOR}- BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/libzstd- BUILD_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --target libzstd_static- BUILD_BYPRODUCTS "${CMAKE_CURRENT_BINARY_DIR}/libzstd/lib/libzstd.a"- INSTALL_COMMAND "true")
-
-add_library(zstd STATIC IMPORTED)
-set_target_properties(zstd PROPERTIES
- INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/src/zstd/lib"- IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/libzstd/lib/libzstd.a")-add_dependencies(zstd zstd_ext)
-
set(zstd_sources
CompressionPluginZstd.cc
)
add_library(ceph_zstd SHARED ${zstd_sources})-target_link_libraries(ceph_zstd PRIVATE zstd $<$<PLATFORM_ID:Windows>:ceph-common>)
+target_link_libraries(ceph_zstd PRIVATE zstd::zstd)
set_target_properties(ceph_zstd PROPERTIES
VERSION 2.0.0
SOVERSION 2
diff --git a/src/compressor/zstd/ZstdCompressor.h b/src/compressor/zstd/ZstdCompressor.h
index 95b492deb87..f4b66a14dcc 100644
--- a/src/compressor/zstd/ZstdCompressor.h
+++ b/src/compressor/zstd/ZstdCompressor.h
@@ -16,7 +16,7 @@
#define CEPH_ZSTDCOMPRESSOR_H
#define ZSTD_STATIC_LINKING_ONLY
-#include "zstd/lib/zstd.h"
+#include <zstd.h>
#include "include/buffer.h"
#include "include/encoding.h"
diff --git a/src/include/buffer.h b/src/include/buffer.h
index 71cb0193500..118af14accf 100644
--- a/src/include/buffer.h
+++ b/src/include/buffer.h
@@ -864,7 +864,9 @@ struct error_code;
if (first_round) {impl_f(first_round);
}
- if (const auto second_round = len - first_round; second_round) {+ // no C++17 for the sake of the C++11 guarantees of librados, sorry.
+ const auto second_round = len - first_round;
+ if (second_round) {_refill(second_round);
impl_f(second_round);
}
diff --git a/src/include/rados/librados.h b/src/include/rados/librados.h
index d9e39eceb1f..a27fb1002b5 100644
--- a/src/include/rados/librados.h
+++ b/src/include/rados/librados.h
@@ -3995,12 +3995,6 @@ CEPH_RADOS_API int rados_pg_command(rados_t cluster, const char *pgstr,
char **outbuf, size_t *outbuflen,
char **outs, size_t *outslen);
-CEPH_RADOS_API int rados_mgr_command(rados_t cluster,
- const char **cmd, size_t cmdlen,
- const char *inbuf, size_t inbuflen,
- char **outbuf, size_t *outbuflen,
- char **outs, size_t *outslen);
-
/*
* This is not a doxygen comment leadin, because doxygen breaks on
* a typedef with function params and returns, and I can't figure out
diff --git a/src/init-ceph.in b/src/init-ceph.in
index 76f0d59c1d8..5ca53954f54 100755
--- a/src/init-ceph.in
+++ b/src/init-ceph.in
@@ -174,7 +174,6 @@ use_default_conf=1
## set variables like cluster or conf
[ -e /etc/sysconfig/ceph ] && . /etc/sysconfig/ceph
-[ -e /etc/default/ceph ] && . /etc/default/ceph
while echo $1 | grep -q '^-'; do # FIXME: why not '^-'?
diff --git a/src/kv/rocksdb_cache/BinnedLRUCache.cc b/src/kv/rocksdb_cache/BinnedLRUCache.cc
index 0d657883e92..47c56e2ddd7 100644
--- a/src/kv/rocksdb_cache/BinnedLRUCache.cc
+++ b/src/kv/rocksdb_cache/BinnedLRUCache.cc
@@ -151,13 +151,20 @@ void BinnedLRUCacheShard::EraseUnRefEntries() {}
}
-void BinnedLRUCacheShard::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
- bool thread_safe) {+void BinnedLRUCacheShard::ApplyToAllCacheEntries(
+ const std::function<void(const rocksdb::Slice& key,
+ void* value,
+ size_t charge,
+ DeleterFn)>& callback,
+ bool thread_safe)
+{ if (thread_safe) {mutex_.lock();
}
table_.ApplyToAllCacheEntries(
- [callback](BinnedLRUHandle* h) { callback(h->value, h->charge); });+ [callback](BinnedLRUHandle* h) {+ callback(h->key(), h->value, h->charge, h->deleter);
+ });
if (thread_safe) {mutex_.unlock();
}
@@ -345,7 +352,7 @@ bool BinnedLRUCacheShard::Release(rocksdb::Cache::Handle* handle, bool force_era
rocksdb::Status BinnedLRUCacheShard::Insert(const rocksdb::Slice& key, uint32_t hash, void* value,
size_t charge,
- void (*deleter)(const rocksdb::Slice& key, void* value),
+ DeleterFn deleter,
rocksdb::Cache::Handle** handle, rocksdb::Cache::Priority priority) {auto e = new BinnedLRUHandle();
rocksdb::Status s;
@@ -464,6 +471,12 @@ std::string BinnedLRUCacheShard::GetPrintableOptions() const {return std::string(buffer);
}
+DeleterFn BinnedLRUCacheShard::GetDeleter(rocksdb::Cache::Handle* h) const
+{+ auto* handle = reinterpret_cast<BinnedLRUHandle*>(h);
+ return handle->deleter;
+}
+
BinnedLRUCache::BinnedLRUCache(CephContext *c,
size_t capacity,
int num_shard_bits,
@@ -519,6 +532,13 @@ void BinnedLRUCache::DisownData() {#endif // !__SANITIZE_ADDRESS__
}
+#if (ROCKSDB_MAJOR >= 6 && ROCKSDB_MINOR >= 22)
+DeleterFn BinnedLRUCache::GetDeleter(Handle* handle) const
+{+ return reinterpret_cast<const BinnedLRUHandle*>(handle)->deleter;
+}
+#endif
+
size_t BinnedLRUCache::TEST_GetLRUSize() {size_t lru_size_of_all_shards = 0;
for (int i = 0; i < num_shards_; i++) {diff --git a/src/kv/rocksdb_cache/BinnedLRUCache.h b/src/kv/rocksdb_cache/BinnedLRUCache.h
index 85608be0e57..88bf4502e89 100644
--- a/src/kv/rocksdb_cache/BinnedLRUCache.h
+++ b/src/kv/rocksdb_cache/BinnedLRUCache.h
@@ -56,7 +56,7 @@ std::shared_ptr<rocksdb::Cache> NewBinnedLRUCache(
struct BinnedLRUHandle {void* value;
- void (*deleter)(const rocksdb::Slice&, void* value);
+ DeleterFn deleter;
BinnedLRUHandle* next_hash;
BinnedLRUHandle* next;
BinnedLRUHandle* prev;
@@ -189,7 +189,7 @@ class alignas(CACHE_LINE_SIZE) BinnedLRUCacheShard : public CacheShard {// Like Cache methods, but with an extra "hash" parameter.
virtual rocksdb::Status Insert(const rocksdb::Slice& key, uint32_t hash, void* value,
size_t charge,
- void (*deleter)(const rocksdb::Slice& key, void* value),
+ DeleterFn deleter,
rocksdb::Cache::Handle** handle,
rocksdb::Cache::Priority priority) override;
virtual rocksdb::Cache::Handle* Lookup(const rocksdb::Slice& key, uint32_t hash) override;
@@ -205,13 +205,19 @@ class alignas(CACHE_LINE_SIZE) BinnedLRUCacheShard : public CacheShard {virtual size_t GetUsage() const override;
virtual size_t GetPinnedUsage() const override;
- virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
- bool thread_safe) override;
+ virtual void ApplyToAllCacheEntries(
+ const std::function<void(const rocksdb::Slice& key,
+ void* value,
+ size_t charge,
+ DeleterFn)>& callback,
+ bool thread_safe) override;
virtual void EraseUnRefEntries() override;
virtual std::string GetPrintableOptions() const override;
+ virtual DeleterFn GetDeleter(rocksdb::Cache::Handle* handle) const override;
+
void TEST_GetLRUList(BinnedLRUHandle** lru, BinnedLRUHandle** lru_low_pri);
// Retrieves number of elements in LRU, for unit test purpose only
@@ -304,7 +310,9 @@ class BinnedLRUCache : public ShardedCache {virtual size_t GetCharge(Handle* handle) const override;
virtual uint32_t GetHash(Handle* handle) const override;
virtual void DisownData() override;
-
+#if (ROCKSDB_MAJOR >= 6 && ROCKSDB_MINOR >= 22)
+ virtual DeleterFn GetDeleter(Handle* handle) const override;
+#endif
// Retrieves number of elements in LRU, for unit test purpose only
size_t TEST_GetLRUSize();
// Sets the high pri pool ratio
diff --git a/src/kv/rocksdb_cache/ShardedCache.cc b/src/kv/rocksdb_cache/ShardedCache.cc
index 367140a94d8..6cbd89ad647 100644
--- a/src/kv/rocksdb_cache/ShardedCache.cc
+++ b/src/kv/rocksdb_cache/ShardedCache.cc
@@ -44,7 +44,7 @@ void ShardedCache::SetStrictCapacityLimit(bool strict_capacity_limit) {}
rocksdb::Status ShardedCache::Insert(const rocksdb::Slice& key, void* value, size_t charge,
- void (*deleter)(const rocksdb::Slice& key, void* value),
+ DeleterFn deleter,
rocksdb::Cache::Handle** handle, Priority priority) {uint32_t hash = HashSlice(key);
return GetShard(Shard(hash))
@@ -109,13 +109,36 @@ size_t ShardedCache::GetPinnedUsage() const {return usage;
}
+#if (ROCKSDB_MAJOR >= 6 && ROCKSDB_MINOR >= 22)
+DeleterFn ShardedCache::GetDeleter(Handle* handle) const
+{+ uint32_t hash = GetHash(handle);
+ return GetShard(Shard(hash))->GetDeleter(handle);
+}
+
+void ShardedCache::ApplyToAllEntries(
+ const std::function<void(const rocksdb::Slice& key, void* value, size_t charge,
+ DeleterFn deleter)>& callback,
+ const ApplyToAllEntriesOptions& opts)
+{+ int num_shards = 1 << num_shard_bits_;
+ for (int s = 0; s < num_shards; s++) {+ GetShard(s)->ApplyToAllCacheEntries(callback, true /* thread_safe */);
+ }
+}
+#else
void ShardedCache::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
bool thread_safe) {int num_shards = 1 << num_shard_bits_;
for (int s = 0; s < num_shards; s++) {- GetShard(s)->ApplyToAllCacheEntries(callback, thread_safe);
+ GetShard(s)->ApplyToAllCacheEntries(
+ [callback](const rocksdb::Slice&, void* value, size_t charge, DeleterFn) {+ callback(value, charge);
+ },
+ thread_safe);
}
}
+#endif
void ShardedCache::EraseUnRefEntries() {int num_shards = 1 << num_shard_bits_;
@@ -131,7 +154,7 @@ std::string ShardedCache::GetPrintableOptions() const {char buffer[kBufferSize];
{std::lock_guard<std::mutex> l(capacity_mutex_);
- snprintf(buffer, kBufferSize, " capacity : %" ROCKSDB_PRIszt "\n",
+ snprintf(buffer, kBufferSize, " capacity : %zu\n",
capacity_);
ret.append(buffer);
snprintf(buffer, kBufferSize, " num_shard_bits : %d\n", num_shard_bits_);
diff --git a/src/kv/rocksdb_cache/ShardedCache.h b/src/kv/rocksdb_cache/ShardedCache.h
index 4d64893ab1c..f98421a09a3 100644
--- a/src/kv/rocksdb_cache/ShardedCache.h
+++ b/src/kv/rocksdb_cache/ShardedCache.h
@@ -14,6 +14,7 @@
#include <string>
#include <mutex>
+#include "rocksdb/version.h"
#include "rocksdb/cache.h"
#include "include/ceph_hash.h"
#include "common/PriorityCache.h"
@@ -22,10 +23,11 @@
#ifndef CACHE_LINE_SIZE
#define CACHE_LINE_SIZE 64 // XXX arch-specific define
#endif
-#define ROCKSDB_PRIszt "zu"
namespace rocksdb_cache {+using DeleterFn = void (*)(const rocksdb::Slice& key, void* value);
+
// Single cache shard interface.
class CacheShard {public:
@@ -34,7 +36,7 @@ class CacheShard {virtual rocksdb::Status Insert(const rocksdb::Slice& key, uint32_t hash, void* value,
size_t charge,
- void (*deleter)(const rocksdb::Slice& key, void* value),
+ DeleterFn deleter,
rocksdb::Cache::Handle** handle, rocksdb::Cache::Priority priority) = 0;
virtual rocksdb::Cache::Handle* Lookup(const rocksdb::Slice& key, uint32_t hash) = 0;
virtual bool Ref(rocksdb::Cache::Handle* handle) = 0;
@@ -44,10 +46,15 @@ class CacheShard {virtual void SetStrictCapacityLimit(bool strict_capacity_limit) = 0;
virtual size_t GetUsage() const = 0;
virtual size_t GetPinnedUsage() const = 0;
- virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
- bool thread_safe) = 0;
+ virtual void ApplyToAllCacheEntries(
+ const std::function<void(const rocksdb::Slice& key,
+ void* value,
+ size_t charge,
+ DeleterFn)>& callback,
+ bool thread_safe) = 0;
virtual void EraseUnRefEntries() = 0;
virtual std::string GetPrintableOptions() const { return ""; }+ virtual DeleterFn GetDeleter(rocksdb::Cache::Handle* handle) const = 0;
};
// Generic cache interface which shards cache by hash of keys. 2^num_shard_bits
@@ -57,34 +64,43 @@ class ShardedCache : public rocksdb::Cache, public PriorityCache::PriCache {public:
ShardedCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit);
virtual ~ShardedCache() = default;
+ // rocksdb::Cache
virtual const char* Name() const override = 0;
- virtual CacheShard* GetShard(int shard) = 0;
- virtual const CacheShard* GetShard(int shard) const = 0;
- virtual void* Value(Handle* handle) override = 0;
- virtual size_t GetCharge(Handle* handle) const = 0;
- virtual uint32_t GetHash(Handle* handle) const = 0;
- virtual void DisownData() override = 0;
-
- virtual void SetCapacity(size_t capacity) override;
- virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
-
virtual rocksdb::Status Insert(const rocksdb::Slice& key, void* value, size_t charge,
- void (*deleter)(const rocksdb::Slice& key, void* value),
+ DeleterFn,
rocksdb::Cache::Handle** handle, Priority priority) override;
virtual rocksdb::Cache::Handle* Lookup(const rocksdb::Slice& key, rocksdb::Statistics* stats) override;
virtual bool Ref(rocksdb::Cache::Handle* handle) override;
virtual bool Release(rocksdb::Cache::Handle* handle, bool force_erase = false) override;
+ virtual void* Value(Handle* handle) override = 0;
virtual void Erase(const rocksdb::Slice& key) override;
virtual uint64_t NewId() override;
- virtual size_t GetCapacity() const override;
+ virtual void SetCapacity(size_t capacity) override;
+ virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
virtual bool HasStrictCapacityLimit() const override;
+ virtual size_t GetCapacity() const override;
virtual size_t GetUsage() const override;
virtual size_t GetUsage(rocksdb::Cache::Handle* handle) const override;
virtual size_t GetPinnedUsage() const override;
+ virtual size_t GetCharge(Handle* handle) const = 0;
+#if (ROCKSDB_MAJOR >= 6 && ROCKSDB_MINOR >= 22)
+ virtual DeleterFn GetDeleter(Handle* handle) const override;
+#endif
+ virtual void DisownData() override = 0;
+#if (ROCKSDB_MAJOR >= 6 && ROCKSDB_MINOR >= 22)
+ virtual void ApplyToAllEntries(
+ const std::function<void(const rocksdb::Slice& key, void* value, size_t charge,
+ DeleterFn deleter)>& callback,
+ const ApplyToAllEntriesOptions& opts) override;
+#else
virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
bool thread_safe) override;
+#endif
virtual void EraseUnRefEntries() override;
virtual std::string GetPrintableOptions() const override;
+ virtual CacheShard* GetShard(int shard) = 0;
+ virtual const CacheShard* GetShard(int shard) const = 0;
+ virtual uint32_t GetHash(Handle* handle) const = 0;
int GetNumShardBits() const { return num_shard_bits_; }@@ -120,7 +136,7 @@ class ShardedCache : public rocksdb::Cache, public PriorityCache::PriCache {// return Hash(s.data(), s.size(), 0);
}
- uint32_t Shard(uint32_t hash) {+ uint32_t Shard(uint32_t hash) const {// Note, hash >> 32 yields hash in gcc, not the zero we expect!
return (num_shard_bits_ > 0) ? (hash >> (32 - num_shard_bits_)) : 0;
}