.gear/clickhouse-avro-gcc10-compat.patch | 12 + .gear/clickhouse-base64-ppc64le.patch | 26 + .gear/clickhouse-fastops-gcc-compat.patch | 15 + .gear/clickhouse-grpc-abseil-cxx17-compat.patch | 26 + .gear/clickhouse-llvm-compat.patch | 27 + .gear/clickhouse-nanodbc-compat.patch | 12 + .gear/clickhouse-system-libuv.patch | 14 + .gear/clickhouse.spec | 542 +++++++++++++++++++++ .gear/clickhouse.watch | 3 + .gear/rules | 51 ++ .gear/submodules | 45 ++ .../tags/8a9f32ea051bbb6a7d29591944c50876949cb763 | 6 + .gear/tags/list | 46 ++ .gear/update-revision.sh | 36 ++ .gear/update-submodules.sh | 64 +++ CMakeLists.txt | 6 + cmake/autogenerated_versions.txt | 6 +- cmake/cpu_features.cmake | 2 +- cmake/find/cassandra.cmake | 5 - cmake/find/libuv.cmake | 6 +- cmake/find/rdkafka.cmake | 2 +- cmake/find/unwind.cmake | 4 +- cmake/linux/default_libs.cmake | 4 +- contrib/CMakeLists.txt | 3 - contrib/abseil-cpp-cmake/CMakeLists.txt | 12 +- contrib/cppkafka-cmake/CMakeLists.txt | 5 + contrib/fmtlib-cmake/CMakeLists.txt | 5 + contrib/jemalloc-cmake/CMakeLists.txt | 128 +---- contrib/libcpuid-cmake/CMakeLists.txt | 23 +- contrib/nuraft-cmake/CMakeLists.txt | 4 + contrib/yaml-cpp-cmake/CMakeLists.txt | 5 + debian/clickhouse-server.service | 4 +- programs/server/Server.cpp | 4 - src/Common/ThreadFuzzer.cpp | 272 ----------- src/Common/ThreadFuzzer.h | 72 --- src/Common/ThreadStatus.cpp | 5 +- src/Common/examples/chaos_sanitizer.cpp | 3 +- src/Common/malloc.cpp | 2 +- src/Common/ya.make | 1 - src/Functions/hasThreadFuzzer.cpp | 53 -- src/Functions/registerFunctionsMiscellaneous.cpp | 2 - src/Functions/ya.make | 1 - .../queries/0_stateless/01193_metadata_loading.sh | 2 +- 43 files changed, 977 insertions(+), 589 deletions(-) diff --git a/.gear/clickhouse-avro-gcc10-compat.patch b/.gear/clickhouse-avro-gcc10-compat.patch new file mode 100644 index 00000000000..0438e908284 --- /dev/null +++ b/.gear/clickhouse-avro-gcc10-compat.patch @@ -0,0 +1,12 @@ +diff --git a/lang/c++/api/Zigzag.hh b/lang/c++/api/Zigzag.hh +index bf831d24..7ce5c2cf 100644 +--- a/lang/c++/api/Zigzag.hh ++++ b/lang/c++/api/Zigzag.hh +@@ -20,6 +20,7 @@ + #define avro_Encoding_hh__ + + #include ++#include + #include + + #include "Config.hh" diff --git a/.gear/clickhouse-base64-ppc64le.patch b/.gear/clickhouse-base64-ppc64le.patch new file mode 100644 index 00000000000..f83dccd4ab1 --- /dev/null +++ b/.gear/clickhouse-base64-ppc64le.patch @@ -0,0 +1,26 @@ +diff --git a/tb64app.c b/tb64app.c +index 488720b..b3dfa7f 100644 +--- a/tb64app.c ++++ b/tb64app.c +@@ -134,7 +134,7 @@ unsigned bench(unsigned char *in, unsigned n, unsigned char *out, unsigned char + switch(id) { + case 1: TMBENCH("",l=tb64senc( in, n, out),n); pr(l,n); TMBENCH2("tb64s", tb64sdec(out, l, cpy), l); break; + case 2: TMBENCH("",l=tb64xenc( in, n, out),n); pr(l,n); TMBENCH2("tb64x", tb64xdec( out, l, cpy), l); break; +- #if defined(__i386__) || defined(__x86_64__) || defined(__ARM_NEON) || defined(__powerpc64__) ++ #if defined(__i386__) || defined(__x86_64__) || defined(__ARM_NEON) + case 3:if(cpuini(0)>=0x33) { TMBENCH("",l=tb64sseenc( in, n, out),n); pr(l,n); TMBENCH2("tb64sse", tb64ssedec(out, l, cpy), l); } break; + case 4: TMBENCH("",l=tb64enc( in, n, out),n); pr(l,n); TMBENCH2("tb64auto", tb64dec(out, l, cpy), l); break; + #else +diff --git a/turbob64sse.c b/turbob64sse.c +index d539f2d..6c5ba9c 100644 +--- a/turbob64sse.c ++++ b/turbob64sse.c +@@ -503,7 +503,7 @@ void tb64ini(unsigned id, unsigned isshort) { + } else + #endif + #endif +- #if defined(__i386__) || defined(__x86_64__) || defined(__ARM_NEON) || defined(__powerpc64__) ++ #if defined(__i386__) || defined(__x86_64__) || defined(__ARM_NEON) + #ifndef NO_SSE + if(i >= IS_SSSE3) { + _tb64e = tb64sseenc; diff --git a/.gear/clickhouse-fastops-gcc-compat.patch b/.gear/clickhouse-fastops-gcc-compat.patch new file mode 100644 index 00000000000..aa3a4203608 --- /dev/null +++ b/.gear/clickhouse-fastops-gcc-compat.patch @@ -0,0 +1,15 @@ +diff --git a/fastops/core/SIMDFunctions.h b/fastops/core/SIMDFunctions.h +index 7341933..85819e5 100644 +--- a/fastops/core/SIMDFunctions.h ++++ b/fastops/core/SIMDFunctions.h +@@ -14,6 +14,10 @@ + # define Y_HAVE_NEW_INTRINSICS + #endif + ++#if __GNUC__ > 10 ++# define Y_HAVE_NEW_INTRINSICS ++#endif ++ + #if !defined(Y_HAVE_NEW_INTRINSICS) + static __inline__ __m128i + _mm_loadu_si32(void const* __a) { diff --git a/.gear/clickhouse-grpc-abseil-cxx17-compat.patch b/.gear/clickhouse-grpc-abseil-cxx17-compat.patch new file mode 100644 index 00000000000..a20ac597f87 --- /dev/null +++ b/.gear/clickhouse-grpc-abseil-cxx17-compat.patch @@ -0,0 +1,26 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index c667c50007..8ce5ba91cd 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -188,21 +188,6 @@ if (NOT DEFINED CMAKE_C_STANDARD) + set(CMAKE_C_STANDARD 99) + endif() + +-# Add c++11 flags +-if (NOT DEFINED CMAKE_CXX_STANDARD) +- set(CMAKE_CXX_STANDARD 11) +-else() +- if (CMAKE_CXX_STANDARD LESS 11) +- message(FATAL_ERROR "CMAKE_CXX_STANDARD is less than 11, please specify at least SET(CMAKE_CXX_STANDARD 11)") +- endif() +-endif() +-if (NOT DEFINED CMAKE_CXX_STANDARD_REQUIRED) +- set(CMAKE_CXX_STANDARD_REQUIRED ON) +-endif() +-if (NOT DEFINED CMAKE_CXX_EXTENSIONS) +- set(CMAKE_CXX_EXTENSIONS OFF) +-endif() +- + if (NOT DEFINED CMAKE_POSITION_INDEPENDENT_CODE) + set(CMAKE_POSITION_INDEPENDENT_CODE TRUE) + endif() diff --git a/.gear/clickhouse-llvm-compat.patch b/.gear/clickhouse-llvm-compat.patch new file mode 100644 index 00000000000..5cc7e45c611 --- /dev/null +++ b/.gear/clickhouse-llvm-compat.patch @@ -0,0 +1,27 @@ +diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h +index 63c7f48a..c510b288 100644 +--- a/llvm/include/llvm/ADT/STLExtras.h ++++ b/llvm/include/llvm/ADT/STLExtras.h +@@ -1820,9 +1820,9 @@ template struct result_pair { + result_pair(std::size_t Index, IterOfRange Iter) + : Index(Index), Iter(Iter) {} + +- result_pair(const result_pair &Other) ++ result_pair(const result_pair &Other) + : Index(Other.Index), Iter(Other.Iter) {} +- result_pair &operator=(const result_pair &Other) { ++ result_pair &operator=(const result_pair &Other) { + Index = Other.Index; + Iter = Other.Iter; + return *this; +@@ -1870,8 +1870,8 @@ public: + return Result.Iter == RHS.Result.Iter; + } + +- enumerator_iter(const enumerator_iter &Other) : Result(Other.Result) {} +- enumerator_iter &operator=(const enumerator_iter &Other) { ++ enumerator_iter(const enumerator_iter &Other) : Result(Other.Result) {} ++ enumerator_iter &operator=(const enumerator_iter &Other) { + Result = Other.Result; + return *this; + } diff --git a/.gear/clickhouse-nanodbc-compat.patch b/.gear/clickhouse-nanodbc-compat.patch new file mode 100644 index 00000000000..d02949eb16b --- /dev/null +++ b/.gear/clickhouse-nanodbc-compat.patch @@ -0,0 +1,12 @@ +diff --git a/nanodbc/nanodbc.cpp b/nanodbc/nanodbc.cpp +index a2c99b5..3986c21 100644 +--- a/nanodbc/nanodbc.cpp ++++ b/nanodbc/nanodbc.cpp +@@ -23,6 +23,7 @@ + #include + #include + #include ++#include + #include + #include + diff --git a/.gear/clickhouse-system-libuv.patch b/.gear/clickhouse-system-libuv.patch new file mode 100644 index 00000000000..92ee5d43534 --- /dev/null +++ b/.gear/clickhouse-system-libuv.patch @@ -0,0 +1,14 @@ +diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake +index 90524723..faef33d5 100644 +--- a/cmake/Dependencies.cmake ++++ b/cmake/Dependencies.cmake +@@ -21,7 +21,8 @@ if(NOT LIBUV_ROOT_DIR) + endif() + + # Ensure libuv was found +-find_package(Libuv "1.0.0") ++include(FindPkgConfig) ++pkg_check_modules(LIBUV REQUIRED libuv) + if(WIN32 AND NOT LIBUV_FOUND) + message(STATUS "Unable to Locate libuv: Third party build step will be performed") + include(ExternalProject-libuv) diff --git a/.gear/clickhouse.spec b/.gear/clickhouse.spec new file mode 100644 index 00000000000..7447e051cf2 --- /dev/null +++ b/.gear/clickhouse.spec @@ -0,0 +1,542 @@ +%define _unpackaged_files_terminate_build 1 + +%def_with clang + +%ifnarch ppc64le +%def_with jemalloc +%else +%def_without jemalloc +%endif + +%if_with clang +# LTO support for clang +%ifnarch aarch64 +%global optflags_lto -flto=thin +%else +%global optflags_lto %nil +%endif +ExclusiveArch: aarch64 x86_64 +%else +# LTO causes random crashes, disable it +%global optflags_lto %nil +ExclusiveArch: aarch64 x86_64 ppc64le +%endif + +Name: clickhouse +Version: 21.8.14.5 +Release: alt1 +Summary: Open-source distributed column-oriented DBMS +License: Apache-2.0 +Group: Databases +Url: https://clickhouse.yandex/ + +# https://github.com/ClickHouse/ClickHouse.git +Source: %name-%version.tar + +Source1: %name-%version-contrib-abseil-cpp.tar +Source2: %name-%version-contrib-AMQP-CPP.tar +Source3: %name-%version-contrib-avro.tar +Source4: %name-%version-contrib-aws.tar +Source5: %name-%version-contrib-aws-c-common.tar +Source6: %name-%version-contrib-aws-c-event-stream.tar +Source7: %name-%version-contrib-aws-checksums.tar +Source8: %name-%version-contrib-base64.tar +Source9: %name-%version-contrib-boost.tar +Source10: %name-%version-contrib-boringssl.tar +Source11: %name-%version-contrib-cassandra.tar +Source12: %name-%version-contrib-cctz.tar +Source13: %name-%version-contrib-cppkafka.tar +Source14: %name-%version-contrib-cppkafka-third_party-Catch2.tar +Source15: %name-%version-contrib-croaring.tar +Source16: %name-%version-contrib-datasketches-cpp.tar +Source17: %name-%version-contrib-dragonbox.tar +Source18: %name-%version-contrib-fast_float.tar +Source19: %name-%version-contrib-fastops.tar +Source20: %name-%version-contrib-fmtlib.tar +Source21: %name-%version-contrib-gcem.tar +Source22: %name-%version-contrib-googletest.tar +Source23: %name-%version-contrib-grpc.tar +Source24: %name-%version-contrib-h3.tar +Source25: %name-%version-contrib-libcxx.tar +Source26: %name-%version-contrib-libcxxabi.tar +Source27: %name-%version-contrib-libhdfs3.tar +Source28: %name-%version-contrib-libpq.tar +Source29: %name-%version-contrib-libpqxx.tar +Source30: %name-%version-contrib-llvm.tar +Source31: %name-%version-contrib-miniselect.tar +Source32: %name-%version-contrib-nanodbc.tar +Source33: %name-%version-contrib-NuRaft.tar +Source34: %name-%version-contrib-poco.tar +Source35: %name-%version-contrib-protobuf.tar +Source36: %name-%version-contrib-re2.tar +Source37: %name-%version-contrib-replxx.tar +Source38: %name-%version-contrib-rocksdb.tar +Source39: %name-%version-contrib-sentry-native.tar +Source40: %name-%version-contrib-simdjson.tar +Source41: %name-%version-contrib-sparsehash-c11.tar +Source42: %name-%version-contrib-stats.tar +Source43: %name-%version-contrib-thrift.tar +Source44: %name-%version-contrib-yaml-cpp.tar +Source45: %name-%version-contrib-zlib-ng.tar + +Source1000: %name.watch + +Patch0: %name-%version-%release.patch +Patch1: %name-base64-ppc64le.patch +Patch2: %name-avro-gcc10-compat.patch +Patch3: %name-grpc-abseil-cxx17-compat.patch +Patch4: %name-system-libuv.patch +Patch5: %name-fastops-gcc-compat.patch +Patch6: %name-nanodbc-compat.patch +Patch7: %name-llvm-compat.patch + +BuildRequires(pre): rpm-build-python3 +%if_with clang +BuildRequires: clang lld +%else +BuildRequires: gcc-c++ +BuildRequires: libabseil-cpp-devel +BuildRequires: boost-complete +BuildRequires: cctz-devel +BuildRequires: libcppkafka-devel +BuildRequires: libfmt-devel +BuildRequires: libprotobuf-devel protobuf-compiler protobuf-c-compiler +BuildRequires: libre2-devel +BuildRequires: librocksdb-devel +BuildRequires: libyaml-cpp-devel +%endif +BuildRequires: cmake libicu-devel libreadline-devel python3 gperf tzdata +BuildRequires: rpm-macros-cmake liblz4-devel /proc libzstd-devel +BuildRequires: libmysqlclient-devel +BuildRequires: libssl-devel +BuildRequires: farmhash-devel metrohash-devel libdouble-conversion-devel librdkafka-devel +BuildRequires: libgsasl-devel libcap-ng-devel libxxhash-devel libunixODBC-devel libgperftools-devel +BuildRequires: libbrotli-devel capnproto-devel libxml2-devel liblzma-devel +# TODO: try unbundling poco when new version is released +#BuildRequires: libpoco-devel +BuildRequires: libtinfo-devel +BuildRequires: perl-JSON-XS libb64-devel libasan-devel-static +BuildRequires: libstdc++-devel-static +BuildRequires: rapidjson-devel +%ifarch x86_64 +BuildRequires: libhyperscan-devel +%endif +BuildRequires: libcurl-devel +BuildRequires: libflatbuffers-devel +# TODO: try unbundling googletest when new version is released +#BuildRequires: libgtest-devel +%ifnarch aarch64 +BuildRequires: libunwind-devel +%endif +%ifarch x86_64 +BuildRequires: libcpuid-devel +%endif +BuildRequires: libuv-devel +BuildRequires: libmsgpack-devel +BuildRequires: libsasl2-devel +BuildRequires: libsnappy-devel +BuildRequires: libltdl-devel +BuildRequires: bzlib-devel libgflags-devel +BuildRequires: liblzma-devel + +%if_with jemalloc +BuildRequires: libjemalloc-devel +%endif + +%add_python3_path %_datadir/clickhouse-test + +%description +ClickHouse is an open-source column-oriented database management system that +allows generating analytical data reports in real time. + +%package common-static +Group: Databases +Summary: Common files for %name +Provides: libclickhouse = %EVR +Conflicts: libclickhouse < %EVR +Obsoletes: libclickhouse < %EVR + +%description common-static +This package provides common files for both clickhouse server and client. + +%package server +Summary: Server binary for ClickHouse +Group: Databases +Requires: %name-common-static = %EVR + +%description server +This package contains server binaries for ClickHouse DBMS. + +%package client +Summary: Client binary for ClickHouse +Group: Databases +Requires: %name-common-static = %EVR + +%description client +This package contains clickhouse-client, clickhouse-local and clickhouse-benchmark + +%package test +Summary: ClickHouse tests +Group: Databases +Requires: %name-client = %EVR + +%description test +ClickHouse tests + +%prep +%setup -a1 -a2 -a3 -a4 -a5 -a6 -a7 -a8 -a9 -a10 -a11 -a12 -a13 -a14 -a15 -a16 -a17 -a18 -a19 -a20 -a21 -a22 -a23 -a24 -a25 -a26 -a27 -a28 -a29 -a30 -a31 -a32 -a33 -a34 -a35 -a36 -a37 -a38 -a39 -a40 -a41 -a42 -a43 -a44 -a45 + +%if_without clang +# remove unneeded bundles +rm -rf contrib/{abseil-cpp,boost,cctz,cppkafka,fmtlib,libcxx,libcxxabi,protobuf,re2,rocksdb,yaml-cpp} +%endif + +%patch0 -p1 + +pushd contrib/base64 +%patch1 -p1 +popd + +pushd contrib/avro +%patch2 -p1 +popd + +pushd contrib/grpc +%patch3 -p1 +popd + +pushd contrib/cassandra +%patch4 -p1 +popd + +pushd contrib/fastops +%patch5 -p1 +popd + +pushd contrib/nanodbc +%patch6 -p1 +popd + +pushd contrib/llvm +%patch7 -p1 +popd + +# remove third-party headers which must not be used +rm -rf contrib/jemalloc-cmake/include* + +%build +if [ %__nprocs -gt 6 ] ; then + export NPROCS=6 +else + export NPROCS=%__nprocs +fi + +# strip debuginfo: with bundled llvm debuginfo takes too much space +%define optflags_debug -g0 + +%cmake \ +%if_with clang + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DUSE_LIBCXX:BOOL=ON \ + -DUSE_INTERNAL_BOOST_LIBRARY:BOOL=ON \ + -DUSE_INTERNAL_CCTZ_LIBRARY:BOOL=ON \ + -DUSE_INTERNAL_LIBCXX_LIBRARY:BOOL=ON \ + -DUSE_INTERNAL_PROTOBUF_LIBRARY:BOOL=ON \ + -DUSE_INTERNAL_RE2_LIBRARY:BOOL=ON \ + -DUSE_INTERNAL_ROCKSDB_LIBRARY:BOOL=ON \ + -DUSE_INTERNAL_ZLIB_LIBRARY:BOOL=ON \ + -DUSE_SYSTEM_ABSEIL_CPP:BOOL=OFF \ + -DUSE_SYSTEM_BOOST:BOOL=OFF \ + -DUSE_SYSTEM_CPPKAFKA:BOOL=OFF \ + -DUSE_SYSTEM_FMTLIB:BOOL=OFF \ + -DUSE_SYSTEM_YAML_CPP:BOOL=OFF \ + -DPARALLEL_COMPILE_JOBS=$NPROCS \ + -DPARALLEL_LINK_JOBS=1 \ +%else + -DUSE_LIBCXX:BOOL=OFF \ + -DUSE_INTERNAL_BOOST_LIBRARY:BOOL=OFF \ + -DUSE_INTERNAL_CCTZ_LIBRARY:BOOL=OFF \ + -DUSE_INTERNAL_LIBCXX_LIBRARY:BOOL=OFF \ + -DUSE_INTERNAL_PROTOBUF_LIBRARY:BOOL=OFF \ + -DUSE_INTERNAL_RE2_LIBRARY:BOOL=OFF \ + -DUSE_INTERNAL_ROCKSDB_LIBRARY:BOOL=OFF \ + -DUSE_INTERNAL_ZLIB_LIBRARY:BOOL=OFF \ + -DUSE_SYSTEM_ABSEIL_CPP:BOOL=ON \ + -DUSE_SYSTEM_BOOST:BOOL=ON \ + -DUSE_SYSTEM_CPPKAFKA:BOOL=ON \ + -DUSE_SYSTEM_FMTLIB:BOOL=ON \ + -DUSE_SYSTEM_YAML_CPP:BOOL=ON \ + -DPARALLEL_COMPILE_JOBS=$NPROCS \ + -DPARALLEL_LINK_JOBS=$NPROCS \ +%endif + -DCLICKHOUSE_SPLIT_BINARY:BOOL=OFF \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF \ +%ifarch x86_64 + -DENABLE_CPUID:BOOL=ON \ +%else + -DENABLE_CPUID:BOOL=OFF \ + -DENABLE_FASTOPS:BOOL=OFF \ + -DENABLE_HDFS:BOOL=OFF \ + -DUSE_INTERNAL_HDFS3_LIBRARY:BOOL=OFF \ +%endif +%if_with jemalloc + -DENABLE_JEMALLOC:BOOL=ON \ +%else + -DENABLE_JEMALLOC:BOOL=OFF \ +%endif + -DENABLE_PARQUET:BOOL=OFF \ + -DENABLE_S3:BOOL=OFF \ + -DENABLE_UTILS:BOOL=OFF \ + -DUNBUNDLED:BOOL=ON \ + -DUSE_INTERNAL_AWS_S3_LIBRARY:BOOL=OFF \ + -DUSE_INTERNAL_BROTLI_LIBRARY:BOOL=OFF \ + -DUSE_INTERNAL_GRPC_LIBRARY:BOOL=ON \ + -DUSE_INTERNAL_GTEST_LIBRARY:BOOL=ON \ + -DUSE_INTERNAL_LIBGSASL_LIBRARY:BOOL=OFF \ +%ifarch x86_64 + -DUSE_INTERNAL_LLVM_LIBRARY:BOOL=ON \ +%else + -DUSE_INTERNAL_LLVM_LIBRARY:BOOL=OFF \ +%endif + -DUSE_INTERNAL_POCO_LIBRARY:BOOL=ON \ + -DUSE_INTERNAL_RDKAFKA_LIBRARY:BOOL=OFF \ + -DUSE_INTERNAL_REPLXX:BOOL=ON \ + -DUSE_INTERNAL_XZ_LIBRARY:BOOL=OFF \ + -DUSE_STATIC_LIBRARIES:BOOL=ON \ +%ifnarch aarch64 + -DUSE_UNWIND:BOOL=ON \ +%else + -DUSE_UNWIND:BOOL=OFF \ +%endif + %nil + +%cmake_build + +%install +%cmake_install +install -Dm0644 debian/clickhouse-server.cron.d %buildroot%_sysconfdir/cron.d/clickhouse-server +install -Dm0644 debian/clickhouse-server.service %buildroot%_unitdir/clickhouse-server.service +mkdir -p %buildroot%_localstatedir/clickhouse +mkdir -p %buildroot%_logdir/clickhouse-server + +# remove unpackaged files +rm -rfv %buildroot%_prefix/cmake +rm -fv %buildroot%_prefix/lib/*.a + +%check +./%_cmake__builddir/src/unit_tests_dbms --gtest_filter='-CoordinationTest.TestRotateIntervalChanges:ReadBufferAIOTest.TestReadAfterAIO:WeakHash32.*' + +%pre server +%_sbindir/groupadd -r -f _clickhouse 2> /dev/null ||: +%_sbindir/useradd -r -g _clickhouse -d %_localstatedir/lib/%name -s /dev/null -c "ClickHouse User" _clickhouse 2> /dev/null ||: + +%post server +%post_service clickhouse-server + +%preun server +%preun_service clickhouse-server + +%post common-static +# CAP_IPC_LOCK capability is needed for binary mlock +# CAP_SYS_NICE capability is needef for os_thread_priority feature +setcap -q cap_ipc_lock,cap_sys_nice=+ep %_bindir/clickhouse 2>/dev/null ||: + +if [ -f /proc/cpuinfo ] ; then + grep -sq sse4_2 /proc/cpuinfo || echo "Warning: No SSE4.2 detected on this CPU, clickhouse may fail." >&2 +fi + +%files common-static +%_bindir/clickhouse +%_bindir/clickhouse-odbc-bridge +%_bindir/clickhouse-library-bridge +%_datadir/bash-completion/completions/clickhouse +%_datadir/bash-completion/completions/clickhouse-bootstrap + +%files server +%dir %_sysconfdir/clickhouse-server +%dir %_sysconfdir/clickhouse-keeper +%config(noreplace) %_sysconfdir/cron.d/clickhouse-server +%config(noreplace) %_sysconfdir/clickhouse-server/config.xml +%config(noreplace) %_sysconfdir/clickhouse-server/users.xml +%config(noreplace) %_sysconfdir/clickhouse-keeper/keeper_config.xml +%_bindir/clickhouse-server +%_bindir/clickhouse-report +%_bindir/clickhouse-copier +%_bindir/clickhouse-keeper +%_bindir/clickhouse-keeper-converter +%_unitdir/clickhouse-server.service +%dir %attr(0750,_clickhouse,_clickhouse) %_logdir/clickhouse-server +%dir %attr(0750,_clickhouse,_clickhouse) %_localstatedir/clickhouse + +%files client +%dir %_sysconfdir/clickhouse-client +%config(noreplace) %_sysconfdir/clickhouse-client/config.xml +%_bindir/clickhouse-client +%_bindir/clickhouse-local +%_bindir/clickhouse-compressor +%_bindir/clickhouse-benchmark +%_bindir/clickhouse-obfuscator +%_bindir/clickhouse-format +%_bindir/clickhouse-extract-from-config +%_bindir/clickhouse-git-import +%_datadir/bash-completion/completions/clickhouse-benchmark +%_datadir/bash-completion/completions/clickhouse-client +%_datadir/bash-completion/completions/clickhouse-local + +%files test +%_bindir/clickhouse-test +%_datadir/clickhouse-test + +%changelog +* Mon Feb 14 2022 Aleksei Nikiforov 21.8.14.5-alt1 +- Updated to lts upstream version 21.8.14.5. + +* Mon Jan 10 2022 Aleksei Nikiforov 21.8.13.6-alt1 +- Updated to lts upstream version 21.8.13.6. + +* Mon Dec 06 2021 Aleksei Nikiforov 21.8.12.29-alt1 +- Updated to lts upstream version 21.8.12.29. + +* Mon Nov 22 2021 Aleksei Nikiforov 21.8.11.4-alt1 +- Updated to lts upstream version 21.8.11.4. + +* Fri Oct 29 2021 Aleksei Nikiforov 21.8.10.19-alt2 +- Rebuilt with clang. +- Rebuilt with mysql instead of mariadb. +- Disabled build for ppc64le when clang is used. +- Enabled tests. + +* Mon Oct 25 2021 Aleksei Nikiforov 21.8.10.19-alt1 +- Updated to lts upstream version 21.8.10.19. +- Disabled thread fuzzer since it's not compatible with glibc-2.34 yet (GH issue 30462). + +* Mon Oct 04 2021 Aleksei Nikiforov 21.8.8.29-alt1 +- Updated to lts upstream version 21.8.8.29. + +* Tue Sep 28 2021 Aleksei Nikiforov 21.8.7.22-alt1 +- Updated to lts upstream version 21.8.7.22. +- Disabled LTO which caused random crashes. + +* Tue Sep 14 2021 Aleksei Nikiforov 21.8.5.7-alt2 +- Updated generated version information. + +* Mon Sep 06 2021 Aleksei Nikiforov 21.8.5.7-alt1 +- Updated to lts upstream version 21.8.5.7. + +* Fri Aug 13 2021 Aleksei Nikiforov 21.8.3.44-alt1 +- Updated to lts upstream version 21.8.3.44. + +* Wed Jul 07 2021 Aleksei Nikiforov 21.3.14.1-alt1 +- Updated to lts upstream version 21.3.14.1. + +* Fri Jun 25 2021 Aleksei Nikiforov 21.3.13.9-alt1 +- Updated to lts upstream version 21.3.13.9. + +* Wed Jun 02 2021 Aleksei Nikiforov 21.3.12.2-alt2 +- Rebuilt with jemalloc. +- Added capabilities to clickhouse executable. + +* Mon May 31 2021 Arseny Maslennikov 21.3.12.2-alt1.1 +- NMU: spec: adapted to new cmake macros. + +* Thu May 27 2021 Aleksei Nikiforov 21.3.12.2-alt1 +- Updated to lts upstream version 21.3.12.2. + +* Mon May 17 2021 Aleksei Nikiforov 21.3.11.5-alt1 +- Updated to lts upstream version 21.3.11.5. + +* Tue May 11 2021 Aleksei Nikiforov 21.3.10.1-alt1 +- Updated to lts upstream version 21.3.10.1. + +* Thu Apr 29 2021 Aleksei Nikiforov 21.3.8.76-alt1 +- Updated to lts upstream version 21.3.8.76. + +* Thu Apr 22 2021 Aleksei Nikiforov 21.3.6.55-alt1 +- Updated to lts upstream version 21.3.6.55. + +* Thu Apr 01 2021 Aleksei Nikiforov 21.3.4.25-alt1 +- Updated to lts upstream version 21.3.4.25. + +* Thu Mar 25 2021 Aleksei Nikiforov 21.3.3.14-alt1 +- Updated to lts upstream version 21.3.3.14. +- Added watch file for upstream lts releases. + +* Tue Mar 16 2021 Aleksei Nikiforov 21.3.2.5-alt1 +- Updated to lts upstream version 21.3.2.5. + +* Wed Jan 13 2021 Aleksei Nikiforov 20.8.11.17-alt1 +- Updated to lts upstream version 20.8.11.17. + +* Fri Sep 25 2020 Aleksei Nikiforov 20.3.19.4-alt1 +- Updated to lts upstream version 20.3.19.4. + +* Fri Sep 18 2020 Aleksei Nikiforov 20.3.18.10-alt1 +- Updated to lts upstream version 20.3.18.10. +- Enabled libunwind dependency. + +* Wed Sep 02 2020 Aleksei Nikiforov 20.3.17.173-alt1 +- Updated to lts upstream version 20.3.17.173. + +* Thu Aug 06 2020 Aleksei Nikiforov 20.3.15.133-alt1 +- Updated to lts upstream version 20.3.15.133. + +* Tue Jun 16 2020 Aleksei Nikiforov 20.3.11.97-alt1 +- Updated to lts upstream version 20.3.11.97. + +* Tue Mar 17 2020 Anton Farygin 19.17.9.60-alt1 +- 19.17.8.54 -> 19.17.9.60 + +* Fri Feb 07 2020 Aleksei Nikiforov 19.17.8.54-alt1 +- Updated to stable upstream version 19.17.8.54. + +* Thu Dec 05 2019 Aleksei Nikiforov 19.16.2.2-alt2 +- Rebuilt with boost-1.71.0. + +* Wed Nov 06 2019 Aleksei Nikiforov 19.16.2.2-alt1 +- Updated to stable upstream version 19.16.2.2. +- Used bundled zlib-ng instead of system zlib library. +- Disabled building shared clickhouse library. +- Built single clickhouse binary. + +* Thu Oct 03 2019 Anton Farygin 19.13.6.51-alt1 +- 19.13.6.51 + +* Wed Sep 25 2019 Anton Farygin 19.13.5.44-alt1 +- 19.13.5.44 + +* Tue Sep 10 2019 Anton Farygin 19.13.4.32-alt1 +- 19.13.4.32 + +* Thu Aug 22 2019 Anton Farygin 19.13.3.26-alt1 +- 19.13.3.26 + +* Wed Aug 14 2019 Anton Farygin 19.13.2.19-alt1 +- 19.13.2.19 + +* Mon Aug 12 2019 Anton Farygin 19.13.1.11-alt1 +- 19.13.1.11 + +* Wed Aug 07 2019 Anton Farygin 19.11.6.31-alt1 +- 19.11.6.31-alt1 + +* Tue Aug 06 2019 Anton Farygin 19.11.5.28-alt2 +- 19.11.5.28 +- fixed build on aarch64 and ppc (thanx to Sergey Bolshakov) +- enabled rapidjson +- build by gcc + +* Mon Jul 22 2019 Anton Farygin 19.9.5.36-alt1 +- new version +- build without jemalloc + +* Mon Jul 01 2019 Anton Farygin 19.9.2.4-alt2 +- config files was marked for prevent rewrite during an update + +* Tue Jun 25 2019 Anton Farygin 19.9.2.4-alt1 +- updated to 19.9.2.4 + +* Wed Jun 19 2019 Anton Farygin 19.8.3.8-alt1 +- first build for ALT + diff --git a/.gear/clickhouse.watch b/.gear/clickhouse.watch new file mode 100644 index 00000000000..46e211cfb9d --- /dev/null +++ b/.gear/clickhouse.watch @@ -0,0 +1,3 @@ +version=4 +opts=filenamemangle=s/.+\/v(\d\S+)-lts\.tar\.gz/clickhouse-$1\.tar\.gz/ \ + https://github.com/ClickHouse/ClickHouse/tags .*/v(\d\S+)-lts\.tar\.gz diff --git a/.gear/rules b/.gear/rules new file mode 100644 index 00000000000..ada0ee53bae --- /dev/null +++ b/.gear/rules @@ -0,0 +1,51 @@ +spec: .gear/clickhouse.spec +tar: v@version@-lts:. +diff: v@version@-lts:. . +copy?: .gear/*.patch +copy?: .gear/*.watch + +tar: contrib/abseil-cpp-@version@:. name=@name@-@version@-contrib-abseil-cpp base=contrib/abseil-cpp +tar: contrib/AMQP-CPP-@version@:. name=@name@-@version@-contrib-AMQP-CPP base=contrib/AMQP-CPP +tar: contrib/avro-@version@:. name=@name@-@version@-contrib-avro base=contrib/avro +tar: contrib/aws-@version@:. name=@name@-@version@-contrib-aws base=contrib/aws +tar: contrib/aws-c-common-@version@:. name=@name@-@version@-contrib-aws-c-common base=contrib/aws-c-common +tar: contrib/aws-c-event-stream-@version@:. name=@name@-@version@-contrib-aws-c-event-stream base=contrib/aws-c-event-stream +tar: contrib/aws-checksums-@version@:. name=@name@-@version@-contrib-aws-checksums base=contrib/aws-checksums +tar: contrib/base64-@version@:. name=@name@-@version@-contrib-base64 base=contrib/base64 +tar: contrib/boost-@version@:. name=@name@-@version@-contrib-boost base=contrib/boost +tar: contrib/boringssl-@version@:. name=@name@-@version@-contrib-boringssl base=contrib/boringssl +tar: contrib/cassandra-@version@:. name=@name@-@version@-contrib-cassandra base=contrib/cassandra +tar: contrib/cctz-@version@:. name=@name@-@version@-contrib-cctz base=contrib/cctz +tar: contrib/cppkafka-@version@:. name=@name@-@version@-contrib-cppkafka base=contrib/cppkafka +tar: contrib/cppkafka/third_party/Catch2-@version@:. name=@name@-@version@-contrib-cppkafka-third_party-Catch2 base=contrib/cppkafka/third_party/Catch2 +tar: contrib/croaring-@version@:. name=@name@-@version@-contrib-croaring base=contrib/croaring +tar: contrib/datasketches-cpp-@version@:. name=@name@-@version@-contrib-datasketches-cpp base=contrib/datasketches-cpp +tar: contrib/dragonbox-@version@:. name=@name@-@version@-contrib-dragonbox base=contrib/dragonbox +tar: contrib/fast_float-@version@:. name=@name@-@version@-contrib-fast_float base=contrib/fast_float +tar: contrib/fastops-@version@:. name=@name@-@version@-contrib-fastops base=contrib/fastops +tar: contrib/fmtlib-@version@:. name=@name@-@version@-contrib-fmtlib base=contrib/fmtlib +tar: contrib/gcem-@version@:. name=@name@-@version@-contrib-gcem base=contrib/gcem +tar: contrib/googletest-@version@:. name=@name@-@version@-contrib-googletest base=contrib/googletest +tar: contrib/grpc-@version@:. name=@name@-@version@-contrib-grpc base=contrib/grpc +tar: contrib/h3-@version@:. name=@name@-@version@-contrib-h3 base=contrib/h3 +tar: contrib/libcxx-@version@:. name=@name@-@version@-contrib-libcxx base=contrib/libcxx +tar: contrib/libcxxabi-@version@:. name=@name@-@version@-contrib-libcxxabi base=contrib/libcxxabi +tar: contrib/libhdfs3-@version@:. name=@name@-@version@-contrib-libhdfs3 base=contrib/libhdfs3 +tar: contrib/libpq-@version@:. name=@name@-@version@-contrib-libpq base=contrib/libpq +tar: contrib/libpqxx-@version@:. name=@name@-@version@-contrib-libpqxx base=contrib/libpqxx +tar: contrib/llvm-@version@:. name=@name@-@version@-contrib-llvm base=contrib/llvm +tar: contrib/miniselect-@version@:. name=@name@-@version@-contrib-miniselect base=contrib/miniselect +tar: contrib/nanodbc-@version@:. name=@name@-@version@-contrib-nanodbc base=contrib/nanodbc +tar: contrib/NuRaft-@version@:. name=@name@-@version@-contrib-NuRaft base=contrib/NuRaft +tar: contrib/poco-@version@:. name=@name@-@version@-contrib-poco base=contrib/poco +tar: contrib/protobuf-@version@:. name=@name@-@version@-contrib-protobuf base=contrib/protobuf +tar: contrib/re2-@version@:. name=@name@-@version@-contrib-re2 base=contrib/re2 +tar: contrib/replxx-@version@:. name=@name@-@version@-contrib-replxx base=contrib/replxx +tar: contrib/rocksdb-@version@:. name=@name@-@version@-contrib-rocksdb base=contrib/rocksdb +tar: contrib/sentry-native-@version@:. name=@name@-@version@-contrib-sentry-native base=contrib/sentry-native +tar: contrib/simdjson-@version@:. name=@name@-@version@-contrib-simdjson base=contrib/simdjson +tar: contrib/sparsehash-c11-@version@:. name=@name@-@version@-contrib-sparsehash-c11 base=contrib/sparsehash-c11 +tar: contrib/stats-@version@:. name=@name@-@version@-contrib-stats base=contrib/stats +tar: contrib/thrift-@version@:. name=@name@-@version@-contrib-thrift base=contrib/thrift +tar: contrib/yaml-cpp-@version@:. name=@name@-@version@-contrib-yaml-cpp base=contrib/yaml-cpp +tar: contrib/zlib-ng-@version@:. name=@name@-@version@-contrib-zlib-ng base=contrib/zlib-ng diff --git a/.gear/submodules b/.gear/submodules new file mode 100644 index 00000000000..6a4ba03fcb2 --- /dev/null +++ b/.gear/submodules @@ -0,0 +1,45 @@ +contrib/AMQP-CPP +contrib/NuRaft +contrib/abseil-cpp +contrib/avro +contrib/aws +contrib/aws-c-common +contrib/aws-c-event-stream +contrib/aws-checksums +contrib/base64 +contrib/boost +contrib/boringssl +contrib/cassandra +contrib/cctz +contrib/cppkafka +contrib/cppkafka/third_party/Catch2 +contrib/croaring +contrib/datasketches-cpp +contrib/dragonbox +contrib/fast_float +contrib/fastops +contrib/fmtlib +contrib/gcem +contrib/googletest +contrib/grpc +contrib/h3 +contrib/libcxx +contrib/libcxxabi +contrib/libhdfs3 +contrib/libpq +contrib/libpqxx +contrib/llvm +contrib/miniselect +contrib/nanodbc +contrib/poco +contrib/protobuf +contrib/re2 +contrib/replxx +contrib/rocksdb +contrib/sentry-native +contrib/simdjson +contrib/sparsehash-c11 +contrib/stats +contrib/thrift +contrib/yaml-cpp +contrib/zlib-ng diff --git a/.gear/tags/8a9f32ea051bbb6a7d29591944c50876949cb763 b/.gear/tags/8a9f32ea051bbb6a7d29591944c50876949cb763 new file mode 100644 index 00000000000..a01db3e5889 --- /dev/null +++ b/.gear/tags/8a9f32ea051bbb6a7d29591944c50876949cb763 @@ -0,0 +1,6 @@ +object 689846062f8bdd8df6333073379d08da581e56f9 +type commit +tag v21.8.14.5-lts +tagger robot-clickhouse 1643183683 +0300 + +v21.8.14.5-lts diff --git a/.gear/tags/list b/.gear/tags/list new file mode 100644 index 00000000000..2cf3e8feb0d --- /dev/null +++ b/.gear/tags/list @@ -0,0 +1,46 @@ +8a9f32ea051bbb6a7d29591944c50876949cb763 v21.8.14.5-lts +4f3b686f86c3ebaba7e4e926e62a79cb1c659a54 contrib/abseil-cpp-21.8.14.5 +03781aaff0f10ef41f902b8cf865fe0067180c10 contrib/AMQP-CPP-21.8.14.5 +e43c46e87fd32eafdc09471e95344555454c5ef8 contrib/avro-21.8.14.5 +7d48b2c8193679cc4516e5bd68ae4a64b94dae7d contrib/aws-21.8.14.5 +736a82d1697c108b04a277e66438a7f4e19b6857 contrib/aws-c-common-21.8.14.5 +3bc33662f9ccff4f4cbcf9509cc78c26e022fde0 contrib/aws-c-event-stream-21.8.14.5 +519d6d9093819b6cf89ffff589a27ef8f83d0f65 contrib/aws-checksums-21.8.14.5 +af9b331f2b4f30b41c70f3a571ff904a8251c1d3 contrib/base64-21.8.14.5 +1ccbb5a522a571ce83b606dbc2e1011c42ecccfb contrib/boost-21.8.14.5 +c1e01a441d6db234f4f12e63a7657d1f9e6db9c1 contrib/boringssl-21.8.14.5 +eb9b68dadbb4417a2c132ad4a1c2fa76e65e6fc1 contrib/cassandra-21.8.14.5 +9edd0861d8328b2ae77e8fb5f4d7dcd1cf33b42b contrib/cctz-21.8.14.5 +5a119f689f8a4d90d10a9635e7ee2bee5c127de1 contrib/cppkafka-21.8.14.5 +d2d8455b571b6c66c4b7003500a77f9a93ecdc28 contrib/cppkafka/third_party/Catch2-21.8.14.5 +2c867e9f9c9e2a3a7032791f94c4c7ae3013f6e0 contrib/croaring-21.8.14.5 +7d73d7610db31d4e1ecde0fb3a7ee90ef371207f contrib/datasketches-cpp-21.8.14.5 +923705af6fd953aa948fc175f6020b15f7359838 contrib/dragonbox-21.8.14.5 +7eae925b51fd0f570ccd5c880c12e3e27a23b86f contrib/fast_float-21.8.14.5 +88752a5e03cf34639a4a37a4b41d8b463fffd2b5 contrib/fastops-21.8.14.5 +c108ee1d590089ccf642fc85652b845924067af2 contrib/fmtlib-21.8.14.5 +8d4f1b5d76ea8f6ff12f3f4f34cda45424556b00 contrib/gcem-21.8.14.5 +e7e591764baba0a0c3c9ad0014430e7a27331d16 contrib/googletest-21.8.14.5 +7eac189a6badddac593580ec2ad1478bd2656fc7 contrib/grpc-21.8.14.5 +e209086ae1b5477307f545a0f6111780edc59940 contrib/h3-21.8.14.5 +2fa892f69acbaa40f8a18c6484854a6183a34482 contrib/libcxx-21.8.14.5 +df8f1e727dbc9e2bedf2282096fa189dc3fe0076 contrib/libcxxabi-21.8.14.5 +85ec1dd420ea9e9cf9d53a3b6eeaa45e8b78b00b contrib/libhdfs3-21.8.14.5 +e071ea570f8985aa00e34f5b9d50a3cfe666327e contrib/libpq-21.8.14.5 +357608d11b7a1961c3fb7db2ef9a5dbb2e87da77 contrib/libpqxx-21.8.14.5 +e5751459412bce1391fb7a2e9bbc01e131bf72f1 contrib/llvm-21.8.14.5 +be0af6bd0b6eb044d1acc4f754b229972d99903a contrib/miniselect-21.8.14.5 +9fc459675515d491401727ec67fca38db721f28c contrib/nanodbc-21.8.14.5 +976874b7aa7f422bf4ea595bb7d1166c617b1c26 contrib/NuRaft-21.8.14.5 +5994506908028612869fee627d68d8212dfe7c1e contrib/poco-21.8.14.5 +73b12814204ad9068ba352914d0dc244648b48ee contrib/protobuf-21.8.14.5 +13ebb377c6ad763ca61d12dd6f88b1126bd0b911 contrib/re2-21.8.14.5 +c81be6c68b146f15f2096b7ef80e3f21fe27004c contrib/replxx-21.8.14.5 +07c77549a20b63ff6981b400085eba36bb5c80c4 contrib/rocksdb-21.8.14.5 +94644e92f0a3ff14bd35ed902a8622a2d15f7be4 contrib/sentry-native-21.8.14.5 +8df32cea3359cb30120795da6020b3b73da01d38 contrib/simdjson-21.8.14.5 +cf0bffaa456f23bc4174462a789b90f8b6f5f42f contrib/sparsehash-c11-21.8.14.5 +b6dd459c10a88c7ea04693c007e9e35820c5d9ad contrib/stats-21.8.14.5 +010ccf0a0c7023fea0f6bf4e4078ebdff7e61982 contrib/thrift-21.8.14.5 +0c86adac6d117ee2b4afcedb8ade19036ca0327d contrib/yaml-cpp-21.8.14.5 +db232d30b4c72fd58e6d7eae2d12cebf9c3d90db contrib/zlib-ng-21.8.14.5 diff --git a/.gear/update-revision.sh b/.gear/update-revision.sh new file mode 100755 index 00000000000..9fd49c4595a --- /dev/null +++ b/.gear/update-revision.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +if [ -z "$1" ] ; then + echo "USAGE: $0 release_tag" 1>&2 + exit 1 +fi + +#echo $1 | sed -r 's/v([[:digit:]]+(\.[[:digit:]]+)*)-.*/\1/' +#exit 0 + +git_describe=$(git describe $1) +git_hash=$(git rev-list -n 1 $1) +VERSION_DATE=$(git show -s --format=%cs ${git_hash} | tail -n 1) + +VERSION_STRING=$(echo $1 | sed -r 's/v([[:digit:]]+(\.[[:digit:]]+)*)-.*/\1/') +VERSION_MAJOR=$(echo $VERSION_STRING | cut -d . -f 1) +VERSION_MINOR=$(echo $VERSION_STRING | cut -d . -f 2) +VERSION_PATCH=$(echo $VERSION_STRING | cut -d . -f 3) + +sed -i \ + -e "s/SET(VERSION_DESCRIBE [^) ]*/SET(VERSION_DESCRIBE $git_describe/g;" \ + -e "s/SET(VERSION_GITHASH [^) ]*/SET(VERSION_GITHASH $git_hash/g;" \ + -e "s/SET(VERSION_DATE [^) ]*/SET(VERSION_DATE $VERSION_DATE/g;" \ + -e "s/SET(VERSION_MAJOR [^) ]*/SET(VERSION_MAJOR $VERSION_MAJOR/g;" \ + -e "s/SET(VERSION_MINOR [^) ]*/SET(VERSION_MINOR $VERSION_MINOR/g;" \ + -e "s/SET(VERSION_PATCH [^) ]*/SET(VERSION_PATCH $VERSION_PATCH/g;" \ + -e "s/SET(VERSION_STRING [^) ]*/SET(VERSION_STRING $VERSION_STRING/g;" \ + cmake/autogenerated_versions.txt + +echo describe $git_describe +echo hash $git_hash +echo version_date $VERSION_DATE +echo version_string $VERSION_STRING +echo version_major $VERSION_MAJOR +echo version_minor $VERSION_MINOR +echo version_patch $VERSION_PATCH diff --git a/.gear/update-submodules.sh b/.gear/update-submodules.sh new file mode 100755 index 00000000000..b17b42524a9 --- /dev/null +++ b/.gear/update-submodules.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +version=$1 + +if [ -z "$version" ] ; then + echo "Error: not enough arguments" 1>&2 + echo "Usage: $0 version" 1>&2 + exit 1 +fi + +rootdir=$(git rev-parse --show-toplevel) + +cd "$rootdir" + +git submodule update --init --recursive + +readarray -t allowed_modules < $rootdir/.gear/submodules + +update_submodules() { + local reporoot=$1 + local relativedir=$2 + local url='' + local hash='' + local submodule='' + local path='' + local new_relativedir='' + + while read submodule ; do + path=$(git config --file .gitmodules submodule.${submodule}.path) + hash=$(git submodule status ${path} | awk '{ print $1}') + url=$(git config --file .gitmodules submodule.${submodule}.url) + + if [ -z "$relativedir" ] ; then + new_relativedir="$path" + else + new_relativedir="$relativedir/$path" + fi + + if [[ " ${allowed_modules[@]} " =~ " ${new_relativedir} " ]]; then + pushd "${reporoot}" 1>&2 + git fetch "${url}" "${hash}" 1>&2 || git fetch "${url}" 1>&2 || git fetch "${url}" --tags 1>&2 + git tag "${new_relativedir}-${version}" "${hash}" 1>&2 + popd 1>&2 + + pushd "${path}" 1>&2 + update_submodules "${reporoot}" "${new_relativedir}" + popd 1>&2 + + echo "${new_relativedir}-${version}" + fi + done < <(git config --file .gitmodules --name-only --get-regexp url | sed -e 's:^submodule\.::' -e 's:\.url$::') +} + +tags=$(update_submodules $rootdir '') +tags="$(echo "${tags}" | sort)" +git merge --no-edit -s ours --allow-unrelated-histories $(echo "${tags}" | xargs echo) + +echo "Following tags should be merged:" +echo "${tags}" +echo +echo "Following gear rules might be needed:" +while read tag ; do + echo "tar: $(echo $tag | sed -e "s:$version:@version@:g"):. name=@name@-@version@-$(echo $tag | sed -e "s:-$version::g" -e "s:/:-:g") base=$(echo $tag | sed -e "s:-$version::g")" +done < <(echo "${tags}") diff --git a/CMakeLists.txt b/CMakeLists.txt index d23e5f540d3..0b835c99edf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -573,10 +573,16 @@ if (TARGET global-group) install (EXPORT global DESTINATION cmake) endif () +if (USE_SYSTEM_ABSEIL_CPP) + find_package(absl REQUIRED) +endif (USE_SYSTEM_ABSEIL_CPP) + add_subdirectory (contrib EXCLUDE_FROM_ALL) if (NOT ENABLE_JEMALLOC) message (WARNING "Non default allocator is disabled. This is not recommended for production builds.") +else () + add_compile_definitions(USE_JEMALLOC=1) endif () macro (add_executable target) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index d1b2e05eb30..fe3fe1a3af8 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -6,7 +6,7 @@ SET(VERSION_REVISION 54453) SET(VERSION_MAJOR 21) SET(VERSION_MINOR 8) SET(VERSION_PATCH 14) -SET(VERSION_GITHASH b3d03e55753069a5a39245ee6eb8681045cdaf62) -SET(VERSION_DESCRIBE v21.8.14.1-lts) -SET(VERSION_STRING 21.8.14.1) +SET(VERSION_GITHASH 689846062f8bdd8df6333073379d08da581e56f9) +SET(VERSION_DESCRIBE v21.8.14.5-lts) +SET(VERSION_STRING 21.8.14.5) # end of autochange diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index d12eac2e3c4..0d5272c5d4e 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -28,7 +28,7 @@ if (HAVE_SSE41) endif () if (ARCH_PPC64LE) - set (COMPILER_FLAGS "${COMPILER_FLAGS} -maltivec -D__SSE2__=1 -DNO_WARN_X86_INTRINSICS") + set (COMPILER_FLAGS "${COMPILER_FLAGS} -maltivec -DNO_WARN_X86_INTRINSICS") endif () # gcc -dM -E -msse4.2 - < /dev/null | sort > gcc-dump-sse42 diff --git a/cmake/find/cassandra.cmake b/cmake/find/cassandra.cmake index b6e97ff5ef8..27450252c28 100644 --- a/cmake/find/cassandra.cmake +++ b/cmake/find/cassandra.cmake @@ -1,8 +1,3 @@ -if (MISSING_INTERNAL_LIBUV_LIBRARY) - message (WARNING "Disabling cassandra due to missing libuv") - set (ENABLE_CASSANDRA OFF CACHE INTERNAL "") -endif() - option(ENABLE_CASSANDRA "Enable Cassandra" ${ENABLE_LIBRARIES}) if (NOT ENABLE_CASSANDRA) diff --git a/cmake/find/libuv.cmake b/cmake/find/libuv.cmake index f0023209309..8a1d928e8a0 100644 --- a/cmake/find/libuv.cmake +++ b/cmake/find/libuv.cmake @@ -10,11 +10,7 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libuv") return() endif() -if (MAKE_STATIC_LIBRARIES) - set (LIBUV_LIBRARY uv_a) -else() - set (LIBUV_LIBRARY uv) -endif() +set (LIBUV_LIBRARY uv) set (LIBUV_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/libuv") set (LIBUV_INCLUDE_DIR "${LIBUV_ROOT_DIR}/include") diff --git a/cmake/find/rdkafka.cmake b/cmake/find/rdkafka.cmake index f6460c1d9a3..16ef3788a20 100644 --- a/cmake/find/rdkafka.cmake +++ b/cmake/find/rdkafka.cmake @@ -19,7 +19,7 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cppkafka/CMakeLists.txt") endif () if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/CMakeLists.txt") - if(USE_INTERNAL_RDKAFKA_LIBRARY OR MISSING_INTERNAL_CPPKAFKA_LIBRARY) + if(USE_INTERNAL_RDKAFKA_LIBRARY AND MISSING_INTERNAL_CPPKAFKA_LIBRARY) message (WARNING "submodule contrib/librdkafka is missing. to fix try run: \n git submodule update --init --recursive") message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal rdkafka") set (USE_INTERNAL_RDKAFKA_LIBRARY 0) diff --git a/cmake/find/unwind.cmake b/cmake/find/unwind.cmake index c9f5f30a5d6..ea4b63c9158 100644 --- a/cmake/find/unwind.cmake +++ b/cmake/find/unwind.cmake @@ -1,8 +1,8 @@ option (USE_UNWIND "Enable libunwind (better stacktraces)" ${ENABLE_LIBRARIES}) if (USE_UNWIND) - add_subdirectory(contrib/libunwind-cmake) - set (UNWIND_LIBRARIES unwind) + include(FindPkgConfig) + pkg_check_modules(UNWIND REQUIRED libunwind) set (EXCEPTION_HANDLING_LIBRARY ${UNWIND_LIBRARIES}) message (STATUS "Using libunwind: ${UNWIND_LIBRARIES}") diff --git a/cmake/linux/default_libs.cmake b/cmake/linux/default_libs.cmake index c1e4d450389..00de988ca09 100644 --- a/cmake/linux/default_libs.cmake +++ b/cmake/linux/default_libs.cmake @@ -1,8 +1,6 @@ # Set standard, system and compiler libraries explicitly. # This is intended for more control of what we are linking. -set (DEFAULT_LIBS "-nodefaultlibs") - # We need builtins from Clang's RT even without libcxx - for ubsan+int128. # See https://bugs.llvm.org/show_bug.cgi?id=16404 if (COMPILER_CLANG AND NOT (CMAKE_CROSSCOMPILING AND ARCH_AARCH64)) @@ -15,7 +13,7 @@ if (OS_ANDROID) # pthread and rt are included in libc set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -ldl") else () - set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread -ldl") + set (DEFAULT_LIBS "${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread -ldl") endif () message(STATUS "Default libraries: ${DEFAULT_LIBS}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 164692fb893..62c0cc77cb9 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -275,9 +275,6 @@ if (USE_FASTOPS) add_subdirectory (fastops-cmake) endif() -if (USE_AMQPCPP OR USE_CASSANDRA) - add_subdirectory (libuv) -endif() if (USE_AMQPCPP) add_subdirectory (amqpcpp-cmake) endif() diff --git a/contrib/abseil-cpp-cmake/CMakeLists.txt b/contrib/abseil-cpp-cmake/CMakeLists.txt index c8cb512066a..26bf119d1ac 100644 --- a/contrib/abseil-cpp-cmake/CMakeLists.txt +++ b/contrib/abseil-cpp-cmake/CMakeLists.txt @@ -1,8 +1,10 @@ -set(ABSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/abseil-cpp") -if(NOT EXISTS "${ABSL_ROOT_DIR}/CMakeLists.txt") - message(FATAL_ERROR " submodule third_party/abseil-cpp is missing. To fix try run: \n git submodule update --init --recursive") -endif() -add_subdirectory("${ABSL_ROOT_DIR}" "${ClickHouse_BINARY_DIR}/contrib/abseil-cpp") +if (NOT USE_SYSTEM_ABSEIL_CPP) + set(ABSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/abseil-cpp") + if(NOT EXISTS "${ABSL_ROOT_DIR}/CMakeLists.txt") + message(FATAL_ERROR " submodule third_party/abseil-cpp is missing. To fix try run: \n git submodule update --init --recursive") + endif() + add_subdirectory("${ABSL_ROOT_DIR}" "${ClickHouse_BINARY_DIR}/contrib/abseil-cpp") +endif (NOT USE_SYSTEM_ABSEIL_CPP) add_library(abseil_swiss_tables INTERFACE) diff --git a/contrib/cppkafka-cmake/CMakeLists.txt b/contrib/cppkafka-cmake/CMakeLists.txt index 0bc33ada529..88d97a0823c 100644 --- a/contrib/cppkafka-cmake/CMakeLists.txt +++ b/contrib/cppkafka-cmake/CMakeLists.txt @@ -1,3 +1,8 @@ +if (USE_SYSTEM_CPPKAFKA) + find_library (cppkafka cppkafka REQUIRED) + return() +endif (USE_SYSTEM_CPPKAFKA) + set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/cppkafka") set(SRCS diff --git a/contrib/fmtlib-cmake/CMakeLists.txt b/contrib/fmtlib-cmake/CMakeLists.txt index f3bf73d7dbc..6fd0b727be9 100644 --- a/contrib/fmtlib-cmake/CMakeLists.txt +++ b/contrib/fmtlib-cmake/CMakeLists.txt @@ -1,3 +1,8 @@ +if (USE_SYSTEM_FMTLIB) + find_package(fmt REQUIRED) + return() +endif (USE_SYSTEM_FMTLIB) + set (SRCS ../fmtlib/src/format.cc ../fmtlib/src/os.cc diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 9531a5a4f9e..4a43f13c225 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -19,129 +19,5 @@ if (NOT ENABLE_JEMALLOC) return() endif () -if (NOT OS_LINUX) - message (WARNING "jemalloc support on non-linux is EXPERIMENTAL") -endif() - -if (OS_LINUX) - # ThreadPool select job randomly, and there can be some threads that had been - # performed some memory heavy task before and will be inactive for some time, - # but until it will became active again, the memory will not be freed since by - # default each thread has it's own arena, but there should be not more then - # 4*CPU arenas (see opt.nareans description). - # - # By enabling percpu_arena number of arenas limited to number of CPUs and hence - # this problem should go away. - # - # muzzy_decay_ms -- use MADV_FREE when available on newer Linuxes, to - # avoid spurious latencies and additional work associated with - # MADV_DONTNEED. See - # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation. - set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000") -else() - set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000") -endif() -# CACHE variable is empty, to allow changing defaults without necessity -# to purge cache -set (JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE "" CACHE STRING "Change default configuration string of JEMalloc" ) -if (JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE) - set (JEMALLOC_CONFIG_MALLOC_CONF "${JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE}") -endif() -message (STATUS "jemalloc malloc_conf: ${JEMALLOC_CONFIG_MALLOC_CONF}") - -set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/jemalloc") - -set (SRCS - "${LIBRARY_DIR}/src/arena.c" - "${LIBRARY_DIR}/src/background_thread.c" - "${LIBRARY_DIR}/src/base.c" - "${LIBRARY_DIR}/src/bin.c" - "${LIBRARY_DIR}/src/bitmap.c" - "${LIBRARY_DIR}/src/ckh.c" - "${LIBRARY_DIR}/src/ctl.c" - "${LIBRARY_DIR}/src/div.c" - "${LIBRARY_DIR}/src/extent.c" - "${LIBRARY_DIR}/src/extent_dss.c" - "${LIBRARY_DIR}/src/extent_mmap.c" - "${LIBRARY_DIR}/src/hash.c" - "${LIBRARY_DIR}/src/hook.c" - "${LIBRARY_DIR}/src/jemalloc.c" - "${LIBRARY_DIR}/src/large.c" - "${LIBRARY_DIR}/src/log.c" - "${LIBRARY_DIR}/src/malloc_io.c" - "${LIBRARY_DIR}/src/mutex.c" - "${LIBRARY_DIR}/src/mutex_pool.c" - "${LIBRARY_DIR}/src/nstime.c" - "${LIBRARY_DIR}/src/pages.c" - "${LIBRARY_DIR}/src/prng.c" - "${LIBRARY_DIR}/src/prof.c" - "${LIBRARY_DIR}/src/rtree.c" - "${LIBRARY_DIR}/src/sc.c" - "${LIBRARY_DIR}/src/stats.c" - "${LIBRARY_DIR}/src/sz.c" - "${LIBRARY_DIR}/src/tcache.c" - "${LIBRARY_DIR}/src/test_hooks.c" - "${LIBRARY_DIR}/src/ticker.c" - "${LIBRARY_DIR}/src/tsd.c" - "${LIBRARY_DIR}/src/witness.c" - "${LIBRARY_DIR}/src/safety_check.c" -) -if (OS_DARWIN) - list(APPEND SRCS "${LIBRARY_DIR}/src/zone.c") -endif () - -add_library(jemalloc ${SRCS}) -target_include_directories(jemalloc PRIVATE "${LIBRARY_DIR}/include") -target_include_directories(jemalloc SYSTEM PUBLIC include) - -set (JEMALLOC_INCLUDE_PREFIX) -# OS_ -if (OS_LINUX) - set (JEMALLOC_INCLUDE_PREFIX "include_linux") -elseif (OS_FREEBSD) - set (JEMALLOC_INCLUDE_PREFIX "include_freebsd") -elseif (OS_DARWIN) - set (JEMALLOC_INCLUDE_PREFIX "include_darwin") -else () - message (FATAL_ERROR "internal jemalloc: This OS is not supported") -endif () -# ARCH_ -if (ARCH_AMD64) - set(JEMALLOC_INCLUDE_PREFIX "${JEMALLOC_INCLUDE_PREFIX}_x86_64") -elseif (ARCH_ARM) - set(JEMALLOC_INCLUDE_PREFIX "${JEMALLOC_INCLUDE_PREFIX}_aarch64") -elseif (ARCH_PPC64LE) - set(JEMALLOC_INCLUDE_PREFIX "${JEMALLOC_INCLUDE_PREFIX}_ppc64le") -else () - message (FATAL_ERROR "internal jemalloc: This arch is not supported") -endif () - -configure_file(${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal/jemalloc_internal_defs.h.in - ${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal/jemalloc_internal_defs.h) -target_include_directories(jemalloc SYSTEM PRIVATE - "${CMAKE_CURRENT_BINARY_DIR}/${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal") - -target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE) - -if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") - target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_DEBUG=1) -endif () - -target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_PROF=1) - -if (USE_UNWIND) - target_compile_definitions (jemalloc PRIVATE -DJEMALLOC_PROF_LIBUNWIND=1) - target_link_libraries (jemalloc PRIVATE unwind) -endif () - -target_compile_options(jemalloc PRIVATE -Wno-redundant-decls) -# for RTLD_NEXT -target_compile_options(jemalloc PRIVATE -D_GNU_SOURCE) - -set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_JEMALLOC=1) -if (MAKE_STATIC_LIBRARIES) - # To detect whether we need to register jemalloc for osx as default zone. - set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS BUNDLED_STATIC_JEMALLOC=1) -endif() - -message (STATUS "Using jemalloc") +include(FindPkgConfig) +pkg_check_modules(JEMALLOC REQUIRED jemalloc) diff --git a/contrib/libcpuid-cmake/CMakeLists.txt b/contrib/libcpuid-cmake/CMakeLists.txt index 9baebb3ba1b..c1f446c3eda 100644 --- a/contrib/libcpuid-cmake/CMakeLists.txt +++ b/contrib/libcpuid-cmake/CMakeLists.txt @@ -13,26 +13,7 @@ if (NOT ENABLE_CPUID) return() endif() -set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/libcpuid") - -set (SRCS - "${LIBRARY_DIR}/libcpuid/asm-bits.c" - "${LIBRARY_DIR}/libcpuid/cpuid_main.c" - "${LIBRARY_DIR}/libcpuid/libcpuid_util.c" - "${LIBRARY_DIR}/libcpuid/msrdriver.c" - "${LIBRARY_DIR}/libcpuid/rdmsr.c" - "${LIBRARY_DIR}/libcpuid/rdtsc.c" - "${LIBRARY_DIR}/libcpuid/recog_amd.c" - "${LIBRARY_DIR}/libcpuid/recog_intel.c" -) - -add_library (cpuid ${SRCS}) - -target_include_directories (cpuid SYSTEM PUBLIC "${LIBRARY_DIR}") -target_compile_definitions (cpuid PUBLIC USE_CPUID=1) -target_compile_definitions (cpuid PRIVATE VERSION="v0.4.1") -if (COMPILER_CLANG) - target_compile_options (cpuid PRIVATE -Wno-reserved-id-macro) -endif () +find_library (CPUID_LIBRARY cpuid REQUIRED) +find_path (CPUID_INCLUDE_DIR NAMES libcpuid/libcpuid.h PATHS ${CPUID_INCLUDE_PATHS}) message (STATUS "Using cpuid") diff --git a/contrib/nuraft-cmake/CMakeLists.txt b/contrib/nuraft-cmake/CMakeLists.txt index 725e86195e1..b80d01b1772 100644 --- a/contrib/nuraft-cmake/CMakeLists.txt +++ b/contrib/nuraft-cmake/CMakeLists.txt @@ -39,7 +39,11 @@ endif() target_include_directories (nuraft SYSTEM PRIVATE "${LIBRARY_DIR}/include/libnuraft") # for some reason include "asio.h" directly without "boost/" prefix. +if (USE_SYSTEM_BOOST) +target_include_directories (nuraft SYSTEM PRIVATE "${Boost_INCLUDE_DIR}/boost") +else (USE_SYSTEM_BOOST) target_include_directories (nuraft SYSTEM PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/boost/boost") +endif (USE_SYSTEM_BOOST) target_link_libraries (nuraft PRIVATE boost::headers_only boost::coroutine) diff --git a/contrib/yaml-cpp-cmake/CMakeLists.txt b/contrib/yaml-cpp-cmake/CMakeLists.txt index ed0287de110..2a07721fd45 100644 --- a/contrib/yaml-cpp-cmake/CMakeLists.txt +++ b/contrib/yaml-cpp-cmake/CMakeLists.txt @@ -1,3 +1,8 @@ +if (USE_SYSTEM_YAML_CPP) + find_package(yaml-cpp REQUIRED) + return() +endif (USE_SYSTEM_YAML_CPP) + set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/yaml-cpp) set (SRCS diff --git a/debian/clickhouse-server.service b/debian/clickhouse-server.service index b9681f9279e..762e51659f8 100644 --- a/debian/clickhouse-server.service +++ b/debian/clickhouse-server.service @@ -5,8 +5,8 @@ After=network-online.target [Service] Type=simple -User=clickhouse -Group=clickhouse +User=_clickhouse +Group=_clickhouse Restart=always RestartSec=30 RuntimeDirectory=clickhouse-server diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 50f7aed3584..148a1da0498 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -67,7 +67,6 @@ #include #include #include -#include #include #include #include @@ -477,9 +476,6 @@ int Server::main(const std::vector & /*args*/) CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::getVersionRevision()); CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger()); - if (ThreadFuzzer::instance().isEffective()) - LOG_WARNING(log, "ThreadFuzzer is enabled. Application will run slowly and unstable."); - #if !defined(NDEBUG) || !defined(__OPTIMIZE__) LOG_WARNING(log, "Server was built in debug mode. It will work slowly."); #endif diff --git a/src/Common/ThreadFuzzer.cpp b/src/Common/ThreadFuzzer.cpp deleted file mode 100644 index a538ba7a49a..00000000000 --- a/src/Common/ThreadFuzzer.cpp +++ /dev/null @@ -1,272 +0,0 @@ -#include -#include -#if defined(OS_LINUX) -# include -#endif -#include - -#include - -#include - -#include -#include - -#include -#include - -#include - - -/// We will also wrap some thread synchronization functions to inject sleep/migration before or after. -#if defined(OS_LINUX) && !defined(THREAD_SANITIZER) && !defined(MEMORY_SANITIZER) - #define THREAD_FUZZER_WRAP_PTHREAD 1 -#else - #define THREAD_FUZZER_WRAP_PTHREAD 0 -#endif - -#if THREAD_FUZZER_WRAP_PTHREAD -# define FOR_EACH_WRAPPED_FUNCTION(M) \ - M(int, pthread_mutex_lock, pthread_mutex_t * arg) \ - M(int, pthread_mutex_unlock, pthread_mutex_t * arg) -#endif - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int CANNOT_MANIPULATE_SIGSET; - extern const int CANNOT_SET_SIGNAL_HANDLER; - extern const int CANNOT_CREATE_TIMER; -} - - -ThreadFuzzer::ThreadFuzzer() -{ - initConfiguration(); - if (!isEffective()) - return; - setup(); -} - - -template -static void initFromEnv(T & what, const char * name) -{ - const char * env = getenv(name); - if (!env) - return; - what = parse(env); -} - -template -static void initFromEnv(std::atomic & what, const char * name) -{ - const char * env = getenv(name); - if (!env) - return; - what.store(parse(env), std::memory_order_relaxed); -} - - -static std::atomic num_cpus = 0; - -#if THREAD_FUZZER_WRAP_PTHREAD -# define DEFINE_WRAPPER_PARAMS(RET, NAME, ...) \ - static std::atomic NAME##_before_yield_probability = 0; \ - static std::atomic NAME##_before_migrate_probability = 0; \ - static std::atomic NAME##_before_sleep_probability = 0; \ - static std::atomic NAME##_before_sleep_time_us = 0; \ -\ - static std::atomic NAME##_after_yield_probability = 0; \ - static std::atomic NAME##_after_migrate_probability = 0; \ - static std::atomic NAME##_after_sleep_probability = 0; \ - static std::atomic NAME##_after_sleep_time_us = 0; - -FOR_EACH_WRAPPED_FUNCTION(DEFINE_WRAPPER_PARAMS) - -# undef DEFINE_WRAPPER_PARAMS -#endif - -void ThreadFuzzer::initConfiguration() -{ -#if defined(OS_LINUX) - num_cpus.store(get_nprocs(), std::memory_order_relaxed); -#else - (void)num_cpus; -#endif - - initFromEnv(cpu_time_period_us, "THREAD_FUZZER_CPU_TIME_PERIOD_US"); - initFromEnv(yield_probability, "THREAD_FUZZER_YIELD_PROBABILITY"); - initFromEnv(migrate_probability, "THREAD_FUZZER_MIGRATE_PROBABILITY"); - initFromEnv(sleep_probability, "THREAD_FUZZER_SLEEP_PROBABILITY"); - initFromEnv(sleep_time_us, "THREAD_FUZZER_SLEEP_TIME_US"); - -#if THREAD_FUZZER_WRAP_PTHREAD -# define INIT_WRAPPER_PARAMS(RET, NAME, ...) \ - initFromEnv(NAME##_before_yield_probability, "THREAD_FUZZER_" #NAME "_BEFORE_YIELD_PROBABILITY"); \ - initFromEnv(NAME##_before_migrate_probability, "THREAD_FUZZER_" #NAME "_BEFORE_MIGRATE_PROBABILITY"); \ - initFromEnv(NAME##_before_sleep_probability, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_PROBABILITY"); \ - initFromEnv(NAME##_before_sleep_time_us, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_TIME_US"); \ -\ - initFromEnv(NAME##_after_yield_probability, "THREAD_FUZZER_" #NAME "_AFTER_YIELD_PROBABILITY"); \ - initFromEnv(NAME##_after_migrate_probability, "THREAD_FUZZER_" #NAME "_AFTER_MIGRATE_PROBABILITY"); \ - initFromEnv(NAME##_after_sleep_probability, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_PROBABILITY"); \ - initFromEnv(NAME##_after_sleep_time_us, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_TIME_US"); - - FOR_EACH_WRAPPED_FUNCTION(INIT_WRAPPER_PARAMS) - -# undef INIT_WRAPPER_PARAMS -#endif -} - - -bool ThreadFuzzer::isEffective() const -{ -#if THREAD_FUZZER_WRAP_PTHREAD -# define CHECK_WRAPPER_PARAMS(RET, NAME, ...) \ - if (NAME##_before_yield_probability.load(std::memory_order_relaxed)) \ - return true; \ - if (NAME##_before_migrate_probability.load(std::memory_order_relaxed)) \ - return true; \ - if (NAME##_before_sleep_probability.load(std::memory_order_relaxed)) \ - return true; \ - if (NAME##_before_sleep_time_us.load(std::memory_order_relaxed)) \ - return true; \ -\ - if (NAME##_after_yield_probability.load(std::memory_order_relaxed)) \ - return true; \ - if (NAME##_after_migrate_probability.load(std::memory_order_relaxed)) \ - return true; \ - if (NAME##_after_sleep_probability.load(std::memory_order_relaxed)) \ - return true; \ - if (NAME##_after_sleep_time_us.load(std::memory_order_relaxed)) \ - return true; - - FOR_EACH_WRAPPED_FUNCTION(CHECK_WRAPPER_PARAMS) - -# undef INIT_WRAPPER_PARAMS -#endif - - return cpu_time_period_us != 0 - && (yield_probability > 0 - || migrate_probability > 0 - || (sleep_probability > 0 && sleep_time_us > 0)); -} - - -static void injection( - double yield_probability, - double migrate_probability, - double sleep_probability, - double sleep_time_us [[maybe_unused]]) -{ - if (yield_probability > 0 - && std::bernoulli_distribution(yield_probability)(thread_local_rng)) - { - sched_yield(); - } - -#if defined(OS_LINUX) - int num_cpus_loaded = num_cpus.load(std::memory_order_relaxed); - if (num_cpus_loaded > 0 - && migrate_probability > 0 - && std::bernoulli_distribution(migrate_probability)(thread_local_rng)) - { - int migrate_to = std::uniform_int_distribution<>(0, num_cpus_loaded - 1)(thread_local_rng); - - cpu_set_t set{}; - CPU_ZERO(&set); - CPU_SET(migrate_to, &set); - - (void)sched_setaffinity(0, sizeof(set), &set); - } -#else - UNUSED(migrate_probability); -#endif - - if (sleep_probability > 0 - && sleep_time_us > 0 - && std::bernoulli_distribution(sleep_probability)(thread_local_rng)) - { - sleepForNanoseconds(sleep_time_us * 1000); - } -} - - -void ThreadFuzzer::signalHandler(int) -{ - DENY_ALLOCATIONS_IN_SCOPE; - auto saved_errno = errno; - - auto & fuzzer = ThreadFuzzer::instance(); - injection(fuzzer.yield_probability, fuzzer.migrate_probability, fuzzer.sleep_probability, fuzzer.sleep_time_us); - - errno = saved_errno; -} - -void ThreadFuzzer::setup() const -{ - struct sigaction sa{}; - sa.sa_handler = signalHandler; - sa.sa_flags = SA_RESTART; - -#if defined(OS_LINUX) - if (sigemptyset(&sa.sa_mask)) - throwFromErrno("Failed to clean signal mask for thread fuzzer", ErrorCodes::CANNOT_MANIPULATE_SIGSET); - - if (sigaddset(&sa.sa_mask, SIGPROF)) - throwFromErrno("Failed to add signal to mask for thread fuzzer", ErrorCodes::CANNOT_MANIPULATE_SIGSET); -#else - // the two following functions always return 0 under mac - sigemptyset(&sa.sa_mask); - sigaddset(&sa.sa_mask, SIGPROF); -#endif - - if (sigaction(SIGPROF, &sa, nullptr)) - throwFromErrno("Failed to setup signal handler for thread fuzzer", ErrorCodes::CANNOT_SET_SIGNAL_HANDLER); - - static constexpr UInt32 timer_precision = 1000000; - - struct timeval interval; - interval.tv_sec = cpu_time_period_us / timer_precision; - interval.tv_usec = cpu_time_period_us % timer_precision; - - struct itimerval timer = {.it_interval = interval, .it_value = interval}; - - if (0 != setitimer(ITIMER_PROF, &timer, nullptr)) - throwFromErrno("Failed to create profiling timer", ErrorCodes::CANNOT_CREATE_TIMER); -} - - -/// We expect that for every function like pthread_mutex_lock there is the same function with two underscores prefix. -/// NOTE We cannot use dlsym(... RTLD_NEXT), because it will call pthread_mutex_lock and it will lead to infinite recursion. - -#if THREAD_FUZZER_WRAP_PTHREAD -# define MAKE_WRAPPER(RET, NAME, ...) \ - extern "C" RET __##NAME(__VA_ARGS__); /* NOLINT */ \ - extern "C" RET NAME(__VA_ARGS__) /* NOLINT */ \ - { \ - injection( \ - NAME##_before_yield_probability.load(std::memory_order_relaxed), \ - NAME##_before_migrate_probability.load(std::memory_order_relaxed), \ - NAME##_before_sleep_probability.load(std::memory_order_relaxed), \ - NAME##_before_sleep_time_us.load(std::memory_order_relaxed)); \ -\ - auto && ret{__##NAME(arg)}; \ -\ - injection( \ - NAME##_after_yield_probability.load(std::memory_order_relaxed), \ - NAME##_after_migrate_probability.load(std::memory_order_relaxed), \ - NAME##_after_sleep_probability.load(std::memory_order_relaxed), \ - NAME##_after_sleep_time_us.load(std::memory_order_relaxed)); \ -\ - return ret; \ - } - -FOR_EACH_WRAPPED_FUNCTION(MAKE_WRAPPER) - -# undef MAKE_WRAPPER -#endif -} diff --git a/src/Common/ThreadFuzzer.h b/src/Common/ThreadFuzzer.h deleted file mode 100644 index 1a9e98ca674..00000000000 --- a/src/Common/ThreadFuzzer.h +++ /dev/null @@ -1,72 +0,0 @@ -#pragma once -#include - - -namespace DB -{ - -/** Allows to randomize thread scheduling and insert various glitches across whole program for testing purposes. - * It is done by setting up a timer that will send PROF signal to every thread when certain amount of CPU time has passed. - * - * To initialize ThreadFuzzer, call ThreadFuzzer::instance(). - * The behaviour is controlled by environment variables: - * - * THREAD_FUZZER_CPU_TIME_PERIOD_US - period of signals in microseconds. - * THREAD_FUZZER_YIELD_PROBABILITY - probability to do 'sched_yield'. - * THREAD_FUZZER_MIGRATE_PROBABILITY - probability to set CPU affinity to random CPU core. - * THREAD_FUZZER_SLEEP_PROBABILITY - probability to sleep. - * THREAD_FUZZER_SLEEP_TIME_US - amount of time to sleep in microseconds. - * - * ThreadFuzzer will do nothing if environment variables are not set accordingly. - * - * The intention is to reproduce thread synchronization bugs (race conditions and deadlocks) more frequently in tests. - * We already have tests with TSan. But TSan only covers "physical" synchronization bugs, but not "logical" ones, - * where all data is protected by synchronization primitives, but we still have race conditions. - * Obviously, TSan cannot debug distributed synchronization bugs. - * - * The motivation for this tool is an evidence, that concurrency bugs are more likely to reproduce - * on bad unstable virtual machines in a dirty environments. - * - * The idea is not new, see also: - * https://channel9.msdn.com/blogs/peli/concurrency-fuzzing-with-cuzz - * - * Notes: - * - it can be also implemented with instrumentation (example: LLVM Xray) instead of signals. - * - we should also make the sleep time random. - * - sleep and migration obviously helps, but the effect of yield is unclear. - * - * In addition, we allow to inject glitches around thread synchronization functions. - * Example: - * - * THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_PROBABILITY=0.001 - * THREAD_FUZZER_pthread_mutex_lock_BEFORE_SLEEP_TIME_US=10000 - * THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_PROBABILITY=0.001 - * THREAD_FUZZER_pthread_mutex_lock_AFTER_SLEEP_TIME_US=10000 - */ -class ThreadFuzzer -{ -public: - static ThreadFuzzer & instance() - { - static ThreadFuzzer res; - return res; - } - - bool isEffective() const; - -private: - uint64_t cpu_time_period_us = 0; - double yield_probability = 0; - double migrate_probability = 0; - double sleep_probability = 0; - double sleep_time_us = 0; - - ThreadFuzzer(); - - void initConfiguration(); - void setup() const; - - static void signalHandler(int); -}; - -} diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index 0e12830e49d..d546f5f4502 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -44,7 +44,7 @@ namespace struct ThreadStack { ThreadStack() - : data(aligned_alloc(getPageSize(), size)) + : data(aligned_alloc(getPageSize(), getSize())) { /// Add a guard page /// (and since the stack grows downward, we need to protect the first page). @@ -56,12 +56,11 @@ struct ThreadStack free(data); } - static size_t getSize() { return size; } + static size_t getSize() { return std::max(16 << 10, MINSIGSTKSZ); } void * getData() const { return data; } private: /// 16 KiB - not too big but enough to handle error. - static constexpr size_t size = std::max(16 << 10, MINSIGSTKSZ); void * data; }; diff --git a/src/Common/examples/chaos_sanitizer.cpp b/src/Common/examples/chaos_sanitizer.cpp index 05bf16e3638..f09876ec8d2 100644 --- a/src/Common/examples/chaos_sanitizer.cpp +++ b/src/Common/examples/chaos_sanitizer.cpp @@ -7,7 +7,6 @@ #include #include -#include /** Proves that ThreadFuzzer helps to find concurrency bugs. @@ -19,7 +18,7 @@ int main(int argc, char ** argv) { const size_t num_iterations = argc >= 2 ? DB::parse(argv[1]) : 1000000000; - std::cerr << (DB::ThreadFuzzer::instance().isEffective() ? "ThreadFuzzer is enabled.\n" : "ThreadFuzzer is not enabled.\n"); + std::cerr << "ThreadFuzzer is not enabled.\n"; std::atomic counter1 = 0; std::atomic counter2 = 0; diff --git a/src/Common/malloc.cpp b/src/Common/malloc.cpp index ec472d5d1d6..275102992ad 100644 --- a/src/Common/malloc.cpp +++ b/src/Common/malloc.cpp @@ -36,7 +36,7 @@ static void dummyFunctionForInterposing() ignore(calloc(0, 0)); // -V575 NOLINT ignore(realloc(nullptr, 0)); // -V575 NOLINT ignore(posix_memalign(&dummy, 0, 0)); // -V575 NOLINT - ignore(aligned_alloc(0, 0)); // -V575 NOLINT + ignore(aligned_alloc(2, 0)); // -V575 NOLINT ignore(valloc(0)); // -V575 NOLINT ignore(memalign(0, 0)); // -V575 NOLINT ignore(pvalloc(0)); // -V575 NOLINT diff --git a/src/Common/ya.make b/src/Common/ya.make index 60dfd5f6bee..3195fc4ea3d 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -81,7 +81,6 @@ SRCS( TLDListsHolder.cpp TaskStatsInfoGetter.cpp TerminalSize.cpp - ThreadFuzzer.cpp ThreadPool.cpp ThreadProfileEvents.cpp ThreadStatus.cpp diff --git a/src/Functions/hasThreadFuzzer.cpp b/src/Functions/hasThreadFuzzer.cpp deleted file mode 100644 index e9b48aa70f9..00000000000 --- a/src/Functions/hasThreadFuzzer.cpp +++ /dev/null @@ -1,53 +0,0 @@ -#include -#include -#include -#include - - -namespace DB -{ -namespace -{ - -/** Returns whether Thread Fuzzer is effective. - * It can be used in tests to prevent too long runs. - */ -class FunctionHasThreadFuzzer : public IFunction -{ -public: - static constexpr auto name = "hasThreadFuzzer"; - static FunctionPtr create(ContextPtr) - { - return std::make_shared(); - } - - String getName() const override - { - return name; - } - - size_t getNumberOfArguments() const override - { - return 0; - } - - DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override - { - return std::make_shared(); - } - - ColumnPtr executeImpl(const ColumnsWithTypeAndName &, const DataTypePtr &, size_t input_rows_count) const override - { - return DataTypeUInt8().createColumnConst(input_rows_count, ThreadFuzzer::instance().isEffective()); - } -}; - -} - -void registerFunctionHasThreadFuzzer(FunctionFactory & factory) -{ - factory.registerFunction(); -} - -} - diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp index 3403390ea72..410b3ff34fc 100644 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/src/Functions/registerFunctionsMiscellaneous.cpp @@ -65,7 +65,6 @@ void registerFunctionIsConstant(FunctionFactory &); void registerFunctionIsDecimalOverflow(FunctionFactory &); void registerFunctionCountDigits(FunctionFactory &); void registerFunctionGlobalVariable(FunctionFactory &); -void registerFunctionHasThreadFuzzer(FunctionFactory &); void registerFunctionInitializeAggregation(FunctionFactory &); void registerFunctionErrorCodeToName(FunctionFactory &); void registerFunctionTcpPort(FunctionFactory &); @@ -139,7 +138,6 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionIsDecimalOverflow(factory); registerFunctionCountDigits(factory); registerFunctionGlobalVariable(factory); - registerFunctionHasThreadFuzzer(factory); registerFunctionInitializeAggregation(factory); registerFunctionErrorCodeToName(factory); registerFunctionTcpPort(factory); diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 5f84511aa52..c0a6d61cc6c 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -301,7 +301,6 @@ SRCS( h3ToString.cpp h3kRing.cpp hasColumnInTable.cpp - hasThreadFuzzer.cpp hasToken.cpp hasTokenCaseInsensitive.cpp hostName.cpp diff --git a/tests/queries/0_stateless/01193_metadata_loading.sh b/tests/queries/0_stateless/01193_metadata_loading.sh index a3154a0784d..c75b5ffbc2e 100755 --- a/tests/queries/0_stateless/01193_metadata_loading.sh +++ b/tests/queries/0_stateless/01193_metadata_loading.sh @@ -12,7 +12,7 @@ threads=10 count_multiplier=1 max_time_ms=1000 -debug_or_sanitizer_build=$($CLICKHOUSE_CLIENT -q "WITH ((SELECT value FROM system.build_options WHERE name='BUILD_TYPE') AS build, (SELECT value FROM system.build_options WHERE name='CXX_FLAGS') as flags) SELECT build='Debug' OR flags LIKE '%fsanitize%' OR hasThreadFuzzer()") +debug_or_sanitizer_build=$($CLICKHOUSE_CLIENT -q "WITH ((SELECT value FROM system.build_options WHERE name='BUILD_TYPE') AS build, (SELECT value FROM system.build_options WHERE name='CXX_FLAGS') as flags) SELECT build='Debug' OR flags LIKE '%fsanitize%'") if [[ debug_or_sanitizer_build -eq 1 ]]; then tables=100; count_multiplier=10; max_time_ms=1500; fi