From ce146c8b558ed2e83fd8c06d2978d3a47abe35fb Mon Sep 17 00:00:00 2001 From: Felix Schwarz Date: Mon, 3 Jun 2019 23:34:20 +0200 Subject: [PATCH 1/2] reference struct "XXH64_state_t" only via an opaque pointer Upstream recommends this whenever xxhash is linked dynamically: https://github.com/Cyan4973/xxHash/issues/63#issuecomment-218376237 This change is required to unbundle xxhash in the next commit. --- src/borg/algorithms/checksums.pyx | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/borg/algorithms/checksums.pyx b/src/borg/algorithms/checksums.pyx index 6645dd0fe..f848a2dd5 100644 --- a/src/borg/algorithms/checksums.pyx +++ b/src/borg/algorithms/checksums.pyx @@ -25,6 +25,8 @@ cdef extern from "xxh64/xxhash.c": XXH_OK, XXH_ERROR + XXH64_state_t* XXH64_createState(); + XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); XXH64_hash_t XXH64(const void* input, size_t length, unsigned long long seed); XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed); @@ -80,17 +82,21 @@ def xxh64(data, seed=0): cdef class StreamingXXH64: - cdef XXH64_state_t state + cdef XXH64_state_t* state def __cinit__(self, seed=0): + self.state = XXH64_createState() cdef unsigned long long _seed = seed - if XXH64_reset(&self.state, _seed) != XXH_OK: + if XXH64_reset(self.state, _seed) != XXH_OK: raise Exception('XXH64_reset failed') + def __dealloc__(self): + XXH64_freeState(self.state) + def update(self, data): cdef Py_buffer data_buf = ro_buffer(data) try: - if XXH64_update(&self.state, data_buf.buf, data_buf.len) != XXH_OK: + if XXH64_update(self.state, data_buf.buf, data_buf.len) != XXH_OK: raise Exception('XXH64_update failed') finally: PyBuffer_Release(&data_buf) @@ -98,7 +104,7 @@ cdef class StreamingXXH64: def digest(self): cdef XXH64_hash_t hash cdef XXH64_canonical_t digest - hash = XXH64_digest(&self.state) + hash = XXH64_digest(self.state) XXH64_canonicalFromHash(&digest, hash) return PyBytes_FromStringAndSize( digest.digest, 8) From 2ff06c58f0488a82e17ea8f22bd3326f54959070 Mon Sep 17 00:00:00 2001 From: Felix Schwarz Date: Mon, 3 Jun 2019 23:37:46 +0200 Subject: [PATCH 2/2] ability to use a system-provided version of "xxhash" The build process can be controlled via environment variables similar to other bundled libraries in borgbackup. The main difference is probably that upstream does not provide a pkgconfig file for xxhash. Therefore borg will probably fail to detect the system-provided version by default (tested on Fedora, seems like Debian and Ubuntu do not ship a pkgconfig file either). I kept the pkgconfig lookup code anyway to keep the code as similar as possible to "setup_compress.py"/"setup_crypto.py". Setting BORG_LIBXXHASH_PREFIX=/usr helps borg to detect xxhash on my system (Fedora). You can force the use of the bundled version of xxhash by setting BORG_USE_BUNDLED_XXHASH=1. --- setup.py | 11 ++++++- setup_checksums.py | 42 ++++++++++++++++++++++++++ src/borg/algorithms/checksums.pyx | 2 +- src/borg/algorithms/xxhash-libselect.h | 5 +++ 4 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 setup_checksums.py create mode 100644 src/borg/algorithms/xxhash-libselect.h diff --git a/src/borg/algorithms/checksums.pyx b/src/borg/algorithms/checksums.pyx index f848a2dd5..95081649b 100644 --- a/src/borg/algorithms/checksums.pyx +++ b/src/borg/algorithms/checksums.pyx @@ -12,7 +12,7 @@ cdef extern from "crc32_dispatch.c": int _have_clmul "have_clmul"() -cdef extern from "xxh64/xxhash.c": +cdef extern from "xxhash-libselect.h": ctypedef struct XXH64_canonical_t: char digest[8] diff --git a/src/borg/algorithms/xxhash-libselect.h b/src/borg/algorithms/xxhash-libselect.h new file mode 100644 index 000000000..ff7edffa1 --- /dev/null +++ b/src/borg/algorithms/xxhash-libselect.h @@ -0,0 +1,5 @@ +#ifdef BORG_USE_BUNDLED_XXHASH +#include "xxh64/xxhash.h" +#else +#include +#endif diff --git a/borg/setup.py b/borg/setup.py index 8cdaef3..5a666c8 100644 --- a/setup.py +++ b/setup.py @@ -826,7 +826,7 @@ def run(self): Extension('borg.hashindex', [hashindex_source]), Extension('borg.item', [item_source]), Extension('borg.chunker', [chunker_source]), - Extension('borg.algorithms.checksums', [checksums_source]), + Extension('borg.algorithms.checksums', [checksums_source], libraries=['xxhash']), ] if not sys.platform.startswith(('win32', )): ext_modules.append(Extension('borg.platform.posix', [platform_posix_source]))