From ce146c8b558ed2e83fd8c06d2978d3a47abe35fb Mon Sep 17 00:00:00 2001

From: Felix Schwarz <felix.schwarz@oss.schwarz.eu>

Date: Mon, 3 Jun 2019 23:34:20 +0200

Subject: [PATCH 1/2] reference struct "XXH64_state_t" only via an opaque

 pointer

Upstream recommends this whenever xxhash is linked dynamically:

https://github.com/Cyan4973/xxHash/issues/63#issuecomment-218376237

This change is required to unbundle xxhash in the next commit.

---

 src/borg/algorithms/checksums.pyx | 14 ++++++++++----

 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/borg/algorithms/checksums.pyx b/src/borg/algorithms/checksums.pyx

index 6645dd0fe..f848a2dd5 100644

--- a/src/borg/algorithms/checksums.pyx

+++ b/src/borg/algorithms/checksums.pyx

@@ -25,6 +25,8 @@ cdef extern from "xxh64/xxhash.c":

         XXH_OK,

         XXH_ERROR

+    XXH64_state_t* XXH64_createState();

+    XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);

     XXH64_hash_t XXH64(const void* input, size_t length, unsigned long long seed);

     XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed);

@@ -80,17 +82,21 @@ def xxh64(data, seed=0):

 cdef class StreamingXXH64:

-    cdef XXH64_state_t state

+    cdef XXH64_state_t* state

     def __cinit__(self, seed=0):

+        self.state = XXH64_createState()

         cdef unsigned long long _seed = seed

-        if XXH64_reset(&self.state, _seed) != XXH_OK:

+        if XXH64_reset(self.state, _seed) != XXH_OK:

             raise Exception('XXH64_reset failed')

+    def __dealloc__(self):

+        XXH64_freeState(self.state)

+

     def update(self, data):

         cdef Py_buffer data_buf = ro_buffer(data)

         try:

-            if XXH64_update(&self.state, data_buf.buf, data_buf.len) != XXH_OK:

+            if XXH64_update(self.state, data_buf.buf, data_buf.len) != XXH_OK:

                 raise Exception('XXH64_update failed')

         finally:

             PyBuffer_Release(&data_buf)

@@ -98,7 +104,7 @@ cdef class StreamingXXH64:

     def digest(self):

         cdef XXH64_hash_t hash

         cdef XXH64_canonical_t digest

-        hash = XXH64_digest(&self.state)

+        hash = XXH64_digest(self.state)

         XXH64_canonicalFromHash(&digest, hash)

         return PyBytes_FromStringAndSize(<const char*> digest.digest, 8)

From 2ff06c58f0488a82e17ea8f22bd3326f54959070 Mon Sep 17 00:00:00 2001

From: Felix Schwarz <felix.schwarz@oss.schwarz.eu>

Date: Mon, 3 Jun 2019 23:37:46 +0200

Subject: [PATCH 2/2] ability to use a system-provided version of "xxhash"

The build process can be controlled via environment variables

similar to other bundled libraries in borgbackup. The main difference

is probably that upstream does not provide a pkgconfig file for

xxhash.

Therefore borg will probably fail to detect the system-provided

version by default (tested on Fedora, seems like Debian and Ubuntu

do not ship a pkgconfig file either). I kept the pkgconfig lookup

code anyway to keep the code as similar as possible to

"setup_compress.py"/"setup_crypto.py".

Setting BORG_LIBXXHASH_PREFIX=/usr helps borg to detect xxhash

on my system (Fedora). You can force the use of the bundled

version of xxhash by setting BORG_USE_BUNDLED_XXHASH=1.

---

 setup.py                               | 11 ++++++-

 setup_checksums.py                     | 42 ++++++++++++++++++++++++++

 src/borg/algorithms/checksums.pyx      |  2 +-

 src/borg/algorithms/xxhash-libselect.h |  5 +++

 4 files changed, 58 insertions(+), 2 deletions(-)

 create mode 100644 setup_checksums.py

 create mode 100644 src/borg/algorithms/xxhash-libselect.h

diff --git a/src/borg/algorithms/checksums.pyx b/src/borg/algorithms/checksums.pyx

index f848a2dd5..95081649b 100644

--- a/src/borg/algorithms/checksums.pyx

+++ b/src/borg/algorithms/checksums.pyx

@@ -12,7 +12,7 @@ cdef extern from "crc32_dispatch.c":

     int _have_clmul "have_clmul"()

-cdef extern from "xxh64/xxhash.c":

+cdef extern from "xxhash-libselect.h":

     ctypedef struct XXH64_canonical_t:

         char digest[8]

diff --git a/src/borg/algorithms/xxhash-libselect.h b/src/borg/algorithms/xxhash-libselect.h

new file mode 100644

index 000000000..ff7edffa1

--- /dev/null

+++ b/src/borg/algorithms/xxhash-libselect.h

@@ -0,0 +1,5 @@

+#ifdef BORG_USE_BUNDLED_XXHASH

+#include "xxh64/xxhash.h"

+#else

+#include <xxhash.h>

+#endif

diff --git a/borg/setup.py b/borg/setup.py

index 8cdaef3..5a666c8 100644

--- a/setup.py

+++ b/setup.py

@@ -826,7 +826,7 @@ def run(self):

         Extension('borg.hashindex', [hashindex_source]),

         Extension('borg.item', [item_source]),

         Extension('borg.chunker', [chunker_source]),

-        Extension('borg.algorithms.checksums', [checksums_source]),

+        Extension('borg.algorithms.checksums', [checksums_source], libraries=['xxhash']),

     ]

     if not sys.platform.startswith(('win32', )):

         ext_modules.append(Extension('borg.platform.posix', [platform_posix_source]))