Group :: Development/Tools
RPM: ccache
Navigation
Main Changelog Spec Patches Sources Download Gear Bugs and FR Repocop
Patch: ccache-2.4-alt-debian-compressed-cache.patch
Download
Download
Index: ccache.1
===================================================================
RCS file: /home/cvsroot/lars/ccache/ccache.1,v
retrieving revision 1.1.1.1.2.1
retrieving revision 1.6
diff -u -r1.1.1.1.2.1 -r1.6
--- ccache.1 21 Nov 2004 17:55:36 -0000 1.1.1.1.2.1
+++ ccache.1 21 Nov 2004 18:19:28 -0000 1.6
@@ -210,7 +210,8 @@
CCACHE_HARDLINK then ccache will attempt to use hard links from the
cache directory when creating the compiler output rather than using a
file copy\&. Using hard links is faster, but can confuse programs like
-\&'make\&' that rely on modification times\&.
+\&'make\&' that rely on modification times\&. Hard links are never made for
+compressed cache files\&.
.IP
.IP "\fBCCACHE_RECACHE\fP"
This forces ccache to not use any cached
@@ -257,6 +258,11 @@
the default\&. On HP-UX set this environment variable to "i" if you use
the aCC compiler\&.
.IP
+.IP "\fBCCACHE_NOCOMPRESS\fP"
+If you set the environment variable
+CCACHE_NOCOMPRESS then there is no compression used on files that go
+into the cache\&.
+.IP
.PP
.SH "CACHE SIZE MANAGEMENT"
.PP
@@ -269,6 +275,14 @@
below the numbers you specified in order to avoid doing the cache
clean operation too often\&.
.PP
+.SH "CACHE COMPRESSION"
+.PP
+By default ccache will compress all files it puts into the cache
+using the zlib compression\&. While this involves a negligible
+performance slowdown, it significantly increases the number of files
+that fit in the cache\&. You can turn off compression setting the
+CCACHE_NOCOMPRESS environment variable\&.
+.PP
.SH "HOW IT WORKS"
.PP
The basic idea is to detect when you are compiling exactly the same
Index: ccache.c
===================================================================
RCS file: /home/cvsroot/lars/ccache/ccache.c,v
retrieving revision 1.1.1.1.2.1
retrieving revision 1.9
diff -u -r1.1.1.1.2.1 -r1.9
--- ccache.c 21 Nov 2004 17:55:36 -0000 1.1.1.1.2.1
+++ ccache.c 21 Nov 2004 18:19:28 -0000 1.9
@@ -199,7 +199,7 @@
fd = open(tmp_stderr, O_RDONLY | O_BINARY);
if (fd != -1) {if (strcmp(output_file, "/dev/null") == 0 ||
- rename(tmp_hashname, output_file) == 0 || errno == ENOENT) {+ move_file(tmp_hashname, output_file) == 0 || errno == ENOENT) { if (cpp_stderr) {/* we might have some stderr from cpp */
int fd2 = open(cpp_stderr, O_RDONLY | O_BINARY);
@@ -231,14 +231,25 @@
x_asprintf(&path_stderr, "%s.stderr", hashname);
if (stat(tmp_stderr, &st1) != 0 ||
- stat(tmp_hashname, &st2) != 0 ||
- rename(tmp_hashname, hashname) != 0 ||
- rename(tmp_stderr, path_stderr) != 0) {+ stat(tmp_hashname, &st2) != 0 ||
+ move_file(tmp_hashname, hashname) != 0 ||
+ move_file(tmp_stderr, path_stderr) != 0) { cc_log("failed to rename tmp files - %s\n", strerror(errno));stats_update(STATS_ERROR);
failed();
}
+#if ENABLE_ZLIB
+ /* do an extra stat on the cache files for
+ the size statistics */
+ if (stat(path_stderr, &st1) != 0 ||
+ stat(hashname, &st2) != 0) {+ cc_log("failed to stat cache files - %s\n", strerror(errno));+ stats_update(STATS_ERROR);
+ failed();
+ }
+#endif
+
cc_log("Placed %s into cache\n", output_file);stats_tocache(file_size(&st1) + file_size(&st2));
@@ -474,7 +485,13 @@
}
/* the user might be disabling cache hits */
+#ifndef ENABLE_ZLIB
+ /* if the cache file is compressed we must recache */
+ if ((first && getenv("CCACHE_RECACHE")) ||+ test_if_compressed(hashname) == 1) {+#else
if (first && getenv("CCACHE_RECACHE")) {+#endif
close(fd_stderr);
unlink(stderr_file);
free(stderr_file);
@@ -487,7 +504,9 @@
ret = 0;
} else {unlink(output_file);
- if (getenv("CCACHE_HARDLINK")) {+ /* only make a hardlink if the cache file is uncompressed */
+ if (getenv("CCACHE_HARDLINK") &&+ test_if_compressed(hashname) == 0) {ret = link(hashname, output_file);
} else {ret = copy_file(hashname, output_file);
Index: ccache.h
===================================================================
RCS file: /home/cvsroot/lars/ccache/ccache.h,v
retrieving revision 1.1.1.1.2.1
retrieving revision 1.7
diff -u -r1.1.1.1.2.1 -r1.7
--- ccache.h 21 Nov 2004 17:55:36 -0000 1.1.1.1.2.1
+++ ccache.h 21 Nov 2004 18:19:28 -0000 1.7
@@ -23,6 +23,10 @@
#include <pwd.h>
#endif
+#ifdef ENABLE_ZLIB
+#include <zlib.h>
+#endif
+
#define STATUS_NOTFOUND 3
#define STATUS_FATAL 4
#define STATUS_NOCACHE 5
@@ -36,6 +40,13 @@
#define DEFAULT_MAXSIZE (1000*1000)
#endif
+/* file copy mode */
+#ifdef ENABLE_ZLIB
+#define COPY_UNCOMPRESSED 0
+#define COPY_FROM_CACHE 1
+#define COPY_TO_CACHE 2
+#endif
+
enum stats {STATS_NONE=0,
STATS_STDOUT,
@@ -79,6 +90,8 @@
void copy_fd(int fd_in, int fd_out);
int copy_file(const char *src, const char *dest);
+int move_file(const char *src, const char *dest);
+int test_if_compressed(const char *filename);
int create_dir(const char *dir);
void x_asprintf(char **ptr, const char *format, ...);
Index: ccache.yo
===================================================================
RCS file: /home/cvsroot/lars/ccache/ccache.yo,v
retrieving revision 1.1.1.1.2.1
retrieving revision 1.5
diff -u -r1.1.1.1.2.1 -r1.5
--- ccache.yo 21 Nov 2004 17:55:36 -0000 1.1.1.1.2.1
+++ ccache.yo 21 Nov 2004 18:19:28 -0000 1.5
@@ -169,6 +169,11 @@
this optimisation, in which case this option could allow ccache to be
used.
+dit(bf(CCACHE_NOCOMPRESS)) If you set the environment variable
+CCACHE_NOCOMPRESS then there is no compression used on files that go
+into the cache. However, this setting has no effect on how files are
+retrieved from the cache, compressed results will still be usable.
+
dit(bf(CCACHE_NOSTATS)) If you set the environment variable
CCACHE_NOSTATS then ccache will not update the statistics files on
each compile.
@@ -181,7 +186,8 @@
CCACHE_HARDLINK then ccache will attempt to use hard links from the
cache directory when creating the compiler output rather than using a
file copy. Using hard links is faster, but can confuse programs like
-'make' that rely on modification times.
+'make' that rely on modification times. Hard links are never made for
+compressed cache files.
dit(bf(CCACHE_RECACHE)) This forces ccache to not use any cached
results, even if it finds them. New results are still cached, but
@@ -236,6 +242,14 @@
below the numbers you specified in order to avoid doing the cache
clean operation too often.
+manpagesection(CACHE COMPRESSION)
+
+By default ccache will compress all files it puts into the cache
+using the zlib compression. While this involves a negligible
+performance slowdown, it significantly increases the number of files
+that fit in the cache. You can turn off compression setting the
+CCACHE_NOCOMPRESS environment variable.
+
manpagesection(HOW IT WORKS)
The basic idea is to detect when you are compiling exactly the same
@@ -294,6 +308,8 @@
cache. This tells the filesystem to inherit group ownership for new
directories. The command "chmod g+s `find $CCACHE_DIR -type d`" might
be useful for this.
+ it() Set bf(CCACHE_NOCOMPRESS) for all users, if there are users with
+ versions of ccache that do not support compression.
)
manpagesection(HISTORY)
Index: config.h.in
===================================================================
RCS file: /home/cvsroot/lars/ccache/config.h.in,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- config.h.in 30 Apr 2004 13:13:41 -0000 1.1.1.1
+++ config.h.in 4 May 2004 20:49:26 -0000 1.2
@@ -98,3 +98,6 @@
/* Define _GNU_SOURCE so that we get all necessary prototypes */
#undef _GNU_SOURCE
+
+/* Define to 1 if you like to have zlib compression for the ccache. */
+#undef ENABLE_ZLIB
Index: configure.in
===================================================================
RCS file: /home/cvsroot/lars/ccache/configure.in,v
retrieving revision 1.1.1.1.2.1
retrieving revision 1.4
diff -u -r1.1.1.1.2.1 -r1.4
--- configure.in 21 Nov 2004 17:55:36 -0000 1.1.1.1.2.1
+++ configure.in 21 Nov 2004 18:19:28 -0000 1.4
@@ -68,5 +68,15 @@
AC_DEFINE(HAVE_C99_VSNPRINTF, 1, [ ])
fi
+dnl Check for zlib.
+AC_ARG_ENABLE([zlib],
+ AS_HELP_STRING([--enable-zlib], [enable zlib support for ccache compression]),,
+ [enable_zlib=yes])
+
+if test x"$enable_zlib" = x"yes"; then
+ AC_CHECK_HEADER(zlib.h, AC_CHECK_LIB(z, gzdopen, LIBS="-lz $LIBS";
+ AC_DEFINE([ENABLE_ZLIB],[1],[Define ENABLE_ZLIB if you want to enable zlib])))
+fi
+
AC_CONFIG_FILES([Makefile])
AC_OUTPUT
Index: util.c
===================================================================
RCS file: /home/cvsroot/lars/ccache/util.c,v
retrieving revision 1.1.1.1.2.1
retrieving revision 1.11
diff -u -r1.1.1.1.2.1 -r1.11
--- util.c 21 Nov 2004 17:55:36 -0000 1.1.1.1.2.1
+++ util.c 21 Nov 2004 18:19:28 -0000 1.11
@@ -44,6 +44,7 @@
exit(1);
}
+#ifndef ENABLE_ZLIB
/* copy all data from one file descriptor to another */
void copy_fd(int fd_in, int fd_out)
{@@ -57,6 +58,11 @@
}
}
+/* move a file using rename */
+int move_file(const char *src, const char *dest) {+ return rename(src, dest);
+}
+
/* copy a file - used when hard links don't work
the copy is done via a temporary file and atomic rename
*/
@@ -120,6 +126,174 @@
return 0;
}
+#else /* ENABLE_ZLIB */
+
+/* copy all data from one file descriptor to another
+ possibly decompressing it
+*/
+void copy_fd(int fd_in, int fd_out) {+ char buf[10240];
+ int n;
+ gzFile gz_in;
+
+ gz_in = gzdopen(dup(fd_in), "rb");
+
+ if (!gz_in) {+ fatal("Failed to copy fd");+ }
+
+ while ((n = gzread(gz_in, buf, sizeof(buf))) > 0) {+ if (write(fd_out, buf, n) != n) {+ fatal("Failed to copy fd");+ }
+ }
+}
+
+static int _copy_file(const char *src, const char *dest, int mode) {+ int fd_in, fd_out;
+ gzFile gz_in, gz_out = NULL;
+ char buf[10240];
+ int n, ret;
+ char *tmp_name;
+ mode_t mask;
+ struct stat st;
+
+ x_asprintf(&tmp_name, "%s.XXXXXX", dest);
+
+ if (getenv("CCACHE_NOCOMPRESS")) {+ mode = COPY_UNCOMPRESSED;
+ }
+
+ /* open source file */
+ fd_in = open(src, O_RDONLY);
+ if (fd_in == -1) {+ return -1;
+ }
+
+ gz_in = gzdopen(fd_in, "rb");
+ if (!gz_in) {+ close(fd_in);
+ return -1;
+ }
+
+ /* open destination file */
+ fd_out = mkstemp(tmp_name);
+ if (fd_out == -1) {+ gzclose(gz_in);
+ free(tmp_name);
+ return -1;
+ }
+
+ if (mode == COPY_TO_CACHE) {+ /* The gzip file format occupies at least 20 bytes. So
+ it will always occupy an entire filesystem block,
+ even for empty files.
+ Since most stderr files will be empty, we turn off
+ compression in this case to save space.
+ */
+ if (fstat(fd_in, &st) != 0) {+ gzclose(gz_in);
+ close(fd_out);
+ free(tmp_name);
+ return -1;
+ }
+ if (file_size(&st) == 0) {+ mode = COPY_UNCOMPRESSED;
+ }
+ }
+
+ if (mode == COPY_TO_CACHE) {+ gz_out = gzdopen(dup(fd_out), "wb");
+ if (!gz_out) {+ gzclose(gz_in);
+ close(fd_out);
+ free(tmp_name);
+ return -1;
+ }
+ }
+
+ while ((n = gzread(gz_in, buf, sizeof(buf))) > 0) {+ if (mode == COPY_TO_CACHE) {+ ret = gzwrite(gz_out, buf, n);
+ } else {+ ret = write(fd_out, buf, n);
+ }
+ if (ret != n) {+ gzclose(gz_in);
+ if (gz_out) {+ gzclose(gz_out);
+ }
+ close(fd_out);
+ unlink(tmp_name);
+ free(tmp_name);
+ return -1;
+ }
+ }
+
+ gzclose(gz_in);
+ if (gz_out) {+ gzclose(gz_out);
+ }
+
+ /* get perms right on the tmp file */
+ mask = umask(0);
+ fchmod(fd_out, 0666 & ~mask);
+ umask(mask);
+
+ /* the close can fail on NFS if out of space */
+ if (close(fd_out) == -1) {+ unlink(tmp_name);
+ free(tmp_name);
+ return -1;
+ }
+
+ unlink(dest);
+
+ if (rename(tmp_name, dest) == -1) {+ unlink(tmp_name);
+ free(tmp_name);
+ return -1;
+ }
+
+ free(tmp_name);
+
+ return 0;
+}
+
+/* move a file to the cache, compressing it */
+int move_file(const char *src, const char *dest) {+ int ret;
+
+ ret = _copy_file(src, dest, COPY_TO_CACHE);
+ if (ret != -1) unlink(src);
+ return ret;
+}
+
+/* copy a file from the cache, decompressing it */
+int copy_file(const char *src, const char *dest) {+ return _copy_file(src, dest, COPY_FROM_CACHE);
+}
+#endif /* ENABLE_ZLIB */
+
+/* test if a file is zlib compressed */
+int test_if_compressed(const char *filename) {+ FILE *f;
+
+ f = fopen(filename, "rb");
+ if (!f) {+ return 0;
+ }
+
+ /* test if file starts with 1F8B, which is zlib's
+ * magic number */
+ if ((fgetc(f) != 0x1f) || (fgetc(f) != 0x8b)) {+ fclose(f);
+ return 0;
+ }
+
+ fclose(f);
+ return 1;
+}
/* make sure a directory exists */
int create_dir(const char *dir)
Index: manage-cache.sh
===================================================================
RCS file: manage-cache.sh
diff -N manage-cache.sh
--- manage-cache.sh 1 Jan 1970 00:00:00 -0000
+++ manage-cache.sh-cache.sh 12 May 2004 19:22:20 -0000 1.1
@@ -0,0 +1,68 @@
+#!/bin/bash
+#
+# 2004-05-12 lars@gustaebel.de
+
+CCACHE_DIR=${CCACHE_DIR:-$HOME/.ccache}+
+echo "Do you want to compress or decompress the ccache in $CCACHE_DIR?"
+read -p "Type c or d: " mode
+
+if [ "$mode" != "c" ] && [ "$mode" != "d" ]
+then
+ exit 1
+fi
+
+is_compressed() {+ test "$(head -c 2 $1)" = $'\x1f\x8b'
+ return $?
+}
+
+tmpfile=$(mktemp)
+
+for dir in 0 1 2 3 4 5 6 7 8 9 a b c d e f
+do
+ # process ccache subdir
+ echo -n "$dir "
+
+ # find cache files
+ find $CCACHE_DIR/$dir -type f -name '*-*' |
+ sort > $tmpfile
+
+ oldsize=$(cat $CCACHE_DIR/$dir/stats | cut -d ' ' -f 13)
+ newsize=0
+
+ while read file
+ do
+ # empty files will be ignored since compressing
+ # them makes them bigger
+ test $(stat -c %s $file) -eq 0 && continue
+
+ if [ $mode = c ]
+ then
+ if ! is_compressed $file
+ then
+ gzip $file
+ mv $file.gz $file
+ fi
+ else
+ if is_compressed $file
+ then
+ mv $file $file.gz
+ gzip -d $file.gz
+ fi
+ fi
+
+ # calculate new size statistic for this subdir
+ let newsize=$newsize+$(stat -c "%B*%b" $file)/1024
+ done < $tmpfile
+
+ # update statistic file
+ read -a numbers < $CCACHE_DIR/$dir/stats
+ numbers[12]=$newsize
+ echo "${numbers[*]} " > $CCACHE_DIR/$dir/stats+done
+echo
+
+# clean up
+rm $tmpfile
+
Index: Makefile.in
===================================================================
RCS file: /home/cvsroot/lars/ccache/Makefile.in,v
retrieving revision 1.1.1.1.2.1
retrieving revision 1.12
diff -u -r1.1.1.1.2.1 -r1.12
--- Makefile.in 21 Nov 2004 17:55:36 -0000 1.1.1.1.2.1
+++ Makefile.in 21 Nov 2004 18:19:28 -0000 1.12
@@ -11,6 +11,7 @@
CFLAGS=@CFLAGS@ -I.
EXEEXT=@EXEEXT@
+LIBS= @LIBS@
OBJS= ccache.o mdfour.o hash.o execute.o util.o args.o stats.o \
cleanup.o snprintf.o unify.o
HEADERS = ccache.h mdfour.h
@@ -20,7 +21,7 @@
docs: ccache.1 web/ccache-man.html
ccache$(EXEEXT): $(OBJS) $(HEADERS)
- $(CC) $(CFLAGS) -o $@ $(OBJS)
+ $(CC) $(CFLAGS) -o $@ $(OBJS) $(LIBS)
ccache.1: ccache.yo
-yodl2man -o ccache.1 ccache.yo