Index: ccache.1 =================================================================== RCS file: /home/cvsroot/lars/ccache/ccache.1,v retrieving revision 1.1.1.1.2.1 retrieving revision 1.6 diff -u -r1.1.1.1.2.1 -r1.6 --- ccache.1 21 Nov 2004 17:55:36 -0000 1.1.1.1.2.1 +++ ccache.1 21 Nov 2004 18:19:28 -0000 1.6 @@ -210,7 +210,8 @@ CCACHE_HARDLINK then ccache will attempt to use hard links from the cache directory when creating the compiler output rather than using a file copy\&. Using hard links is faster, but can confuse programs like -\&'make\&' that rely on modification times\&. +\&'make\&' that rely on modification times\&. Hard links are never made for +compressed cache files\&. .IP .IP "\fBCCACHE_RECACHE\fP" This forces ccache to not use any cached @@ -257,6 +258,11 @@ the default\&. On HP-UX set this environment variable to "i" if you use the aCC compiler\&. .IP +.IP "\fBCCACHE_NOCOMPRESS\fP" +If you set the environment variable +CCACHE_NOCOMPRESS then there is no compression used on files that go +into the cache\&. +.IP .PP .SH "CACHE SIZE MANAGEMENT" .PP @@ -269,6 +275,14 @@ below the numbers you specified in order to avoid doing the cache clean operation too often\&. .PP +.SH "CACHE COMPRESSION" +.PP +By default ccache will compress all files it puts into the cache +using the zlib compression\&. While this involves a negligible +performance slowdown, it significantly increases the number of files +that fit in the cache\&. You can turn off compression setting the +CCACHE_NOCOMPRESS environment variable\&. +.PP .SH "HOW IT WORKS" .PP The basic idea is to detect when you are compiling exactly the same Index: ccache.c =================================================================== RCS file: /home/cvsroot/lars/ccache/ccache.c,v retrieving revision 1.1.1.1.2.1 retrieving revision 1.9 diff -u -r1.1.1.1.2.1 -r1.9 --- ccache.c 21 Nov 2004 17:55:36 -0000 1.1.1.1.2.1 +++ ccache.c 21 Nov 2004 18:19:28 -0000 1.9 @@ -199,7 +199,7 @@ fd = open(tmp_stderr, O_RDONLY | O_BINARY); if (fd != -1) { if (strcmp(output_file, "/dev/null") == 0 || - rename(tmp_hashname, output_file) == 0 || errno == ENOENT) { + move_file(tmp_hashname, output_file) == 0 || errno == ENOENT) { if (cpp_stderr) { /* we might have some stderr from cpp */ int fd2 = open(cpp_stderr, O_RDONLY | O_BINARY); @@ -231,14 +231,25 @@ x_asprintf(&path_stderr, "%s.stderr", hashname); if (stat(tmp_stderr, &st1) != 0 || - stat(tmp_hashname, &st2) != 0 || - rename(tmp_hashname, hashname) != 0 || - rename(tmp_stderr, path_stderr) != 0) { + stat(tmp_hashname, &st2) != 0 || + move_file(tmp_hashname, hashname) != 0 || + move_file(tmp_stderr, path_stderr) != 0) { cc_log("failed to rename tmp files - %s\n", strerror(errno)); stats_update(STATS_ERROR); failed(); } +#if ENABLE_ZLIB + /* do an extra stat on the cache files for + the size statistics */ + if (stat(path_stderr, &st1) != 0 || + stat(hashname, &st2) != 0) { + cc_log("failed to stat cache files - %s\n", strerror(errno)); + stats_update(STATS_ERROR); + failed(); + } +#endif + cc_log("Placed %s into cache\n", output_file); stats_tocache(file_size(&st1) + file_size(&st2)); @@ -474,7 +485,13 @@ } /* the user might be disabling cache hits */ +#ifndef ENABLE_ZLIB + /* if the cache file is compressed we must recache */ + if ((first && getenv("CCACHE_RECACHE")) || + test_if_compressed(hashname) == 1) { +#else if (first && getenv("CCACHE_RECACHE")) { +#endif close(fd_stderr); unlink(stderr_file); free(stderr_file); @@ -487,7 +504,9 @@ ret = 0; } else { unlink(output_file); - if (getenv("CCACHE_HARDLINK")) { + /* only make a hardlink if the cache file is uncompressed */ + if (getenv("CCACHE_HARDLINK") && + test_if_compressed(hashname) == 0) { ret = link(hashname, output_file); } else { ret = copy_file(hashname, output_file); Index: ccache.h =================================================================== RCS file: /home/cvsroot/lars/ccache/ccache.h,v retrieving revision 1.1.1.1.2.1 retrieving revision 1.7 diff -u -r1.1.1.1.2.1 -r1.7 --- ccache.h 21 Nov 2004 17:55:36 -0000 1.1.1.1.2.1 +++ ccache.h 21 Nov 2004 18:19:28 -0000 1.7 @@ -23,6 +23,10 @@ #include #endif +#ifdef ENABLE_ZLIB +#include +#endif + #define STATUS_NOTFOUND 3 #define STATUS_FATAL 4 #define STATUS_NOCACHE 5 @@ -36,6 +40,13 @@ #define DEFAULT_MAXSIZE (1000*1000) #endif +/* file copy mode */ +#ifdef ENABLE_ZLIB +#define COPY_UNCOMPRESSED 0 +#define COPY_FROM_CACHE 1 +#define COPY_TO_CACHE 2 +#endif + enum stats { STATS_NONE=0, STATS_STDOUT, @@ -79,6 +90,8 @@ void copy_fd(int fd_in, int fd_out); int copy_file(const char *src, const char *dest); +int move_file(const char *src, const char *dest); +int test_if_compressed(const char *filename); int create_dir(const char *dir); void x_asprintf(char **ptr, const char *format, ...); Index: ccache.yo =================================================================== RCS file: /home/cvsroot/lars/ccache/ccache.yo,v retrieving revision 1.1.1.1.2.1 retrieving revision 1.5 diff -u -r1.1.1.1.2.1 -r1.5 --- ccache.yo 21 Nov 2004 17:55:36 -0000 1.1.1.1.2.1 +++ ccache.yo 21 Nov 2004 18:19:28 -0000 1.5 @@ -169,6 +169,11 @@ this optimisation, in which case this option could allow ccache to be used. +dit(bf(CCACHE_NOCOMPRESS)) If you set the environment variable +CCACHE_NOCOMPRESS then there is no compression used on files that go +into the cache. However, this setting has no effect on how files are +retrieved from the cache, compressed results will still be usable. + dit(bf(CCACHE_NOSTATS)) If you set the environment variable CCACHE_NOSTATS then ccache will not update the statistics files on each compile. @@ -181,7 +186,8 @@ CCACHE_HARDLINK then ccache will attempt to use hard links from the cache directory when creating the compiler output rather than using a file copy. Using hard links is faster, but can confuse programs like -'make' that rely on modification times. +'make' that rely on modification times. Hard links are never made for +compressed cache files. dit(bf(CCACHE_RECACHE)) This forces ccache to not use any cached results, even if it finds them. New results are still cached, but @@ -236,6 +242,14 @@ below the numbers you specified in order to avoid doing the cache clean operation too often. +manpagesection(CACHE COMPRESSION) + +By default ccache will compress all files it puts into the cache +using the zlib compression. While this involves a negligible +performance slowdown, it significantly increases the number of files +that fit in the cache. You can turn off compression setting the +CCACHE_NOCOMPRESS environment variable. + manpagesection(HOW IT WORKS) The basic idea is to detect when you are compiling exactly the same @@ -294,6 +308,8 @@ cache. This tells the filesystem to inherit group ownership for new directories. The command "chmod g+s `find $CCACHE_DIR -type d`" might be useful for this. + it() Set bf(CCACHE_NOCOMPRESS) for all users, if there are users with + versions of ccache that do not support compression. ) manpagesection(HISTORY) Index: config.h.in =================================================================== RCS file: /home/cvsroot/lars/ccache/config.h.in,v retrieving revision 1.1.1.1 retrieving revision 1.2 diff -u -r1.1.1.1 -r1.2 --- config.h.in 30 Apr 2004 13:13:41 -0000 1.1.1.1 +++ config.h.in 4 May 2004 20:49:26 -0000 1.2 @@ -98,3 +98,6 @@ /* Define _GNU_SOURCE so that we get all necessary prototypes */ #undef _GNU_SOURCE + +/* Define to 1 if you like to have zlib compression for the ccache. */ +#undef ENABLE_ZLIB Index: configure.in =================================================================== RCS file: /home/cvsroot/lars/ccache/configure.in,v retrieving revision 1.1.1.1.2.1 retrieving revision 1.4 diff -u -r1.1.1.1.2.1 -r1.4 --- configure.in 21 Nov 2004 17:55:36 -0000 1.1.1.1.2.1 +++ configure.in 21 Nov 2004 18:19:28 -0000 1.4 @@ -68,5 +68,15 @@ AC_DEFINE(HAVE_C99_VSNPRINTF, 1, [ ]) fi +dnl Check for zlib. +AC_ARG_ENABLE([zlib], + AS_HELP_STRING([--enable-zlib], [enable zlib support for ccache compression]),, + [enable_zlib=yes]) + +if test x"$enable_zlib" = x"yes"; then + AC_CHECK_HEADER(zlib.h, AC_CHECK_LIB(z, gzdopen, LIBS="-lz $LIBS"; + AC_DEFINE([ENABLE_ZLIB],[1],[Define ENABLE_ZLIB if you want to enable zlib]))) +fi + AC_CONFIG_FILES([Makefile]) AC_OUTPUT Index: util.c =================================================================== RCS file: /home/cvsroot/lars/ccache/util.c,v retrieving revision 1.1.1.1.2.1 retrieving revision 1.11 diff -u -r1.1.1.1.2.1 -r1.11 --- util.c 21 Nov 2004 17:55:36 -0000 1.1.1.1.2.1 +++ util.c 21 Nov 2004 18:19:28 -0000 1.11 @@ -44,6 +44,7 @@ exit(1); } +#ifndef ENABLE_ZLIB /* copy all data from one file descriptor to another */ void copy_fd(int fd_in, int fd_out) { @@ -57,6 +58,11 @@ } } +/* move a file using rename */ +int move_file(const char *src, const char *dest) { + return rename(src, dest); +} + /* copy a file - used when hard links don't work the copy is done via a temporary file and atomic rename */ @@ -120,6 +126,174 @@ return 0; } +#else /* ENABLE_ZLIB */ + +/* copy all data from one file descriptor to another + possibly decompressing it +*/ +void copy_fd(int fd_in, int fd_out) { + char buf[10240]; + int n; + gzFile gz_in; + + gz_in = gzdopen(dup(fd_in), "rb"); + + if (!gz_in) { + fatal("Failed to copy fd"); + } + + while ((n = gzread(gz_in, buf, sizeof(buf))) > 0) { + if (write(fd_out, buf, n) != n) { + fatal("Failed to copy fd"); + } + } +} + +static int _copy_file(const char *src, const char *dest, int mode) { + int fd_in, fd_out; + gzFile gz_in, gz_out = NULL; + char buf[10240]; + int n, ret; + char *tmp_name; + mode_t mask; + struct stat st; + + x_asprintf(&tmp_name, "%s.XXXXXX", dest); + + if (getenv("CCACHE_NOCOMPRESS")) { + mode = COPY_UNCOMPRESSED; + } + + /* open source file */ + fd_in = open(src, O_RDONLY); + if (fd_in == -1) { + return -1; + } + + gz_in = gzdopen(fd_in, "rb"); + if (!gz_in) { + close(fd_in); + return -1; + } + + /* open destination file */ + fd_out = mkstemp(tmp_name); + if (fd_out == -1) { + gzclose(gz_in); + free(tmp_name); + return -1; + } + + if (mode == COPY_TO_CACHE) { + /* The gzip file format occupies at least 20 bytes. So + it will always occupy an entire filesystem block, + even for empty files. + Since most stderr files will be empty, we turn off + compression in this case to save space. + */ + if (fstat(fd_in, &st) != 0) { + gzclose(gz_in); + close(fd_out); + free(tmp_name); + return -1; + } + if (file_size(&st) == 0) { + mode = COPY_UNCOMPRESSED; + } + } + + if (mode == COPY_TO_CACHE) { + gz_out = gzdopen(dup(fd_out), "wb"); + if (!gz_out) { + gzclose(gz_in); + close(fd_out); + free(tmp_name); + return -1; + } + } + + while ((n = gzread(gz_in, buf, sizeof(buf))) > 0) { + if (mode == COPY_TO_CACHE) { + ret = gzwrite(gz_out, buf, n); + } else { + ret = write(fd_out, buf, n); + } + if (ret != n) { + gzclose(gz_in); + if (gz_out) { + gzclose(gz_out); + } + close(fd_out); + unlink(tmp_name); + free(tmp_name); + return -1; + } + } + + gzclose(gz_in); + if (gz_out) { + gzclose(gz_out); + } + + /* get perms right on the tmp file */ + mask = umask(0); + fchmod(fd_out, 0666 & ~mask); + umask(mask); + + /* the close can fail on NFS if out of space */ + if (close(fd_out) == -1) { + unlink(tmp_name); + free(tmp_name); + return -1; + } + + unlink(dest); + + if (rename(tmp_name, dest) == -1) { + unlink(tmp_name); + free(tmp_name); + return -1; + } + + free(tmp_name); + + return 0; +} + +/* move a file to the cache, compressing it */ +int move_file(const char *src, const char *dest) { + int ret; + + ret = _copy_file(src, dest, COPY_TO_CACHE); + if (ret != -1) unlink(src); + return ret; +} + +/* copy a file from the cache, decompressing it */ +int copy_file(const char *src, const char *dest) { + return _copy_file(src, dest, COPY_FROM_CACHE); +} +#endif /* ENABLE_ZLIB */ + +/* test if a file is zlib compressed */ +int test_if_compressed(const char *filename) { + FILE *f; + + f = fopen(filename, "rb"); + if (!f) { + return 0; + } + + /* test if file starts with 1F8B, which is zlib's + * magic number */ + if ((fgetc(f) != 0x1f) || (fgetc(f) != 0x8b)) { + fclose(f); + return 0; + } + + fclose(f); + return 1; +} /* make sure a directory exists */ int create_dir(const char *dir) Index: manage-cache.sh =================================================================== RCS file: manage-cache.sh diff -N manage-cache.sh --- manage-cache.sh 1 Jan 1970 00:00:00 -0000 +++ manage-cache.sh-cache.sh 12 May 2004 19:22:20 -0000 1.1 @@ -0,0 +1,68 @@ +#!/bin/bash +# +# 2004-05-12 lars@gustaebel.de + +CCACHE_DIR=${CCACHE_DIR:-$HOME/.ccache} + +echo "Do you want to compress or decompress the ccache in $CCACHE_DIR?" +read -p "Type c or d: " mode + +if [ "$mode" != "c" ] && [ "$mode" != "d" ] +then + exit 1 +fi + +is_compressed() { + test "$(head -c 2 $1)" = $'\x1f\x8b' + return $? +} + +tmpfile=$(mktemp) + +for dir in 0 1 2 3 4 5 6 7 8 9 a b c d e f +do + # process ccache subdir + echo -n "$dir " + + # find cache files + find $CCACHE_DIR/$dir -type f -name '*-*' | + sort > $tmpfile + + oldsize=$(cat $CCACHE_DIR/$dir/stats | cut -d ' ' -f 13) + newsize=0 + + while read file + do + # empty files will be ignored since compressing + # them makes them bigger + test $(stat -c %s $file) -eq 0 && continue + + if [ $mode = c ] + then + if ! is_compressed $file + then + gzip $file + mv $file.gz $file + fi + else + if is_compressed $file + then + mv $file $file.gz + gzip -d $file.gz + fi + fi + + # calculate new size statistic for this subdir + let newsize=$newsize+$(stat -c "%B*%b" $file)/1024 + done < $tmpfile + + # update statistic file + read -a numbers < $CCACHE_DIR/$dir/stats + numbers[12]=$newsize + echo "${numbers[*]} " > $CCACHE_DIR/$dir/stats +done +echo + +# clean up +rm $tmpfile + Index: Makefile.in =================================================================== RCS file: /home/cvsroot/lars/ccache/Makefile.in,v retrieving revision 1.1.1.1.2.1 retrieving revision 1.12 diff -u -r1.1.1.1.2.1 -r1.12 --- Makefile.in 21 Nov 2004 17:55:36 -0000 1.1.1.1.2.1 +++ Makefile.in 21 Nov 2004 18:19:28 -0000 1.12 @@ -11,6 +11,7 @@ CFLAGS=@CFLAGS@ -I. EXEEXT=@EXEEXT@ +LIBS= @LIBS@ OBJS= ccache.o mdfour.o hash.o execute.o util.o args.o stats.o \ cleanup.o snprintf.o unify.o HEADERS = ccache.h mdfour.h @@ -20,7 +21,7 @@ docs: ccache.1 web/ccache-man.html ccache$(EXEEXT): $(OBJS) $(HEADERS) - $(CC) $(CFLAGS) -o $@ $(OBJS) + $(CC) $(CFLAGS) -o $@ $(OBJS) $(LIBS) ccache.1: ccache.yo -yodl2man -o ccache.1 ccache.yo