1 /* gzappend -- command to append to a gzip file

     2 

     3   Copyright (C) 2003 Mark Adler, all rights reserved

     4   version 1.1, 4 Nov 2003

     5 

     6   This software is provided 'as-is', without any express or implied

     7   warranty.  In no event will the author be held liable for any damages

     8   arising from the use of this software.

     9 

    10   Permission is granted to anyone to use this software for any purpose,

    11   including commercial applications, and to alter it and redistribute it

    12   freely, subject to the following restrictions:

    13 

    14   1. The origin of this software must not be misrepresented; you must not

    15      claim that you wrote the original software. If you use this software

    16      in a product, an acknowledgment in the product documentation would be

    17      appreciated but is not required.

    18   2. Altered source versions must be plainly marked as such, and must not be

    19      misrepresented as being the original software.

    20   3. This notice may not be removed or altered from any source distribution.

    21 

    22   Mark Adler    madler@alumni.caltech.edu

    23  */

    24 

    25 /*

    26  * Change history:

    27  *

    28  * 1.0  19 Oct 2003     - First version

    29  * 1.1   4 Nov 2003     - Expand and clarify some comments and notes

    30  *                      - Add version and copyright to help

    31  *                      - Send help to stdout instead of stderr

    32  *                      - Add some preemptive typecasts

    33  *                      - Add L to constants in lseek() calls

    34  *                      - Remove some debugging information in error messages

    35  *                      - Use new data_type definition for zlib 1.2.1

    36  *                      - Simplfy and unify file operations

    37  *                      - Finish off gzip file in gztack()

    38  *                      - Use deflatePrime() instead of adding empty blocks

    39  *                      - Keep gzip file clean on appended file read errors

    40  *                      - Use in-place rotate instead of auxiliary buffer

    41  *                        (Why you ask?  Because it was fun to write!)

    42  */

    43 

    44 /*

    45    gzappend takes a gzip file and appends to it, compressing files from the

    46    command line or data from stdin.  The gzip file is written to directly, to

    47    avoid copying that file, in case it's large.  Note that this results in the

    48    unfriendly behavior that if gzappend fails, the gzip file is corrupted.

    49 

    50    This program was written to illustrate the use of the new Z_BLOCK option of

    51    zlib 1.2.x's inflate() function.  This option returns from inflate() at each

    52    block boundary to facilitate locating and modifying the last block bit at

    53    the start of the final deflate block.  Also whether using Z_BLOCK or not,

    54    another required feature of zlib 1.2.x is that inflate() now provides the

    55    number of unusued bits in the last input byte used.  gzappend will not work

    56    with versions of zlib earlier than 1.2.1.

    57 

    58    gzappend first decompresses the gzip file internally, discarding all but

    59    the last 32K of uncompressed data, and noting the location of the last block

    60    bit and the number of unused bits in the last byte of the compressed data.

    61    The gzip trailer containing the CRC-32 and length of the uncompressed data

    62    is verified.  This trailer will be later overwritten.

    63 

    64    Then the last block bit is cleared by seeking back in the file and rewriting

    65    the byte that contains it.  Seeking forward, the last byte of the compressed

    66    data is saved along with the number of unused bits to initialize deflate.

    67 

    68    A deflate process is initialized, using the last 32K of the uncompressed

    69    data from the gzip file to initialize the dictionary.  If the total

    70    uncompressed data was less than 32K, then all of it is used to initialize

    71    the dictionary.  The deflate output bit buffer is also initialized with the

    72    last bits from the original deflate stream.  From here on, the data to

    73    append is simply compressed using deflate, and written to the gzip file.

    74    When that is complete, the new CRC-32 and uncompressed length are written

    75    as the trailer of the gzip file.

    76  */

    77 

    78 #include <stdio.h>

    79 #include <stdlib.h>

    80 #include <string.h>

    81 #include <fcntl.h>

    82 #include <unistd.h>

    83 #include "zlib.h"

    84 

    85 #define local static

    86 #define LGCHUNK 14

    87 #define CHUNK (1U << LGCHUNK)

    88 #define DSIZE 32768U

    89 

    90 /* print an error message and terminate with extreme prejudice */

    91 local void bye(char *msg1, char *msg2)

    92 {

    93     fprintf(stderr, "gzappend error: %s%s\n", msg1, msg2);

    94     exit(1);

    95 }

    96 

    97 /* return the greatest common divisor of a and b using Euclid's algorithm,

    98    modified to be fast when one argument much greater than the other, and

    99    coded to avoid unnecessary swapping */

   100 local unsigned gcd(unsigned a, unsigned b)

   101 {

   102     unsigned c;

   103 

   104     while (a && b)

   105         if (a > b) {

   106             c = b;

   107             while (a - c >= c)

   108                 c <<= 1;

   109             a -= c;

   110         }

   111         else {

   112             c = a;

   113             while (b - c >= c)

   114                 c <<= 1;

   115             b -= c;

   116         }

   117     return a + b;

   118 }

   119 

   120 /* rotate list[0..len-1] left by rot positions, in place */

   121 local void rotate(unsigned char *list, unsigned len, unsigned rot)

   122 {

   123     unsigned char tmp;

   124     unsigned cycles;

   125     unsigned char *start, *last, *to, *from;

   126 

   127     /* normalize rot and handle degenerate cases */

   128     if (len < 2) return;

   129     if (rot >= len) rot %= len;

   130     if (rot == 0) return;

   131 

   132     /* pointer to last entry in list */

   133     last = list + (len - 1);

   134 

   135     /* do simple left shift by one */

   136     if (rot == 1) {

   137         tmp = *list;

   138         memcpy(list, list + 1, len - 1);

   139         *last = tmp;

   140         return;

   141     }

   142 

   143     /* do simple right shift by one */

   144     if (rot == len - 1) {

   145         tmp = *last;

   146         memmove(list + 1, list, len - 1);

   147         *list = tmp;

   148         return;

   149     }

   150 

   151     /* otherwise do rotate as a set of cycles in place */

   152     cycles = gcd(len, rot);             /* number of cycles */

   153     do {

   154         start = from = list + cycles;   /* start index is arbitrary */

   155         tmp = *from;                    /* save entry to be overwritten */

   156         for (;;) {

   157             to = from;                  /* next step in cycle */

   158             from += rot;                /* go right rot positions */

   159             if (from > last) from -= len;   /* (pointer better not wrap) */

   160             if (from == start) break;   /* all but one shifted */

   161             *to = *from;                /* shift left */

   162         }

   163         *to = tmp;                      /* complete the circle */

   164     } while (--cycles);

   165 }

   166 

   167 /* structure for gzip file read operations */

   168 typedef struct {

   169     int fd;                     /* file descriptor */

   170     int size;                   /* 1 << size is bytes in buf */

   171     unsigned left;              /* bytes available at next */

   172     unsigned char *buf;         /* buffer */

   173     unsigned char *next;        /* next byte in buffer */

   174     char *name;                 /* file name for error messages */

   175 } file;

   176 

   177 /* reload buffer */

   178 local int readin(file *in)

   179 {

   180     int len;

   181 

   182     len = read(in->fd, in->buf, 1 << in->size);

   183     if (len == -1) bye("error reading ", in->name);

   184     in->left = (unsigned)len;

   185     in->next = in->buf;

   186     return len;

   187 }

   188 

   189 /* read from file in, exit if end-of-file */

   190 local int readmore(file *in)

   191 {

   192     if (readin(in) == 0) bye("unexpected end of ", in->name);

   193     return 0;

   194 }

   195 

   196 #define read1(in) (in->left == 0 ? readmore(in) : 0, \

   197                    in->left--, *(in->next)++)

   198 

   199 /* skip over n bytes of in */

   200 local void skip(file *in, unsigned n)

   201 {

   202     unsigned bypass;

   203 

   204     if (n > in->left) {

   205         n -= in->left;

   206         bypass = n & ~((1U << in->size) - 1);

   207         if (bypass) {

   208             if (lseek(in->fd, (off_t)bypass, SEEK_CUR) == -1)

   209                 bye("seeking ", in->name);

   210             n -= bypass;

   211         }

   212         readmore(in);

   213         if (n > in->left)

   214             bye("unexpected end of ", in->name);

   215     }

   216     in->left -= n;

   217     in->next += n;

   218 }

   219 

   220 /* read a four-byte unsigned integer, little-endian, from in */

   221 unsigned long read4(file *in)

   222 {

   223     unsigned long val;

   224 

   225     val = read1(in);

   226     val += (unsigned)read1(in) << 8;

   227     val += (unsigned long)read1(in) << 16;

   228     val += (unsigned long)read1(in) << 24;

   229     return val;

   230 }

   231 

   232 /* skip over gzip header */

   233 local void gzheader(file *in)

   234 {

   235     int flags;

   236     unsigned n;

   237 

   238     if (read1(in) != 31 || read1(in) != 139) bye(in->name, " not a gzip file");

   239     if (read1(in) != 8) bye("unknown compression method in", in->name);

   240     flags = read1(in);

   241     if (flags & 0xe0) bye("unknown header flags set in", in->name);

   242     skip(in, 6);

   243     if (flags & 4) {

   244         n = read1(in);

   245         n += (unsigned)(read1(in)) << 8;

   246         skip(in, n);

   247     }

   248     if (flags & 8) while (read1(in) != 0) ;

   249     if (flags & 16) while (read1(in) != 0) ;

   250     if (flags & 2) skip(in, 2);

   251 }

   252 

   253 /* decompress gzip file "name", return strm with a deflate stream ready to

   254    continue compression of the data in the gzip file, and return a file

   255    descriptor pointing to where to write the compressed data -- the deflate

   256    stream is initialized to compress using level "level" */

   257 local int gzscan(char *name, z_stream *strm, int level)

   258 {

   259     int ret, lastbit, left, full;

   260     unsigned have;

   261     unsigned long crc, tot;

   262     unsigned char *window;

   263     off_t lastoff, end;

   264     file gz;

   265 

   266     /* open gzip file */

   267     gz.name = name;

   268     gz.fd = open(name, O_RDWR, 0);

   269     if (gz.fd == -1) bye("cannot open ", name);

   270     gz.buf = malloc(CHUNK);

   271     if (gz.buf == NULL) bye("out of memory", "");

   272     gz.size = LGCHUNK;

   273     gz.left = 0;

   274 

   275     /* skip gzip header */

   276     gzheader(&gz);

   277 

   278     /* prepare to decompress */

   279     window = malloc(DSIZE);

   280     if (window == NULL) bye("out of memory", "");

   281     strm->zalloc = Z_NULL;

   282     strm->zfree = Z_NULL;

   283     strm->opaque = Z_NULL;

   284     ret = inflateInit2(strm, -15);

   285     if (ret != Z_OK) bye("out of memory", " or library mismatch");

   286 

   287     /* decompress the deflate stream, saving append information */

   288     lastbit = 0;

   289     lastoff = lseek(gz.fd, 0L, SEEK_CUR) - gz.left;

   290     left = 0;

   291     strm->avail_in = gz.left;

   292     strm->next_in = gz.next;

   293     crc = crc32(0L, Z_NULL, 0);

   294     have = full = 0;

   295     do {

   296         /* if needed, get more input */

   297         if (strm->avail_in == 0) {

   298             readmore(&gz);

   299             strm->avail_in = gz.left;

   300             strm->next_in = gz.next;

   301         }

   302 

   303         /* set up output to next available section of sliding window */

   304         strm->avail_out = DSIZE - have;

   305         strm->next_out = window + have;

   306 

   307         /* inflate and check for errors */

   308         ret = inflate(strm, Z_BLOCK);

   309         if (ret == Z_STREAM_ERROR) bye("internal stream error!", "");

   310         if (ret == Z_MEM_ERROR) bye("out of memory", "");

   311         if (ret == Z_DATA_ERROR)

   312             bye("invalid compressed data--format violated in", name);

   313 

   314         /* update crc and sliding window pointer */

   315         crc = crc32(crc, window + have, DSIZE - have - strm->avail_out);

   316         if (strm->avail_out)

   317             have = DSIZE - strm->avail_out;

   318         else {

   319             have = 0;

   320             full = 1;

   321         }

   322 

   323         /* process end of block */

   324         if (strm->data_type & 128) {

   325             if (strm->data_type & 64)

   326                 left = strm->data_type & 0x1f;

   327             else {

   328                 lastbit = strm->data_type & 0x1f;

   329                 lastoff = lseek(gz.fd, 0L, SEEK_CUR) - strm->avail_in;

   330             }

   331         }

   332     } while (ret != Z_STREAM_END);

   333     inflateEnd(strm);

   334     gz.left = strm->avail_in;

   335     gz.next = strm->next_in;

   336 

   337     /* save the location of the end of the compressed data */

   338     end = lseek(gz.fd, 0L, SEEK_CUR) - gz.left;

   339 

   340     /* check gzip trailer and save total for deflate */

   341     if (crc != read4(&gz))

   342         bye("invalid compressed data--crc mismatch in ", name);

   343     tot = strm->total_out;

   344     if ((tot & 0xffffffffUL) != read4(&gz))

   345         bye("invalid compressed data--length mismatch in", name);

   346 

   347     /* if not at end of file, warn */

   348     if (gz.left || readin(&gz))

   349         fprintf(stderr,

   350             "gzappend warning: junk at end of gzip file overwritten\n");

   351 

   352     /* clear last block bit */

   353     lseek(gz.fd, lastoff - (lastbit != 0), SEEK_SET);

   354     if (read(gz.fd, gz.buf, 1) != 1) bye("reading after seek on ", name);

   355     *gz.buf = (unsigned char)(*gz.buf ^ (1 << ((8 - lastbit) & 7)));

   356     lseek(gz.fd, -1L, SEEK_CUR);

   357     if (write(gz.fd, gz.buf, 1) != 1) bye("writing after seek to ", name);

   358 

   359     /* if window wrapped, build dictionary from window by rotating */

   360     if (full) {

   361         rotate(window, DSIZE, have);

   362         have = DSIZE;

   363     }

   364 

   365     /* set up deflate stream with window, crc, total_in, and leftover bits */

   366     ret = deflateInit2(strm, level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY);

   367     if (ret != Z_OK) bye("out of memory", "");

   368     deflateSetDictionary(strm, window, have);

   369     strm->adler = crc;

   370     strm->total_in = tot;

   371     if (left) {

   372         lseek(gz.fd, --end, SEEK_SET);

   373         if (read(gz.fd, gz.buf, 1) != 1) bye("reading after seek on ", name);

   374         deflatePrime(strm, 8 - left, *gz.buf);

   375     }

   376     lseek(gz.fd, end, SEEK_SET);

   377 

   378     /* clean up and return */

   379     free(window);

   380     free(gz.buf);

   381     return gz.fd;

   382 }

   383 

   384 /* append file "name" to gzip file gd using deflate stream strm -- if last

   385    is true, then finish off the deflate stream at the end */

   386 local void gztack(char *name, int gd, z_stream *strm, int last)

   387 {

   388     int fd, len, ret;

   389     unsigned left;

   390     unsigned char *in, *out;

   391 

   392     /* open file to compress and append */

   393     fd = 0;

   394     if (name != NULL) {

   395         fd = open(name, O_RDONLY, 0);

   396         if (fd == -1)

   397             fprintf(stderr, "gzappend warning: %s not found, skipping ...\n",

   398                     name);

   399     }

   400 

   401     /* allocate buffers */

   402     in = fd == -1 ? NULL : malloc(CHUNK);

   403     out = malloc(CHUNK);

   404     if (out == NULL) bye("out of memory", "");

   405 

   406     /* compress input file and append to gzip file */

   407     do {

   408         /* get more input */

   409         len = fd == -1 ? 0 : read(fd, in, CHUNK);

   410         if (len == -1) {

   411             fprintf(stderr,

   412                     "gzappend warning: error reading %s, skipping rest ...\n",

   413                     name);

   414             len = 0;

   415         }

   416         strm->avail_in = (unsigned)len;

   417         strm->next_in = in;

   418         if (len) strm->adler = crc32(strm->adler, in, (unsigned)len);

   419 

   420         /* compress and write all available output */

   421         do {

   422             strm->avail_out = CHUNK;

   423             strm->next_out = out;

   424             ret = deflate(strm, last && len == 0 ? Z_FINISH : Z_NO_FLUSH);

   425             left = CHUNK - strm->avail_out;

   426             while (left) {

   427                 len = write(gd, out + CHUNK - strm->avail_out - left, left);

   428                 if (len == -1) bye("writing gzip file", "");

   429                 left -= (unsigned)len;

   430             }

   431         } while (strm->avail_out == 0 && ret != Z_STREAM_END);

   432     } while (len != 0);

   433 

   434     /* write trailer after last entry */

   435     if (last) {

   436         deflateEnd(strm);

   437         out[0] = (unsigned char)(strm->adler);

   438         out[1] = (unsigned char)(strm->adler >> 8);

   439         out[2] = (unsigned char)(strm->adler >> 16);

   440         out[3] = (unsigned char)(strm->adler >> 24);

   441         out[4] = (unsigned char)(strm->total_in);

   442         out[5] = (unsigned char)(strm->total_in >> 8);

   443         out[6] = (unsigned char)(strm->total_in >> 16);

   444         out[7] = (unsigned char)(strm->total_in >> 24);

   445         len = 8;

   446         do {

   447             ret = write(gd, out + 8 - len, len);

   448             if (ret == -1) bye("writing gzip file", "");

   449             len -= ret;

   450         } while (len);

   451         close(gd);

   452     }

   453 

   454     /* clean up and return */

   455     free(out);

   456     if (in != NULL) free(in);

   457     if (fd > 0) close(fd);

   458 }

   459 

   460 /* process the compression level option if present, scan the gzip file, and

   461    append the specified files, or append the data from stdin if no other file

   462    names are provided on the command line -- the gzip file must be writable

   463    and seekable */

   464 int main(int argc, char **argv)

   465 {

   466     int gd, level;

   467     z_stream strm;

   468 

   469     /* ignore command name */

   470     argv++;

   471 

   472     /* provide usage if no arguments */

   473     if (*argv == NULL) {

   474         printf("gzappend 1.1 (4 Nov 2003) Copyright (C) 2003 Mark Adler\n");

   475         printf(

   476             "usage: gzappend [-level] file.gz [ addthis [ andthis ... ]]\n");

   477         return 0;

   478     }

   479 

   480     /* set compression level */

   481     level = Z_DEFAULT_COMPRESSION;

   482     if (argv[0][0] == '-') {

   483         if (argv[0][1] < '0' || argv[0][1] > '9' || argv[0][2] != 0)

   484             bye("invalid compression level", "");

   485         level = argv[0][1] - '0';

   486         if (*++argv == NULL) bye("no gzip file name after options", "");

   487     }

   488 

   489     /* prepare to append to gzip file */

   490     gd = gzscan(*argv++, &strm, level);

   491 

   492     /* append files on command line, or from stdin if none */

   493     if (*argv == NULL)

   494         gztack(NULL, gd, &strm, 1);

   495     else

   496         do {

   497             gztack(*argv, gd, &strm, argv[1] == NULL);

   498         } while (*++argv != NULL);

   499     return 0;

   500 }