1 | /* |
---|
2 | * gzlog.c |
---|
3 | * Copyright (C) 2004 Mark Adler |
---|
4 | * For conditions of distribution and use, see copyright notice in gzlog.h |
---|
5 | * version 1.0, 26 Nov 2004 |
---|
6 | * |
---|
7 | */ |
---|
8 | |
---|
9 | #include <string.h> /* memcmp() */ |
---|
10 | #include <stdlib.h> /* malloc(), free(), NULL */ |
---|
11 | #include <sys/types.h> /* size_t, off_t */ |
---|
12 | #include <unistd.h> /* read(), close(), sleep(), ftruncate(), */ |
---|
13 | /* lseek() */ |
---|
14 | #include <fcntl.h> /* open() */ |
---|
15 | #include <sys/file.h> /* flock() */ |
---|
16 | #include "zlib.h" /* deflateInit2(), deflate(), deflateEnd() */ |
---|
17 | |
---|
18 | #include "gzlog.h" /* interface */ |
---|
19 | #define local static |
---|
20 | |
---|
21 | /* log object structure */ |
---|
22 | typedef struct { |
---|
23 | int id; /* object identifier */ |
---|
24 | int fd; /* log file descriptor */ |
---|
25 | off_t extra; /* offset of extra "ap" subfield */ |
---|
26 | off_t mark_off; /* offset of marked data */ |
---|
27 | off_t last_off; /* offset of last block */ |
---|
28 | unsigned long crc; /* uncompressed crc */ |
---|
29 | unsigned long len; /* uncompressed length (modulo 2^32) */ |
---|
30 | unsigned stored; /* length of current stored block */ |
---|
31 | } gz_log; |
---|
32 | |
---|
33 | #define GZLOGID 19334 /* gz_log object identifier */ |
---|
34 | |
---|
35 | #define LOCK_RETRY 1 /* retry lock once a second */ |
---|
36 | #define LOCK_PATIENCE 1200 /* try about twenty minutes before forcing */ |
---|
37 | |
---|
38 | /* acquire a lock on a file */ |
---|
39 | local int lock(int fd) |
---|
40 | { |
---|
41 | int patience; |
---|
42 | |
---|
43 | /* try to lock every LOCK_RETRY seconds for LOCK_PATIENCE seconds */ |
---|
44 | patience = LOCK_PATIENCE; |
---|
45 | do { |
---|
46 | if (flock(fd, LOCK_EX + LOCK_NB) == 0) |
---|
47 | return 0; |
---|
48 | (void)sleep(LOCK_RETRY); |
---|
49 | patience -= LOCK_RETRY; |
---|
50 | } while (patience > 0); |
---|
51 | |
---|
52 | /* we've run out of patience -- give up */ |
---|
53 | return -1; |
---|
54 | } |
---|
55 | |
---|
56 | /* release lock */ |
---|
57 | local void unlock(int fd) |
---|
58 | { |
---|
59 | (void)flock(fd, LOCK_UN); |
---|
60 | } |
---|
61 | |
---|
62 | /* release a log object */ |
---|
63 | local void log_clean(gz_log *log) |
---|
64 | { |
---|
65 | unlock(log->fd); |
---|
66 | (void)close(log->fd); |
---|
67 | free(log); |
---|
68 | } |
---|
69 | |
---|
70 | /* read an unsigned long from a byte buffer little-endian */ |
---|
71 | local unsigned long make_ulg(unsigned char *buf) |
---|
72 | { |
---|
73 | int n; |
---|
74 | unsigned long val; |
---|
75 | |
---|
76 | val = (unsigned long)(*buf++); |
---|
77 | for (n = 8; n < 32; n += 8) |
---|
78 | val += (unsigned long)(*buf++) << n; |
---|
79 | return val; |
---|
80 | } |
---|
81 | |
---|
82 | /* read an off_t from a byte buffer little-endian */ |
---|
83 | local off_t make_off(unsigned char *buf) |
---|
84 | { |
---|
85 | int n; |
---|
86 | off_t val; |
---|
87 | |
---|
88 | val = (off_t)(*buf++); |
---|
89 | for (n = 8; n < 64; n += 8) |
---|
90 | val += (off_t)(*buf++) << n; |
---|
91 | return val; |
---|
92 | } |
---|
93 | |
---|
94 | /* write an unsigned long little-endian to byte buffer */ |
---|
95 | local void dice_ulg(unsigned long val, unsigned char *buf) |
---|
96 | { |
---|
97 | int n; |
---|
98 | |
---|
99 | for (n = 0; n < 4; n++) { |
---|
100 | *buf++ = val & 0xff; |
---|
101 | val >>= 8; |
---|
102 | } |
---|
103 | } |
---|
104 | |
---|
105 | /* write an off_t little-endian to byte buffer */ |
---|
106 | local void dice_off(off_t val, unsigned char *buf) |
---|
107 | { |
---|
108 | int n; |
---|
109 | |
---|
110 | for (n = 0; n < 8; n++) { |
---|
111 | *buf++ = val & 0xff; |
---|
112 | val >>= 8; |
---|
113 | } |
---|
114 | } |
---|
115 | |
---|
116 | /* initial, empty gzip file for appending */ |
---|
117 | local char empty_gz[] = { |
---|
118 | 0x1f, 0x8b, /* magic gzip id */ |
---|
119 | 8, /* compression method is deflate */ |
---|
120 | 4, /* there is an extra field */ |
---|
121 | 0, 0, 0, 0, /* no modification time provided */ |
---|
122 | 0, 0xff, /* no extra flags, no OS */ |
---|
123 | 20, 0, 'a', 'p', 16, 0, /* extra field with "ap" subfield */ |
---|
124 | 32, 0, 0, 0, 0, 0, 0, 0, /* offset of uncompressed data */ |
---|
125 | 32, 0, 0, 0, 0, 0, 0, 0, /* offset of last block */ |
---|
126 | 1, 0, 0, 0xff, 0xff, /* empty stored block (last) */ |
---|
127 | 0, 0, 0, 0, /* crc */ |
---|
128 | 0, 0, 0, 0 /* uncompressed length */ |
---|
129 | }; |
---|
130 | |
---|
131 | /* initialize a log object with locking */ |
---|
132 | void *gzlog_open(char *path) |
---|
133 | { |
---|
134 | unsigned xlen; |
---|
135 | unsigned char temp[20]; |
---|
136 | unsigned sub_len; |
---|
137 | int good; |
---|
138 | gz_log *log; |
---|
139 | |
---|
140 | /* allocate log structure */ |
---|
141 | log = malloc(sizeof(gz_log)); |
---|
142 | if (log == NULL) |
---|
143 | return NULL; |
---|
144 | log->id = GZLOGID; |
---|
145 | |
---|
146 | /* open file, creating it if necessary, and locking it */ |
---|
147 | log->fd = open(path, O_RDWR | O_CREAT, 0600); |
---|
148 | if (log->fd < 0) { |
---|
149 | free(log); |
---|
150 | return NULL; |
---|
151 | } |
---|
152 | if (lock(log->fd)) { |
---|
153 | close(log->fd); |
---|
154 | free(log); |
---|
155 | return NULL; |
---|
156 | } |
---|
157 | |
---|
158 | /* if file is empty, write new gzip stream */ |
---|
159 | if (lseek(log->fd, 0, SEEK_END) == 0) { |
---|
160 | if (write(log->fd, empty_gz, sizeof(empty_gz)) != sizeof(empty_gz)) { |
---|
161 | log_clean(log); |
---|
162 | return NULL; |
---|
163 | } |
---|
164 | } |
---|
165 | |
---|
166 | /* check gzip header */ |
---|
167 | (void)lseek(log->fd, 0, SEEK_SET); |
---|
168 | if (read(log->fd, temp, 12) != 12 || temp[0] != 0x1f || |
---|
169 | temp[1] != 0x8b || temp[2] != 8 || (temp[3] & 4) == 0) { |
---|
170 | log_clean(log); |
---|
171 | return NULL; |
---|
172 | } |
---|
173 | |
---|
174 | /* process extra field to find "ap" sub-field */ |
---|
175 | xlen = temp[10] + (temp[11] << 8); |
---|
176 | good = 0; |
---|
177 | while (xlen) { |
---|
178 | if (xlen < 4 || read(log->fd, temp, 4) != 4) |
---|
179 | break; |
---|
180 | sub_len = temp[2]; |
---|
181 | sub_len += temp[3] << 8; |
---|
182 | xlen -= 4; |
---|
183 | if (memcmp(temp, "ap", 2) == 0 && sub_len == 16) { |
---|
184 | good = 1; |
---|
185 | break; |
---|
186 | } |
---|
187 | if (xlen < sub_len) |
---|
188 | break; |
---|
189 | (void)lseek(log->fd, sub_len, SEEK_CUR); |
---|
190 | xlen -= sub_len; |
---|
191 | } |
---|
192 | if (!good) { |
---|
193 | log_clean(log); |
---|
194 | return NULL; |
---|
195 | } |
---|
196 | |
---|
197 | /* read in "ap" sub-field */ |
---|
198 | log->extra = lseek(log->fd, 0, SEEK_CUR); |
---|
199 | if (read(log->fd, temp, 16) != 16) { |
---|
200 | log_clean(log); |
---|
201 | return NULL; |
---|
202 | } |
---|
203 | log->mark_off = make_off(temp); |
---|
204 | log->last_off = make_off(temp + 8); |
---|
205 | |
---|
206 | /* get crc, length of gzip file */ |
---|
207 | (void)lseek(log->fd, log->last_off, SEEK_SET); |
---|
208 | if (read(log->fd, temp, 13) != 13 || |
---|
209 | memcmp(temp, "\001\000\000\377\377", 5) != 0) { |
---|
210 | log_clean(log); |
---|
211 | return NULL; |
---|
212 | } |
---|
213 | log->crc = make_ulg(temp + 5); |
---|
214 | log->len = make_ulg(temp + 9); |
---|
215 | |
---|
216 | /* set up to write over empty last block */ |
---|
217 | (void)lseek(log->fd, log->last_off + 5, SEEK_SET); |
---|
218 | log->stored = 0; |
---|
219 | return (void *)log; |
---|
220 | } |
---|
221 | |
---|
222 | /* maximum amount to put in a stored block before starting a new one */ |
---|
223 | #define MAX_BLOCK 16384 |
---|
224 | |
---|
225 | /* write a block to a log object */ |
---|
226 | int gzlog_write(void *obj, char *data, size_t len) |
---|
227 | { |
---|
228 | size_t some; |
---|
229 | unsigned char temp[5]; |
---|
230 | gz_log *log; |
---|
231 | |
---|
232 | /* check object */ |
---|
233 | log = (gz_log *)obj; |
---|
234 | if (log == NULL || log->id != GZLOGID) |
---|
235 | return 1; |
---|
236 | |
---|
237 | /* write stored blocks until all of the input is written */ |
---|
238 | do { |
---|
239 | some = MAX_BLOCK - log->stored; |
---|
240 | if (some > len) |
---|
241 | some = len; |
---|
242 | if (write(log->fd, data, some) != some) |
---|
243 | return 1; |
---|
244 | log->crc = crc32(log->crc, data, some); |
---|
245 | log->len += some; |
---|
246 | len -= some; |
---|
247 | data += some; |
---|
248 | log->stored += some; |
---|
249 | |
---|
250 | /* if the stored block is full, end it and start another */ |
---|
251 | if (log->stored == MAX_BLOCK) { |
---|
252 | (void)lseek(log->fd, log->last_off, SEEK_SET); |
---|
253 | temp[0] = 0; |
---|
254 | dice_ulg(log->stored + ((unsigned long)(~log->stored) << 16), |
---|
255 | temp + 1); |
---|
256 | if (write(log->fd, temp, 5) != 5) |
---|
257 | return 1; |
---|
258 | log->last_off = lseek(log->fd, log->stored, SEEK_CUR); |
---|
259 | (void)lseek(log->fd, 5, SEEK_CUR); |
---|
260 | log->stored = 0; |
---|
261 | } |
---|
262 | } while (len); |
---|
263 | return 0; |
---|
264 | } |
---|
265 | |
---|
266 | /* recompress the remaining stored deflate data in place */ |
---|
267 | local int recomp(gz_log *log) |
---|
268 | { |
---|
269 | z_stream strm; |
---|
270 | size_t len, max; |
---|
271 | unsigned char *in; |
---|
272 | unsigned char *out; |
---|
273 | unsigned char temp[16]; |
---|
274 | |
---|
275 | /* allocate space and read it all in (it's around 1 MB) */ |
---|
276 | len = log->last_off - log->mark_off; |
---|
277 | max = len + (len >> 12) + (len >> 14) + 11; |
---|
278 | out = malloc(max); |
---|
279 | if (out == NULL) |
---|
280 | return 1; |
---|
281 | in = malloc(len); |
---|
282 | if (in == NULL) { |
---|
283 | free(out); |
---|
284 | return 1; |
---|
285 | } |
---|
286 | (void)lseek(log->fd, log->mark_off, SEEK_SET); |
---|
287 | if (read(log->fd, in, len) != len) { |
---|
288 | free(in); |
---|
289 | free(out); |
---|
290 | return 1; |
---|
291 | } |
---|
292 | |
---|
293 | /* recompress in memory, decoding stored data as we go */ |
---|
294 | /* note: this assumes that unsigned is four bytes or more */ |
---|
295 | /* consider not making that assumption */ |
---|
296 | strm.zalloc = Z_NULL; |
---|
297 | strm.zfree = Z_NULL; |
---|
298 | strm.opaque = Z_NULL; |
---|
299 | if (deflateInit2(&strm, Z_BEST_COMPRESSION, Z_DEFLATED, -15, 8, |
---|
300 | Z_DEFAULT_STRATEGY) != Z_OK) { |
---|
301 | free(in); |
---|
302 | free(out); |
---|
303 | return 1; |
---|
304 | } |
---|
305 | strm.next_in = in; |
---|
306 | strm.avail_out = max; |
---|
307 | strm.next_out = out; |
---|
308 | while (len >= 5) { |
---|
309 | if (strm.next_in[0] != 0) |
---|
310 | break; |
---|
311 | strm.avail_in = strm.next_in[1] + (strm.next_in[2] << 8); |
---|
312 | strm.next_in += 5; |
---|
313 | len -= 5; |
---|
314 | if (strm.avail_in != 0) { |
---|
315 | if (len < strm.avail_in) |
---|
316 | break; |
---|
317 | len -= strm.avail_in; |
---|
318 | (void)deflate(&strm, Z_NO_FLUSH); |
---|
319 | if (strm.avail_in != 0 || strm.avail_out == 0) |
---|
320 | break; |
---|
321 | } |
---|
322 | } |
---|
323 | (void)deflate(&strm, Z_SYNC_FLUSH); |
---|
324 | (void)deflateEnd(&strm); |
---|
325 | free(in); |
---|
326 | if (len != 0 || strm.avail_out == 0) { |
---|
327 | free(out); |
---|
328 | return 1; |
---|
329 | } |
---|
330 | |
---|
331 | /* overwrite stored data with compressed data */ |
---|
332 | (void)lseek(log->fd, log->mark_off, SEEK_SET); |
---|
333 | len = max - strm.avail_out; |
---|
334 | if (write(log->fd, out, len) != len) { |
---|
335 | free(out); |
---|
336 | return 1; |
---|
337 | } |
---|
338 | free(out); |
---|
339 | |
---|
340 | /* write last empty block, crc, and length */ |
---|
341 | log->mark_off = log->last_off = lseek(log->fd, 0, SEEK_CUR); |
---|
342 | temp[0] = 1; |
---|
343 | dice_ulg(0xffffL << 16, temp + 1); |
---|
344 | dice_ulg(log->crc, temp + 5); |
---|
345 | dice_ulg(log->len, temp + 9); |
---|
346 | if (write(log->fd, temp, 13) != 13) |
---|
347 | return 1; |
---|
348 | |
---|
349 | /* truncate file to discard remaining stored data and old trailer */ |
---|
350 | ftruncate(log->fd, lseek(log->fd, 0, SEEK_CUR)); |
---|
351 | |
---|
352 | /* update extra field to point to new last empty block */ |
---|
353 | (void)lseek(log->fd, log->extra, SEEK_SET); |
---|
354 | dice_off(log->mark_off, temp); |
---|
355 | dice_off(log->last_off, temp + 8); |
---|
356 | if (write(log->fd, temp, 16) != 16) |
---|
357 | return 1; |
---|
358 | return 0; |
---|
359 | } |
---|
360 | |
---|
361 | /* maximum accumulation of stored blocks before compressing */ |
---|
362 | #define MAX_STORED 1048576 |
---|
363 | |
---|
364 | /* close log object */ |
---|
365 | int gzlog_close(void *obj) |
---|
366 | { |
---|
367 | unsigned char temp[8]; |
---|
368 | gz_log *log; |
---|
369 | |
---|
370 | /* check object */ |
---|
371 | log = (gz_log *)obj; |
---|
372 | if (log == NULL || log->id != GZLOGID) |
---|
373 | return 1; |
---|
374 | |
---|
375 | /* go to start of most recent block being written */ |
---|
376 | (void)lseek(log->fd, log->last_off, SEEK_SET); |
---|
377 | |
---|
378 | /* if some stuff was put there, update block */ |
---|
379 | if (log->stored) { |
---|
380 | temp[0] = 0; |
---|
381 | dice_ulg(log->stored + ((unsigned long)(~log->stored) << 16), |
---|
382 | temp + 1); |
---|
383 | if (write(log->fd, temp, 5) != 5) |
---|
384 | return 1; |
---|
385 | log->last_off = lseek(log->fd, log->stored, SEEK_CUR); |
---|
386 | } |
---|
387 | |
---|
388 | /* write last block (empty) */ |
---|
389 | if (write(log->fd, "\001\000\000\377\377", 5) != 5) |
---|
390 | return 1; |
---|
391 | |
---|
392 | /* write updated crc and uncompressed length */ |
---|
393 | dice_ulg(log->crc, temp); |
---|
394 | dice_ulg(log->len, temp + 4); |
---|
395 | if (write(log->fd, temp, 8) != 8) |
---|
396 | return 1; |
---|
397 | |
---|
398 | /* put offset of that last block in gzip extra block */ |
---|
399 | (void)lseek(log->fd, log->extra + 8, SEEK_SET); |
---|
400 | dice_off(log->last_off, temp); |
---|
401 | if (write(log->fd, temp, 8) != 8) |
---|
402 | return 1; |
---|
403 | |
---|
404 | /* if more than 1 MB stored, then time to compress it */ |
---|
405 | if (log->last_off - log->mark_off > MAX_STORED) { |
---|
406 | if (recomp(log)) |
---|
407 | return 1; |
---|
408 | } |
---|
409 | |
---|
410 | /* unlock and close file */ |
---|
411 | log_clean(log); |
---|
412 | return 0; |
---|
413 | } |
---|