gzlog.c revision 1.1.1.1.76.1 1 1.1 christos /*
2 1.1 christos * gzlog.c
3 1.1.1.1.76.1 pgoyette * Copyright (C) 2004, 2008, 2012, 2016 Mark Adler, all rights reserved
4 1.1 christos * For conditions of distribution and use, see copyright notice in gzlog.h
5 1.1.1.1.76.1 pgoyette * version 2.2, 14 Aug 2012
6 1.1.1.1.76.1 pgoyette */
7 1.1.1.1.76.1 pgoyette
8 1.1.1.1.76.1 pgoyette /*
9 1.1.1.1.76.1 pgoyette gzlog provides a mechanism for frequently appending short strings to a gzip
10 1.1.1.1.76.1 pgoyette file that is efficient both in execution time and compression ratio. The
11 1.1.1.1.76.1 pgoyette strategy is to write the short strings in an uncompressed form to the end of
12 1.1.1.1.76.1 pgoyette the gzip file, only compressing when the amount of uncompressed data has
13 1.1.1.1.76.1 pgoyette reached a given threshold.
14 1.1.1.1.76.1 pgoyette
15 1.1.1.1.76.1 pgoyette gzlog also provides protection against interruptions in the process due to
16 1.1.1.1.76.1 pgoyette system crashes. The status of the operation is recorded in an extra field
17 1.1.1.1.76.1 pgoyette in the gzip file, and is only updated once the gzip file is brought to a
18 1.1.1.1.76.1 pgoyette valid state. The last data to be appended or compressed is saved in an
19 1.1.1.1.76.1 pgoyette auxiliary file, so that if the operation is interrupted, it can be completed
20 1.1.1.1.76.1 pgoyette the next time an append operation is attempted.
21 1.1.1.1.76.1 pgoyette
22 1.1.1.1.76.1 pgoyette gzlog maintains another auxiliary file with the last 32K of data from the
23 1.1.1.1.76.1 pgoyette compressed portion, which is preloaded for the compression of the subsequent
24 1.1.1.1.76.1 pgoyette data. This minimizes the impact to the compression ratio of appending.
25 1.1 christos */
26 1.1 christos
27 1.1.1.1.76.1 pgoyette /*
28 1.1.1.1.76.1 pgoyette Operations Concept:
29 1.1.1.1.76.1 pgoyette
30 1.1.1.1.76.1 pgoyette Files (log name "foo"):
31 1.1.1.1.76.1 pgoyette foo.gz -- gzip file with the complete log
32 1.1.1.1.76.1 pgoyette foo.add -- last message to append or last data to compress
33 1.1.1.1.76.1 pgoyette foo.dict -- dictionary of the last 32K of data for next compression
34 1.1.1.1.76.1 pgoyette foo.temp -- temporary dictionary file for compression after this one
35 1.1.1.1.76.1 pgoyette foo.lock -- lock file for reading and writing the other files
36 1.1.1.1.76.1 pgoyette foo.repairs -- log file for log file recovery operations (not compressed)
37 1.1.1.1.76.1 pgoyette
38 1.1.1.1.76.1 pgoyette gzip file structure:
39 1.1.1.1.76.1 pgoyette - fixed-length (no file name) header with extra field (see below)
40 1.1.1.1.76.1 pgoyette - compressed data ending initially with empty stored block
41 1.1.1.1.76.1 pgoyette - uncompressed data filling out originally empty stored block and
42 1.1.1.1.76.1 pgoyette subsequent stored blocks as needed (16K max each)
43 1.1.1.1.76.1 pgoyette - gzip trailer
44 1.1.1.1.76.1 pgoyette - no junk at end (no other gzip streams)
45 1.1.1.1.76.1 pgoyette
46 1.1.1.1.76.1 pgoyette When appending data, the information in the first three items above plus the
47 1.1.1.1.76.1 pgoyette foo.add file are sufficient to recover an interrupted append operation. The
48 1.1.1.1.76.1 pgoyette extra field has the necessary information to restore the start of the last
49 1.1.1.1.76.1 pgoyette stored block and determine where to append the data in the foo.add file, as
50 1.1.1.1.76.1 pgoyette well as the crc and length of the gzip data before the append operation.
51 1.1.1.1.76.1 pgoyette
52 1.1.1.1.76.1 pgoyette The foo.add file is created before the gzip file is marked for append, and
53 1.1.1.1.76.1 pgoyette deleted after the gzip file is marked as complete. So if the append
54 1.1.1.1.76.1 pgoyette operation is interrupted, the data to add will still be there. If due to
55 1.1.1.1.76.1 pgoyette some external force, the foo.add file gets deleted between when the append
56 1.1.1.1.76.1 pgoyette operation was interrupted and when recovery is attempted, the gzip file will
57 1.1.1.1.76.1 pgoyette still be restored, but without the appended data.
58 1.1.1.1.76.1 pgoyette
59 1.1.1.1.76.1 pgoyette When compressing data, the information in the first two items above plus the
60 1.1.1.1.76.1 pgoyette foo.add file are sufficient to recover an interrupted compress operation.
61 1.1.1.1.76.1 pgoyette The extra field has the necessary information to find the end of the
62 1.1.1.1.76.1 pgoyette compressed data, and contains both the crc and length of just the compressed
63 1.1.1.1.76.1 pgoyette data and of the complete set of data including the contents of the foo.add
64 1.1.1.1.76.1 pgoyette file.
65 1.1.1.1.76.1 pgoyette
66 1.1.1.1.76.1 pgoyette Again, the foo.add file is maintained during the compress operation in case
67 1.1.1.1.76.1 pgoyette of an interruption. If in the unlikely event the foo.add file with the data
68 1.1.1.1.76.1 pgoyette to be compressed is missing due to some external force, a gzip file with
69 1.1.1.1.76.1 pgoyette just the previous compressed data will be reconstructed. In this case, all
70 1.1.1.1.76.1 pgoyette of the data that was to be compressed is lost (approximately one megabyte).
71 1.1.1.1.76.1 pgoyette This will not occur if all that happened was an interruption of the compress
72 1.1.1.1.76.1 pgoyette operation.
73 1.1.1.1.76.1 pgoyette
74 1.1.1.1.76.1 pgoyette The third state that is marked is the replacement of the old dictionary with
75 1.1.1.1.76.1 pgoyette the new dictionary after a compress operation. Once compression is
76 1.1.1.1.76.1 pgoyette complete, the gzip file is marked as being in the replace state. This
77 1.1.1.1.76.1 pgoyette completes the gzip file, so an interrupt after being so marked does not
78 1.1.1.1.76.1 pgoyette result in recompression. Then the dictionary file is replaced, and the gzip
79 1.1.1.1.76.1 pgoyette file is marked as completed. This state prevents the possibility of
80 1.1.1.1.76.1 pgoyette restarting compression with the wrong dictionary file.
81 1.1.1.1.76.1 pgoyette
82 1.1.1.1.76.1 pgoyette All three operations are wrapped by a lock/unlock procedure. In order to
83 1.1.1.1.76.1 pgoyette gain exclusive access to the log files, first a foo.lock file must be
84 1.1.1.1.76.1 pgoyette exclusively created. When all operations are complete, the lock is
85 1.1.1.1.76.1 pgoyette released by deleting the foo.lock file. If when attempting to create the
86 1.1.1.1.76.1 pgoyette lock file, it already exists and the modify time of the lock file is more
87 1.1.1.1.76.1 pgoyette than five minutes old (set by the PATIENCE define below), then the old
88 1.1.1.1.76.1 pgoyette lock file is considered stale and deleted, and the exclusive creation of
89 1.1.1.1.76.1 pgoyette the lock file is retried. To assure that there are no false assessments
90 1.1.1.1.76.1 pgoyette of the staleness of the lock file, the operations periodically touch the
91 1.1.1.1.76.1 pgoyette lock file to update the modified date.
92 1.1.1.1.76.1 pgoyette
93 1.1.1.1.76.1 pgoyette Following is the definition of the extra field with all of the information
94 1.1.1.1.76.1 pgoyette required to enable the above append and compress operations and their
95 1.1.1.1.76.1 pgoyette recovery if interrupted. Multi-byte values are stored little endian
96 1.1.1.1.76.1 pgoyette (consistent with the gzip format). File pointers are eight bytes long.
97 1.1.1.1.76.1 pgoyette The crc's and lengths for the gzip trailer are four bytes long. (Note that
98 1.1.1.1.76.1 pgoyette the length at the end of a gzip file is used for error checking only, and
99 1.1.1.1.76.1 pgoyette for large files is actually the length modulo 2^32.) The stored block
100 1.1.1.1.76.1 pgoyette length is two bytes long. The gzip extra field two-byte identification is
101 1.1.1.1.76.1 pgoyette "ap" for append. It is assumed that writing the extra field to the file is
102 1.1.1.1.76.1 pgoyette an "atomic" operation. That is, either all of the extra field is written
103 1.1.1.1.76.1 pgoyette to the file, or none of it is, if the operation is interrupted right at the
104 1.1.1.1.76.1 pgoyette point of updating the extra field. This is a reasonable assumption, since
105 1.1.1.1.76.1 pgoyette the extra field is within the first 52 bytes of the file, which is smaller
106 1.1.1.1.76.1 pgoyette than any expected block size for a mass storage device (usually 512 bytes or
107 1.1.1.1.76.1 pgoyette larger).
108 1.1.1.1.76.1 pgoyette
109 1.1.1.1.76.1 pgoyette Extra field (35 bytes):
110 1.1.1.1.76.1 pgoyette - Pointer to first stored block length -- this points to the two-byte length
111 1.1.1.1.76.1 pgoyette of the first stored block, which is followed by the two-byte, one's
112 1.1.1.1.76.1 pgoyette complement of that length. The stored block length is preceded by the
113 1.1.1.1.76.1 pgoyette three-bit header of the stored block, which is the actual start of the
114 1.1.1.1.76.1 pgoyette stored block in the deflate format. See the bit offset field below.
115 1.1.1.1.76.1 pgoyette - Pointer to the last stored block length. This is the same as above, but
116 1.1.1.1.76.1 pgoyette for the last stored block of the uncompressed data in the gzip file.
117 1.1.1.1.76.1 pgoyette Initially this is the same as the first stored block length pointer.
118 1.1.1.1.76.1 pgoyette When the stored block gets to 16K (see the MAX_STORE define), then a new
119 1.1.1.1.76.1 pgoyette stored block as added, at which point the last stored block length pointer
120 1.1.1.1.76.1 pgoyette is different from the first stored block length pointer. When they are
121 1.1.1.1.76.1 pgoyette different, the first bit of the last stored block header is eight bits, or
122 1.1.1.1.76.1 pgoyette one byte back from the block length.
123 1.1.1.1.76.1 pgoyette - Compressed data crc and length. This is the crc and length of the data
124 1.1.1.1.76.1 pgoyette that is in the compressed portion of the deflate stream. These are used
125 1.1.1.1.76.1 pgoyette only in the event that the foo.add file containing the data to compress is
126 1.1.1.1.76.1 pgoyette lost after a compress operation is interrupted.
127 1.1.1.1.76.1 pgoyette - Total data crc and length. This is the crc and length of all of the data
128 1.1.1.1.76.1 pgoyette stored in the gzip file, compressed and uncompressed. It is used to
129 1.1.1.1.76.1 pgoyette reconstruct the gzip trailer when compressing, as well as when recovering
130 1.1.1.1.76.1 pgoyette interrupted operations.
131 1.1.1.1.76.1 pgoyette - Final stored block length. This is used to quickly find where to append,
132 1.1.1.1.76.1 pgoyette and allows the restoration of the original final stored block state when
133 1.1.1.1.76.1 pgoyette an append operation is interrupted.
134 1.1.1.1.76.1 pgoyette - First stored block start as the number of bits back from the final stored
135 1.1.1.1.76.1 pgoyette block first length byte. This value is in the range of 3..10, and is
136 1.1.1.1.76.1 pgoyette stored as the low three bits of the final byte of the extra field after
137 1.1.1.1.76.1 pgoyette subtracting three (0..7). This allows the last-block bit of the stored
138 1.1.1.1.76.1 pgoyette block header to be updated when a new stored block is added, for the case
139 1.1.1.1.76.1 pgoyette when the first stored block and the last stored block are the same. (When
140 1.1.1.1.76.1 pgoyette they are different, the numbers of bits back is known to be eight.) This
141 1.1.1.1.76.1 pgoyette also allows for new compressed data to be appended to the old compressed
142 1.1.1.1.76.1 pgoyette data in the compress operation, overwriting the previous first stored
143 1.1.1.1.76.1 pgoyette block, or for the compressed data to be terminated and a valid gzip file
144 1.1.1.1.76.1 pgoyette reconstructed on the off chance that a compression operation was
145 1.1.1.1.76.1 pgoyette interrupted and the data to compress in the foo.add file was deleted.
146 1.1.1.1.76.1 pgoyette - The operation in process. This is the next two bits in the last byte (the
147 1.1.1.1.76.1 pgoyette bits under the mask 0x18). The are interpreted as 0: nothing in process,
148 1.1.1.1.76.1 pgoyette 1: append in process, 2: compress in process, 3: replace in process.
149 1.1.1.1.76.1 pgoyette - The top three bits of the last byte in the extra field are reserved and
150 1.1.1.1.76.1 pgoyette are currently set to zero.
151 1.1.1.1.76.1 pgoyette
152 1.1.1.1.76.1 pgoyette Main procedure:
153 1.1.1.1.76.1 pgoyette - Exclusively create the foo.lock file using the O_CREAT and O_EXCL modes of
154 1.1.1.1.76.1 pgoyette the system open() call. If the modify time of an existing lock file is
155 1.1.1.1.76.1 pgoyette more than PATIENCE seconds old, then the lock file is deleted and the
156 1.1.1.1.76.1 pgoyette exclusive create is retried.
157 1.1.1.1.76.1 pgoyette - Load the extra field from the foo.gz file, and see if an operation was in
158 1.1.1.1.76.1 pgoyette progress but not completed. If so, apply the recovery procedure below.
159 1.1.1.1.76.1 pgoyette - Perform the append procedure with the provided data.
160 1.1.1.1.76.1 pgoyette - If the uncompressed data in the foo.gz file is 1MB or more, apply the
161 1.1.1.1.76.1 pgoyette compress procedure.
162 1.1.1.1.76.1 pgoyette - Delete the foo.lock file.
163 1.1.1.1.76.1 pgoyette
164 1.1.1.1.76.1 pgoyette Append procedure:
165 1.1.1.1.76.1 pgoyette - Put what to append in the foo.add file so that the operation can be
166 1.1.1.1.76.1 pgoyette restarted if this procedure is interrupted.
167 1.1.1.1.76.1 pgoyette - Mark the foo.gz extra field with the append operation in progress.
168 1.1.1.1.76.1 pgoyette + Restore the original last-block bit and stored block length of the last
169 1.1.1.1.76.1 pgoyette stored block from the information in the extra field, in case a previous
170 1.1.1.1.76.1 pgoyette append operation was interrupted.
171 1.1.1.1.76.1 pgoyette - Append the provided data to the last stored block, creating new stored
172 1.1.1.1.76.1 pgoyette blocks as needed and updating the stored blocks last-block bits and
173 1.1.1.1.76.1 pgoyette lengths.
174 1.1.1.1.76.1 pgoyette - Update the crc and length with the new data, and write the gzip trailer.
175 1.1.1.1.76.1 pgoyette - Write over the extra field (with a single write operation) with the new
176 1.1.1.1.76.1 pgoyette pointers, lengths, and crc's, and mark the gzip file as not in process.
177 1.1.1.1.76.1 pgoyette Though there is still a foo.add file, it will be ignored since nothing
178 1.1.1.1.76.1 pgoyette is in process. If a foo.add file is leftover from a previously
179 1.1.1.1.76.1 pgoyette completed operation, it is truncated when writing new data to it.
180 1.1.1.1.76.1 pgoyette - Delete the foo.add file.
181 1.1.1.1.76.1 pgoyette
182 1.1.1.1.76.1 pgoyette Compress and replace procedures:
183 1.1.1.1.76.1 pgoyette - Read all of the uncompressed data in the stored blocks in foo.gz and write
184 1.1.1.1.76.1 pgoyette it to foo.add. Also write foo.temp with the last 32K of that data to
185 1.1.1.1.76.1 pgoyette provide a dictionary for the next invocation of this procedure.
186 1.1.1.1.76.1 pgoyette - Rewrite the extra field marking foo.gz with a compression in process.
187 1.1.1.1.76.1 pgoyette * If there is no data provided to compress (due to a missing foo.add file
188 1.1.1.1.76.1 pgoyette when recovering), reconstruct and truncate the foo.gz file to contain
189 1.1.1.1.76.1 pgoyette only the previous compressed data and proceed to the step after the next
190 1.1.1.1.76.1 pgoyette one. Otherwise ...
191 1.1.1.1.76.1 pgoyette - Compress the data with the dictionary in foo.dict, and write to the
192 1.1.1.1.76.1 pgoyette foo.gz file starting at the bit immediately following the last previously
193 1.1.1.1.76.1 pgoyette compressed block. If there is no foo.dict, proceed anyway with the
194 1.1.1.1.76.1 pgoyette compression at slightly reduced efficiency. (For the foo.dict file to be
195 1.1.1.1.76.1 pgoyette missing requires some external failure beyond simply the interruption of
196 1.1.1.1.76.1 pgoyette a compress operation.) During this process, the foo.lock file is
197 1.1.1.1.76.1 pgoyette periodically touched to assure that that file is not considered stale by
198 1.1.1.1.76.1 pgoyette another process before we're done. The deflation is terminated with a
199 1.1.1.1.76.1 pgoyette non-last empty static block (10 bits long), that is then located and
200 1.1.1.1.76.1 pgoyette written over by a last-bit-set empty stored block.
201 1.1.1.1.76.1 pgoyette - Append the crc and length of the data in the gzip file (previously
202 1.1.1.1.76.1 pgoyette calculated during the append operations).
203 1.1.1.1.76.1 pgoyette - Write over the extra field with the updated stored block offsets, bits
204 1.1.1.1.76.1 pgoyette back, crc's, and lengths, and mark foo.gz as in process for a replacement
205 1.1.1.1.76.1 pgoyette of the dictionary.
206 1.1.1.1.76.1 pgoyette @ Delete the foo.add file.
207 1.1.1.1.76.1 pgoyette - Replace foo.dict with foo.temp.
208 1.1.1.1.76.1 pgoyette - Write over the extra field, marking foo.gz as complete.
209 1.1.1.1.76.1 pgoyette
210 1.1.1.1.76.1 pgoyette Recovery procedure:
211 1.1.1.1.76.1 pgoyette - If not a replace recovery, read in the foo.add file, and provide that data
212 1.1.1.1.76.1 pgoyette to the appropriate recovery below. If there is no foo.add file, provide
213 1.1.1.1.76.1 pgoyette a zero data length to the recovery. In that case, the append recovery
214 1.1.1.1.76.1 pgoyette restores the foo.gz to the previous compressed + uncompressed data state.
215 1.1.1.1.76.1 pgoyette For the the compress recovery, a missing foo.add file results in foo.gz
216 1.1.1.1.76.1 pgoyette being restored to the previous compressed-only data state.
217 1.1.1.1.76.1 pgoyette - Append recovery:
218 1.1.1.1.76.1 pgoyette - Pick up append at + step above
219 1.1.1.1.76.1 pgoyette - Compress recovery:
220 1.1.1.1.76.1 pgoyette - Pick up compress at * step above
221 1.1.1.1.76.1 pgoyette - Replace recovery:
222 1.1.1.1.76.1 pgoyette - Pick up compress at @ step above
223 1.1.1.1.76.1 pgoyette - Log the repair with a date stamp in foo.repairs
224 1.1.1.1.76.1 pgoyette */
225 1.1.1.1.76.1 pgoyette
226 1.1.1.1.76.1 pgoyette #include <sys/types.h>
227 1.1.1.1.76.1 pgoyette #include <stdio.h> /* rename, fopen, fprintf, fclose */
228 1.1.1.1.76.1 pgoyette #include <stdlib.h> /* malloc, free */
229 1.1.1.1.76.1 pgoyette #include <string.h> /* strlen, strrchr, strcpy, strncpy, strcmp */
230 1.1.1.1.76.1 pgoyette #include <fcntl.h> /* open */
231 1.1.1.1.76.1 pgoyette #include <unistd.h> /* lseek, read, write, close, unlink, sleep, */
232 1.1.1.1.76.1 pgoyette /* ftruncate, fsync */
233 1.1.1.1.76.1 pgoyette #include <errno.h> /* errno */
234 1.1.1.1.76.1 pgoyette #include <time.h> /* time, ctime */
235 1.1.1.1.76.1 pgoyette #include <sys/stat.h> /* stat */
236 1.1.1.1.76.1 pgoyette #include <sys/time.h> /* utimes */
237 1.1.1.1.76.1 pgoyette #include "zlib.h" /* crc32 */
238 1.1.1.1.76.1 pgoyette
239 1.1.1.1.76.1 pgoyette #include "gzlog.h" /* header for external access */
240 1.1 christos
241 1.1 christos #define local static
242 1.1.1.1.76.1 pgoyette typedef unsigned int uint;
243 1.1.1.1.76.1 pgoyette typedef unsigned long ulong;
244 1.1 christos
245 1.1.1.1.76.1 pgoyette /* Macro for debugging to deterministically force recovery operations */
246 1.1.1.1.76.1 pgoyette #ifdef GZLOG_DEBUG
247 1.1.1.1.76.1 pgoyette #include <setjmp.h> /* longjmp */
248 1.1.1.1.76.1 pgoyette jmp_buf gzlog_jump; /* where to go back to */
249 1.1.1.1.76.1 pgoyette int gzlog_bail = 0; /* which point to bail at (1..8) */
250 1.1.1.1.76.1 pgoyette int gzlog_count = -1; /* number of times through to wait */
251 1.1.1.1.76.1 pgoyette # define BAIL(n) do { if (n == gzlog_bail && gzlog_count-- == 0) \
252 1.1.1.1.76.1 pgoyette longjmp(gzlog_jump, gzlog_bail); } while (0)
253 1.1.1.1.76.1 pgoyette #else
254 1.1.1.1.76.1 pgoyette # define BAIL(n)
255 1.1.1.1.76.1 pgoyette #endif
256 1.1.1.1.76.1 pgoyette
257 1.1.1.1.76.1 pgoyette /* how old the lock file can be in seconds before considering it stale */
258 1.1.1.1.76.1 pgoyette #define PATIENCE 300
259 1.1.1.1.76.1 pgoyette
260 1.1.1.1.76.1 pgoyette /* maximum stored block size in Kbytes -- must be in 1..63 */
261 1.1.1.1.76.1 pgoyette #define MAX_STORE 16
262 1.1.1.1.76.1 pgoyette
263 1.1.1.1.76.1 pgoyette /* number of stored Kbytes to trigger compression (must be >= 32 to allow
264 1.1.1.1.76.1 pgoyette dictionary construction, and <= 204 * MAX_STORE, in order for >> 10 to
265 1.1.1.1.76.1 pgoyette discard the stored block headers contribution of five bytes each) */
266 1.1.1.1.76.1 pgoyette #define TRIGGER 1024
267 1.1.1.1.76.1 pgoyette
268 1.1.1.1.76.1 pgoyette /* size of a deflate dictionary (this cannot be changed) */
269 1.1.1.1.76.1 pgoyette #define DICT 32768U
270 1.1.1.1.76.1 pgoyette
271 1.1.1.1.76.1 pgoyette /* values for the operation (2 bits) */
272 1.1.1.1.76.1 pgoyette #define NO_OP 0
273 1.1.1.1.76.1 pgoyette #define APPEND_OP 1
274 1.1.1.1.76.1 pgoyette #define COMPRESS_OP 2
275 1.1.1.1.76.1 pgoyette #define REPLACE_OP 3
276 1.1.1.1.76.1 pgoyette
277 1.1.1.1.76.1 pgoyette /* macros to extract little-endian integers from an unsigned byte buffer */
278 1.1.1.1.76.1 pgoyette #define PULL2(p) ((p)[0]+((uint)((p)[1])<<8))
279 1.1.1.1.76.1 pgoyette #define PULL4(p) (PULL2(p)+((ulong)PULL2(p+2)<<16))
280 1.1.1.1.76.1 pgoyette #define PULL8(p) (PULL4(p)+((off_t)PULL4(p+4)<<32))
281 1.1.1.1.76.1 pgoyette
282 1.1.1.1.76.1 pgoyette /* macros to store integers into a byte buffer in little-endian order */
283 1.1.1.1.76.1 pgoyette #define PUT2(p,a) do {(p)[0]=a;(p)[1]=(a)>>8;} while(0)
284 1.1.1.1.76.1 pgoyette #define PUT4(p,a) do {PUT2(p,a);PUT2(p+2,a>>16);} while(0)
285 1.1.1.1.76.1 pgoyette #define PUT8(p,a) do {PUT4(p,a);PUT4(p+4,a>>32);} while(0)
286 1.1.1.1.76.1 pgoyette
287 1.1.1.1.76.1 pgoyette /* internal structure for log information */
288 1.1.1.1.76.1 pgoyette #define LOGID "\106\035\172" /* should be three non-zero characters */
289 1.1.1.1.76.1 pgoyette struct log {
290 1.1.1.1.76.1 pgoyette char id[4]; /* contains LOGID to detect inadvertent overwrites */
291 1.1.1.1.76.1 pgoyette int fd; /* file descriptor for .gz file, opened read/write */
292 1.1.1.1.76.1 pgoyette char *path; /* allocated path, e.g. "/var/log/foo" or "foo" */
293 1.1.1.1.76.1 pgoyette char *end; /* end of path, for appending suffices such as ".gz" */
294 1.1.1.1.76.1 pgoyette off_t first; /* offset of first stored block first length byte */
295 1.1.1.1.76.1 pgoyette int back; /* location of first block id in bits back from first */
296 1.1.1.1.76.1 pgoyette uint stored; /* bytes currently in last stored block */
297 1.1.1.1.76.1 pgoyette off_t last; /* offset of last stored block first length byte */
298 1.1.1.1.76.1 pgoyette ulong ccrc; /* crc of compressed data */
299 1.1.1.1.76.1 pgoyette ulong clen; /* length (modulo 2^32) of compressed data */
300 1.1.1.1.76.1 pgoyette ulong tcrc; /* crc of total data */
301 1.1.1.1.76.1 pgoyette ulong tlen; /* length (modulo 2^32) of total data */
302 1.1.1.1.76.1 pgoyette time_t lock; /* last modify time of our lock file */
303 1.1.1.1.76.1 pgoyette };
304 1.1 christos
305 1.1.1.1.76.1 pgoyette /* gzip header for gzlog */
306 1.1.1.1.76.1 pgoyette local unsigned char log_gzhead[] = {
307 1.1.1.1.76.1 pgoyette 0x1f, 0x8b, /* magic gzip id */
308 1.1.1.1.76.1 pgoyette 8, /* compression method is deflate */
309 1.1.1.1.76.1 pgoyette 4, /* there is an extra field (no file name) */
310 1.1.1.1.76.1 pgoyette 0, 0, 0, 0, /* no modification time provided */
311 1.1.1.1.76.1 pgoyette 0, 0xff, /* no extra flags, no OS specified */
312 1.1.1.1.76.1 pgoyette 39, 0, 'a', 'p', 35, 0 /* extra field with "ap" subfield */
313 1.1.1.1.76.1 pgoyette /* 35 is EXTRA, 39 is EXTRA + 4 */
314 1.1.1.1.76.1 pgoyette };
315 1.1.1.1.76.1 pgoyette
316 1.1.1.1.76.1 pgoyette #define HEAD sizeof(log_gzhead) /* should be 16 */
317 1.1.1.1.76.1 pgoyette
318 1.1.1.1.76.1 pgoyette /* initial gzip extra field content (52 == HEAD + EXTRA + 1) */
319 1.1.1.1.76.1 pgoyette local unsigned char log_gzext[] = {
320 1.1.1.1.76.1 pgoyette 52, 0, 0, 0, 0, 0, 0, 0, /* offset of first stored block length */
321 1.1.1.1.76.1 pgoyette 52, 0, 0, 0, 0, 0, 0, 0, /* offset of last stored block length */
322 1.1.1.1.76.1 pgoyette 0, 0, 0, 0, 0, 0, 0, 0, /* compressed data crc and length */
323 1.1.1.1.76.1 pgoyette 0, 0, 0, 0, 0, 0, 0, 0, /* total data crc and length */
324 1.1.1.1.76.1 pgoyette 0, 0, /* final stored block data length */
325 1.1.1.1.76.1 pgoyette 5 /* op is NO_OP, last bit 8 bits back */
326 1.1.1.1.76.1 pgoyette };
327 1.1.1.1.76.1 pgoyette
328 1.1.1.1.76.1 pgoyette #define EXTRA sizeof(log_gzext) /* should be 35 */
329 1.1.1.1.76.1 pgoyette
330 1.1.1.1.76.1 pgoyette /* initial gzip data and trailer */
331 1.1.1.1.76.1 pgoyette local unsigned char log_gzbody[] = {
332 1.1.1.1.76.1 pgoyette 1, 0, 0, 0xff, 0xff, /* empty stored block (last) */
333 1.1.1.1.76.1 pgoyette 0, 0, 0, 0, /* crc */
334 1.1.1.1.76.1 pgoyette 0, 0, 0, 0 /* uncompressed length */
335 1.1.1.1.76.1 pgoyette };
336 1.1.1.1.76.1 pgoyette
337 1.1.1.1.76.1 pgoyette #define BODY sizeof(log_gzbody)
338 1.1.1.1.76.1 pgoyette
339 1.1.1.1.76.1 pgoyette /* Exclusively create foo.lock in order to negotiate exclusive access to the
340 1.1.1.1.76.1 pgoyette foo.* files. If the modify time of an existing lock file is greater than
341 1.1.1.1.76.1 pgoyette PATIENCE seconds in the past, then consider the lock file to have been
342 1.1.1.1.76.1 pgoyette abandoned, delete it, and try the exclusive create again. Save the lock
343 1.1.1.1.76.1 pgoyette file modify time for verification of ownership. Return 0 on success, or -1
344 1.1.1.1.76.1 pgoyette on failure, usually due to an access restriction or invalid path. Note that
345 1.1.1.1.76.1 pgoyette if stat() or unlink() fails, it may be due to another process noticing the
346 1.1.1.1.76.1 pgoyette abandoned lock file a smidge sooner and deleting it, so those are not
347 1.1.1.1.76.1 pgoyette flagged as an error. */
348 1.1.1.1.76.1 pgoyette local int log_lock(struct log *log)
349 1.1 christos {
350 1.1.1.1.76.1 pgoyette int fd;
351 1.1.1.1.76.1 pgoyette struct stat st;
352 1.1 christos
353 1.1.1.1.76.1 pgoyette strcpy(log->end, ".lock");
354 1.1.1.1.76.1 pgoyette while ((fd = open(log->path, O_CREAT | O_EXCL, 0644)) < 0) {
355 1.1.1.1.76.1 pgoyette if (errno != EEXIST)
356 1.1.1.1.76.1 pgoyette return -1;
357 1.1.1.1.76.1 pgoyette if (stat(log->path, &st) == 0 && time(NULL) - st.st_mtime > PATIENCE) {
358 1.1.1.1.76.1 pgoyette unlink(log->path);
359 1.1.1.1.76.1 pgoyette continue;
360 1.1.1.1.76.1 pgoyette }
361 1.1.1.1.76.1 pgoyette sleep(2); /* relinquish the CPU for two seconds while waiting */
362 1.1.1.1.76.1 pgoyette }
363 1.1.1.1.76.1 pgoyette close(fd);
364 1.1.1.1.76.1 pgoyette if (stat(log->path, &st) == 0)
365 1.1.1.1.76.1 pgoyette log->lock = st.st_mtime;
366 1.1.1.1.76.1 pgoyette return 0;
367 1.1.1.1.76.1 pgoyette }
368 1.1 christos
369 1.1.1.1.76.1 pgoyette /* Update the modify time of the lock file to now, in order to prevent another
370 1.1.1.1.76.1 pgoyette task from thinking that the lock is stale. Save the lock file modify time
371 1.1.1.1.76.1 pgoyette for verification of ownership. */
372 1.1.1.1.76.1 pgoyette local void log_touch(struct log *log)
373 1.1.1.1.76.1 pgoyette {
374 1.1.1.1.76.1 pgoyette struct stat st;
375 1.1.1.1.76.1 pgoyette
376 1.1.1.1.76.1 pgoyette strcpy(log->end, ".lock");
377 1.1.1.1.76.1 pgoyette utimes(log->path, NULL);
378 1.1.1.1.76.1 pgoyette if (stat(log->path, &st) == 0)
379 1.1.1.1.76.1 pgoyette log->lock = st.st_mtime;
380 1.1 christos }
381 1.1 christos
382 1.1.1.1.76.1 pgoyette /* Check the log file modify time against what is expected. Return true if
383 1.1.1.1.76.1 pgoyette this is not our lock. If it is our lock, touch it to keep it. */
384 1.1.1.1.76.1 pgoyette local int log_check(struct log *log)
385 1.1 christos {
386 1.1.1.1.76.1 pgoyette struct stat st;
387 1.1.1.1.76.1 pgoyette
388 1.1.1.1.76.1 pgoyette strcpy(log->end, ".lock");
389 1.1.1.1.76.1 pgoyette if (stat(log->path, &st) || st.st_mtime != log->lock)
390 1.1.1.1.76.1 pgoyette return 1;
391 1.1.1.1.76.1 pgoyette log_touch(log);
392 1.1.1.1.76.1 pgoyette return 0;
393 1.1 christos }
394 1.1 christos
395 1.1.1.1.76.1 pgoyette /* Unlock a previously acquired lock, but only if it's ours. */
396 1.1.1.1.76.1 pgoyette local void log_unlock(struct log *log)
397 1.1 christos {
398 1.1.1.1.76.1 pgoyette if (log_check(log))
399 1.1.1.1.76.1 pgoyette return;
400 1.1.1.1.76.1 pgoyette strcpy(log->end, ".lock");
401 1.1.1.1.76.1 pgoyette unlink(log->path);
402 1.1.1.1.76.1 pgoyette log->lock = 0;
403 1.1 christos }
404 1.1 christos
405 1.1.1.1.76.1 pgoyette /* Check the gzip header and read in the extra field, filling in the values in
406 1.1.1.1.76.1 pgoyette the log structure. Return op on success or -1 if the gzip header was not as
407 1.1.1.1.76.1 pgoyette expected. op is the current operation in progress last written to the extra
408 1.1.1.1.76.1 pgoyette field. This assumes that the gzip file has already been opened, with the
409 1.1.1.1.76.1 pgoyette file descriptor log->fd. */
410 1.1.1.1.76.1 pgoyette local int log_head(struct log *log)
411 1.1 christos {
412 1.1.1.1.76.1 pgoyette int op;
413 1.1.1.1.76.1 pgoyette unsigned char buf[HEAD + EXTRA];
414 1.1 christos
415 1.1.1.1.76.1 pgoyette if (lseek(log->fd, 0, SEEK_SET) < 0 ||
416 1.1.1.1.76.1 pgoyette read(log->fd, buf, HEAD + EXTRA) != HEAD + EXTRA ||
417 1.1.1.1.76.1 pgoyette memcmp(buf, log_gzhead, HEAD)) {
418 1.1.1.1.76.1 pgoyette return -1;
419 1.1.1.1.76.1 pgoyette }
420 1.1.1.1.76.1 pgoyette log->first = PULL8(buf + HEAD);
421 1.1.1.1.76.1 pgoyette log->last = PULL8(buf + HEAD + 8);
422 1.1.1.1.76.1 pgoyette log->ccrc = PULL4(buf + HEAD + 16);
423 1.1.1.1.76.1 pgoyette log->clen = PULL4(buf + HEAD + 20);
424 1.1.1.1.76.1 pgoyette log->tcrc = PULL4(buf + HEAD + 24);
425 1.1.1.1.76.1 pgoyette log->tlen = PULL4(buf + HEAD + 28);
426 1.1.1.1.76.1 pgoyette log->stored = PULL2(buf + HEAD + 32);
427 1.1.1.1.76.1 pgoyette log->back = 3 + (buf[HEAD + 34] & 7);
428 1.1.1.1.76.1 pgoyette op = (buf[HEAD + 34] >> 3) & 3;
429 1.1.1.1.76.1 pgoyette return op;
430 1.1 christos }
431 1.1 christos
432 1.1.1.1.76.1 pgoyette /* Write over the extra field contents, marking the operation as op. Use fsync
433 1.1.1.1.76.1 pgoyette to assure that the device is written to, and in the requested order. This
434 1.1.1.1.76.1 pgoyette operation, and only this operation, is assumed to be atomic in order to
435 1.1.1.1.76.1 pgoyette assure that the log is recoverable in the event of an interruption at any
436 1.1.1.1.76.1 pgoyette point in the process. Return -1 if the write to foo.gz failed. */
437 1.1.1.1.76.1 pgoyette local int log_mark(struct log *log, int op)
438 1.1 christos {
439 1.1.1.1.76.1 pgoyette int ret;
440 1.1.1.1.76.1 pgoyette unsigned char ext[EXTRA];
441 1.1 christos
442 1.1.1.1.76.1 pgoyette PUT8(ext, log->first);
443 1.1.1.1.76.1 pgoyette PUT8(ext + 8, log->last);
444 1.1.1.1.76.1 pgoyette PUT4(ext + 16, log->ccrc);
445 1.1.1.1.76.1 pgoyette PUT4(ext + 20, log->clen);
446 1.1.1.1.76.1 pgoyette PUT4(ext + 24, log->tcrc);
447 1.1.1.1.76.1 pgoyette PUT4(ext + 28, log->tlen);
448 1.1.1.1.76.1 pgoyette PUT2(ext + 32, log->stored);
449 1.1.1.1.76.1 pgoyette ext[34] = log->back - 3 + (op << 3);
450 1.1.1.1.76.1 pgoyette fsync(log->fd);
451 1.1.1.1.76.1 pgoyette ret = lseek(log->fd, HEAD, SEEK_SET) < 0 ||
452 1.1.1.1.76.1 pgoyette write(log->fd, ext, EXTRA) != EXTRA ? -1 : 0;
453 1.1.1.1.76.1 pgoyette fsync(log->fd);
454 1.1.1.1.76.1 pgoyette return ret;
455 1.1 christos }
456 1.1 christos
457 1.1.1.1.76.1 pgoyette /* Rewrite the last block header bits and subsequent zero bits to get to a byte
458 1.1.1.1.76.1 pgoyette boundary, setting the last block bit if last is true, and then write the
459 1.1.1.1.76.1 pgoyette remainder of the stored block header (length and one's complement). Leave
460 1.1.1.1.76.1 pgoyette the file pointer after the end of the last stored block data. Return -1 if
461 1.1.1.1.76.1 pgoyette there is a read or write failure on the foo.gz file */
462 1.1.1.1.76.1 pgoyette local int log_last(struct log *log, int last)
463 1.1 christos {
464 1.1.1.1.76.1 pgoyette int back, len, mask;
465 1.1.1.1.76.1 pgoyette unsigned char buf[6];
466 1.1 christos
467 1.1.1.1.76.1 pgoyette /* determine the locations of the bytes and bits to modify */
468 1.1.1.1.76.1 pgoyette back = log->last == log->first ? log->back : 8;
469 1.1.1.1.76.1 pgoyette len = back > 8 ? 2 : 1; /* bytes back from log->last */
470 1.1.1.1.76.1 pgoyette mask = 0x80 >> ((back - 1) & 7); /* mask for block last-bit */
471 1.1.1.1.76.1 pgoyette
472 1.1.1.1.76.1 pgoyette /* get the byte to modify (one or two back) into buf[0] -- don't need to
473 1.1.1.1.76.1 pgoyette read the byte if the last-bit is eight bits back, since in that case
474 1.1.1.1.76.1 pgoyette the entire byte will be modified */
475 1.1.1.1.76.1 pgoyette buf[0] = 0;
476 1.1.1.1.76.1 pgoyette if (back != 8 && (lseek(log->fd, log->last - len, SEEK_SET) < 0 ||
477 1.1.1.1.76.1 pgoyette read(log->fd, buf, 1) != 1))
478 1.1.1.1.76.1 pgoyette return -1;
479 1.1.1.1.76.1 pgoyette
480 1.1.1.1.76.1 pgoyette /* change the last-bit of the last stored block as requested -- note
481 1.1.1.1.76.1 pgoyette that all bits above the last-bit are set to zero, per the type bits
482 1.1.1.1.76.1 pgoyette of a stored block being 00 and per the convention that the bits to
483 1.1.1.1.76.1 pgoyette bring the stream to a byte boundary are also zeros */
484 1.1.1.1.76.1 pgoyette buf[1] = 0;
485 1.1.1.1.76.1 pgoyette buf[2 - len] = (*buf & (mask - 1)) + (last ? mask : 0);
486 1.1.1.1.76.1 pgoyette
487 1.1.1.1.76.1 pgoyette /* write the modified stored block header and lengths, move the file
488 1.1.1.1.76.1 pgoyette pointer to after the last stored block data */
489 1.1.1.1.76.1 pgoyette PUT2(buf + 2, log->stored);
490 1.1.1.1.76.1 pgoyette PUT2(buf + 4, log->stored ^ 0xffff);
491 1.1.1.1.76.1 pgoyette return lseek(log->fd, log->last - len, SEEK_SET) < 0 ||
492 1.1.1.1.76.1 pgoyette write(log->fd, buf + 2 - len, len + 4) != len + 4 ||
493 1.1.1.1.76.1 pgoyette lseek(log->fd, log->stored, SEEK_CUR) < 0 ? -1 : 0;
494 1.1 christos }
495 1.1 christos
496 1.1.1.1.76.1 pgoyette /* Append len bytes from data to the locked and open log file. len may be zero
497 1.1.1.1.76.1 pgoyette if recovering and no .add file was found. In that case, the previous state
498 1.1.1.1.76.1 pgoyette of the foo.gz file is restored. The data is appended uncompressed in
499 1.1.1.1.76.1 pgoyette deflate stored blocks. Return -1 if there was an error reading or writing
500 1.1.1.1.76.1 pgoyette the foo.gz file. */
501 1.1.1.1.76.1 pgoyette local int log_append(struct log *log, unsigned char *data, size_t len)
502 1.1 christos {
503 1.1.1.1.76.1 pgoyette uint put;
504 1.1.1.1.76.1 pgoyette off_t end;
505 1.1.1.1.76.1 pgoyette unsigned char buf[8];
506 1.1.1.1.76.1 pgoyette
507 1.1.1.1.76.1 pgoyette /* set the last block last-bit and length, in case recovering an
508 1.1.1.1.76.1 pgoyette interrupted append, then position the file pointer to append to the
509 1.1.1.1.76.1 pgoyette block */
510 1.1.1.1.76.1 pgoyette if (log_last(log, 1))
511 1.1.1.1.76.1 pgoyette return -1;
512 1.1.1.1.76.1 pgoyette
513 1.1.1.1.76.1 pgoyette /* append, adding stored blocks and updating the offset of the last stored
514 1.1.1.1.76.1 pgoyette block as needed, and update the total crc and length */
515 1.1.1.1.76.1 pgoyette while (len) {
516 1.1.1.1.76.1 pgoyette /* append as much as we can to the last block */
517 1.1.1.1.76.1 pgoyette put = (MAX_STORE << 10) - log->stored;
518 1.1.1.1.76.1 pgoyette if (put > len)
519 1.1.1.1.76.1 pgoyette put = (uint)len;
520 1.1.1.1.76.1 pgoyette if (put) {
521 1.1.1.1.76.1 pgoyette if (write(log->fd, data, put) != put)
522 1.1.1.1.76.1 pgoyette return -1;
523 1.1.1.1.76.1 pgoyette BAIL(1);
524 1.1.1.1.76.1 pgoyette log->tcrc = crc32(log->tcrc, data, put);
525 1.1.1.1.76.1 pgoyette log->tlen += put;
526 1.1.1.1.76.1 pgoyette log->stored += put;
527 1.1.1.1.76.1 pgoyette data += put;
528 1.1.1.1.76.1 pgoyette len -= put;
529 1.1.1.1.76.1 pgoyette }
530 1.1.1.1.76.1 pgoyette
531 1.1.1.1.76.1 pgoyette /* if we need to, add a new empty stored block */
532 1.1.1.1.76.1 pgoyette if (len) {
533 1.1.1.1.76.1 pgoyette /* mark current block as not last */
534 1.1.1.1.76.1 pgoyette if (log_last(log, 0))
535 1.1.1.1.76.1 pgoyette return -1;
536 1.1 christos
537 1.1.1.1.76.1 pgoyette /* point to new, empty stored block */
538 1.1.1.1.76.1 pgoyette log->last += 4 + log->stored + 1;
539 1.1.1.1.76.1 pgoyette log->stored = 0;
540 1.1.1.1.76.1 pgoyette }
541 1.1.1.1.76.1 pgoyette
542 1.1.1.1.76.1 pgoyette /* mark last block as last, update its length */
543 1.1.1.1.76.1 pgoyette if (log_last(log, 1))
544 1.1.1.1.76.1 pgoyette return -1;
545 1.1.1.1.76.1 pgoyette BAIL(2);
546 1.1 christos }
547 1.1 christos
548 1.1.1.1.76.1 pgoyette /* write the new crc and length trailer, and truncate just in case (could
549 1.1.1.1.76.1 pgoyette be recovering from partial append with a missing foo.add file) */
550 1.1.1.1.76.1 pgoyette PUT4(buf, log->tcrc);
551 1.1.1.1.76.1 pgoyette PUT4(buf + 4, log->tlen);
552 1.1.1.1.76.1 pgoyette if (write(log->fd, buf, 8) != 8 ||
553 1.1.1.1.76.1 pgoyette (end = lseek(log->fd, 0, SEEK_CUR)) < 0 || ftruncate(log->fd, end))
554 1.1.1.1.76.1 pgoyette return -1;
555 1.1.1.1.76.1 pgoyette
556 1.1.1.1.76.1 pgoyette /* write the extra field, marking the log file as done, delete .add file */
557 1.1.1.1.76.1 pgoyette if (log_mark(log, NO_OP))
558 1.1.1.1.76.1 pgoyette return -1;
559 1.1.1.1.76.1 pgoyette strcpy(log->end, ".add");
560 1.1.1.1.76.1 pgoyette unlink(log->path); /* ignore error, since may not exist */
561 1.1.1.1.76.1 pgoyette return 0;
562 1.1.1.1.76.1 pgoyette }
563 1.1 christos
564 1.1.1.1.76.1 pgoyette /* Replace the foo.dict file with the foo.temp file. Also delete the foo.add
565 1.1.1.1.76.1 pgoyette file, since the compress operation may have been interrupted before that was
566 1.1.1.1.76.1 pgoyette done. Returns 1 if memory could not be allocated, or -1 if reading or
567 1.1.1.1.76.1 pgoyette writing foo.gz fails, or if the rename fails for some reason other than
568 1.1.1.1.76.1 pgoyette foo.temp not existing. foo.temp not existing is a permitted error, since
569 1.1.1.1.76.1 pgoyette the replace operation may have been interrupted after the rename is done,
570 1.1.1.1.76.1 pgoyette but before foo.gz is marked as complete. */
571 1.1.1.1.76.1 pgoyette local int log_replace(struct log *log)
572 1.1 christos {
573 1.1.1.1.76.1 pgoyette int ret;
574 1.1.1.1.76.1 pgoyette char *dest;
575 1.1 christos
576 1.1.1.1.76.1 pgoyette /* delete foo.add file */
577 1.1.1.1.76.1 pgoyette strcpy(log->end, ".add");
578 1.1.1.1.76.1 pgoyette unlink(log->path); /* ignore error, since may not exist */
579 1.1.1.1.76.1 pgoyette BAIL(3);
580 1.1.1.1.76.1 pgoyette
581 1.1.1.1.76.1 pgoyette /* rename foo.name to foo.dict, replacing foo.dict if it exists */
582 1.1.1.1.76.1 pgoyette strcpy(log->end, ".dict");
583 1.1.1.1.76.1 pgoyette dest = malloc(strlen(log->path) + 1);
584 1.1.1.1.76.1 pgoyette if (dest == NULL)
585 1.1.1.1.76.1 pgoyette return -2;
586 1.1.1.1.76.1 pgoyette strcpy(dest, log->path);
587 1.1.1.1.76.1 pgoyette strcpy(log->end, ".temp");
588 1.1.1.1.76.1 pgoyette ret = rename(log->path, dest);
589 1.1.1.1.76.1 pgoyette free(dest);
590 1.1.1.1.76.1 pgoyette if (ret && errno != ENOENT)
591 1.1.1.1.76.1 pgoyette return -1;
592 1.1.1.1.76.1 pgoyette BAIL(4);
593 1.1 christos
594 1.1.1.1.76.1 pgoyette /* mark the foo.gz file as done */
595 1.1.1.1.76.1 pgoyette return log_mark(log, NO_OP);
596 1.1.1.1.76.1 pgoyette }
597 1.1 christos
598 1.1.1.1.76.1 pgoyette /* Compress the len bytes at data and append the compressed data to the
599 1.1.1.1.76.1 pgoyette foo.gz deflate data immediately after the previous compressed data. This
600 1.1.1.1.76.1 pgoyette overwrites the previous uncompressed data, which was stored in foo.add
601 1.1.1.1.76.1 pgoyette and is the data provided in data[0..len-1]. If this operation is
602 1.1.1.1.76.1 pgoyette interrupted, it picks up at the start of this routine, with the foo.add
603 1.1.1.1.76.1 pgoyette file read in again. If there is no data to compress (len == 0), then we
604 1.1.1.1.76.1 pgoyette simply terminate the foo.gz file after the previously compressed data,
605 1.1.1.1.76.1 pgoyette appending a final empty stored block and the gzip trailer. Return -1 if
606 1.1.1.1.76.1 pgoyette reading or writing the log.gz file failed, or -2 if there was a memory
607 1.1.1.1.76.1 pgoyette allocation failure. */
608 1.1.1.1.76.1 pgoyette local int log_compress(struct log *log, unsigned char *data, size_t len)
609 1.1.1.1.76.1 pgoyette {
610 1.1.1.1.76.1 pgoyette int fd;
611 1.1.1.1.76.1 pgoyette uint got, max;
612 1.1.1.1.76.1 pgoyette ssize_t dict;
613 1.1.1.1.76.1 pgoyette off_t end;
614 1.1.1.1.76.1 pgoyette z_stream strm;
615 1.1.1.1.76.1 pgoyette unsigned char buf[DICT];
616 1.1.1.1.76.1 pgoyette
617 1.1.1.1.76.1 pgoyette /* compress and append compressed data */
618 1.1.1.1.76.1 pgoyette if (len) {
619 1.1.1.1.76.1 pgoyette /* set up for deflate, allocating memory */
620 1.1.1.1.76.1 pgoyette strm.zalloc = Z_NULL;
621 1.1.1.1.76.1 pgoyette strm.zfree = Z_NULL;
622 1.1.1.1.76.1 pgoyette strm.opaque = Z_NULL;
623 1.1.1.1.76.1 pgoyette if (deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 8,
624 1.1.1.1.76.1 pgoyette Z_DEFAULT_STRATEGY) != Z_OK)
625 1.1.1.1.76.1 pgoyette return -2;
626 1.1.1.1.76.1 pgoyette
627 1.1.1.1.76.1 pgoyette /* read in dictionary (last 32K of data that was compressed) */
628 1.1.1.1.76.1 pgoyette strcpy(log->end, ".dict");
629 1.1.1.1.76.1 pgoyette fd = open(log->path, O_RDONLY, 0);
630 1.1.1.1.76.1 pgoyette if (fd >= 0) {
631 1.1.1.1.76.1 pgoyette dict = read(fd, buf, DICT);
632 1.1.1.1.76.1 pgoyette close(fd);
633 1.1.1.1.76.1 pgoyette if (dict < 0) {
634 1.1.1.1.76.1 pgoyette deflateEnd(&strm);
635 1.1.1.1.76.1 pgoyette return -1;
636 1.1.1.1.76.1 pgoyette }
637 1.1.1.1.76.1 pgoyette if (dict)
638 1.1.1.1.76.1 pgoyette deflateSetDictionary(&strm, buf, (uint)dict);
639 1.1 christos }
640 1.1.1.1.76.1 pgoyette log_touch(log);
641 1.1 christos
642 1.1.1.1.76.1 pgoyette /* prime deflate with last bits of previous block, position write
643 1.1.1.1.76.1 pgoyette pointer to write those bits and overwrite what follows */
644 1.1.1.1.76.1 pgoyette if (lseek(log->fd, log->first - (log->back > 8 ? 2 : 1),
645 1.1.1.1.76.1 pgoyette SEEK_SET) < 0 ||
646 1.1.1.1.76.1 pgoyette read(log->fd, buf, 1) != 1 || lseek(log->fd, -1, SEEK_CUR) < 0) {
647 1.1.1.1.76.1 pgoyette deflateEnd(&strm);
648 1.1.1.1.76.1 pgoyette return -1;
649 1.1.1.1.76.1 pgoyette }
650 1.1.1.1.76.1 pgoyette deflatePrime(&strm, (8 - log->back) & 7, *buf);
651 1.1 christos
652 1.1.1.1.76.1 pgoyette /* compress, finishing with a partial non-last empty static block */
653 1.1.1.1.76.1 pgoyette strm.next_in = data;
654 1.1.1.1.76.1 pgoyette max = (((uint)0 - 1) >> 1) + 1; /* in case int smaller than size_t */
655 1.1.1.1.76.1 pgoyette do {
656 1.1.1.1.76.1 pgoyette strm.avail_in = len > max ? max : (uint)len;
657 1.1.1.1.76.1 pgoyette len -= strm.avail_in;
658 1.1.1.1.76.1 pgoyette do {
659 1.1.1.1.76.1 pgoyette strm.avail_out = DICT;
660 1.1.1.1.76.1 pgoyette strm.next_out = buf;
661 1.1.1.1.76.1 pgoyette deflate(&strm, len ? Z_NO_FLUSH : Z_PARTIAL_FLUSH);
662 1.1.1.1.76.1 pgoyette got = DICT - strm.avail_out;
663 1.1.1.1.76.1 pgoyette if (got && write(log->fd, buf, got) != got) {
664 1.1.1.1.76.1 pgoyette deflateEnd(&strm);
665 1.1.1.1.76.1 pgoyette return -1;
666 1.1.1.1.76.1 pgoyette }
667 1.1.1.1.76.1 pgoyette log_touch(log);
668 1.1.1.1.76.1 pgoyette } while (strm.avail_out == 0);
669 1.1.1.1.76.1 pgoyette } while (len);
670 1.1.1.1.76.1 pgoyette deflateEnd(&strm);
671 1.1.1.1.76.1 pgoyette BAIL(5);
672 1.1.1.1.76.1 pgoyette
673 1.1.1.1.76.1 pgoyette /* find start of empty static block -- scanning backwards the first one
674 1.1.1.1.76.1 pgoyette bit is the second bit of the block, if the last byte is zero, then
675 1.1.1.1.76.1 pgoyette we know the byte before that has a one in the top bit, since an
676 1.1.1.1.76.1 pgoyette empty static block is ten bits long */
677 1.1.1.1.76.1 pgoyette if ((log->first = lseek(log->fd, -1, SEEK_CUR)) < 0 ||
678 1.1.1.1.76.1 pgoyette read(log->fd, buf, 1) != 1)
679 1.1.1.1.76.1 pgoyette return -1;
680 1.1.1.1.76.1 pgoyette log->first++;
681 1.1.1.1.76.1 pgoyette if (*buf) {
682 1.1.1.1.76.1 pgoyette log->back = 1;
683 1.1.1.1.76.1 pgoyette while ((*buf & ((uint)1 << (8 - log->back++))) == 0)
684 1.1.1.1.76.1 pgoyette ; /* guaranteed to terminate, since *buf != 0 */
685 1.1 christos }
686 1.1.1.1.76.1 pgoyette else
687 1.1.1.1.76.1 pgoyette log->back = 10;
688 1.1.1.1.76.1 pgoyette
689 1.1.1.1.76.1 pgoyette /* update compressed crc and length */
690 1.1.1.1.76.1 pgoyette log->ccrc = log->tcrc;
691 1.1.1.1.76.1 pgoyette log->clen = log->tlen;
692 1.1 christos }
693 1.1.1.1.76.1 pgoyette else {
694 1.1.1.1.76.1 pgoyette /* no data to compress -- fix up existing gzip stream */
695 1.1.1.1.76.1 pgoyette log->tcrc = log->ccrc;
696 1.1.1.1.76.1 pgoyette log->tlen = log->clen;
697 1.1 christos }
698 1.1 christos
699 1.1.1.1.76.1 pgoyette /* complete and truncate gzip stream */
700 1.1.1.1.76.1 pgoyette log->last = log->first;
701 1.1.1.1.76.1 pgoyette log->stored = 0;
702 1.1.1.1.76.1 pgoyette PUT4(buf, log->tcrc);
703 1.1.1.1.76.1 pgoyette PUT4(buf + 4, log->tlen);
704 1.1.1.1.76.1 pgoyette if (log_last(log, 1) || write(log->fd, buf, 8) != 8 ||
705 1.1.1.1.76.1 pgoyette (end = lseek(log->fd, 0, SEEK_CUR)) < 0 || ftruncate(log->fd, end))
706 1.1.1.1.76.1 pgoyette return -1;
707 1.1.1.1.76.1 pgoyette BAIL(6);
708 1.1.1.1.76.1 pgoyette
709 1.1.1.1.76.1 pgoyette /* mark as being in the replace operation */
710 1.1.1.1.76.1 pgoyette if (log_mark(log, REPLACE_OP))
711 1.1.1.1.76.1 pgoyette return -1;
712 1.1.1.1.76.1 pgoyette
713 1.1.1.1.76.1 pgoyette /* execute the replace operation and mark the file as done */
714 1.1.1.1.76.1 pgoyette return log_replace(log);
715 1.1.1.1.76.1 pgoyette }
716 1.1.1.1.76.1 pgoyette
717 1.1.1.1.76.1 pgoyette /* log a repair record to the .repairs file */
718 1.1.1.1.76.1 pgoyette local void log_log(struct log *log, int op, char *record)
719 1.1.1.1.76.1 pgoyette {
720 1.1.1.1.76.1 pgoyette time_t now;
721 1.1.1.1.76.1 pgoyette FILE *rec;
722 1.1.1.1.76.1 pgoyette
723 1.1.1.1.76.1 pgoyette now = time(NULL);
724 1.1.1.1.76.1 pgoyette strcpy(log->end, ".repairs");
725 1.1.1.1.76.1 pgoyette rec = fopen(log->path, "a");
726 1.1.1.1.76.1 pgoyette if (rec == NULL)
727 1.1.1.1.76.1 pgoyette return;
728 1.1.1.1.76.1 pgoyette fprintf(rec, "%.24s %s recovery: %s\n", ctime(&now), op == APPEND_OP ?
729 1.1.1.1.76.1 pgoyette "append" : (op == COMPRESS_OP ? "compress" : "replace"), record);
730 1.1.1.1.76.1 pgoyette fclose(rec);
731 1.1.1.1.76.1 pgoyette return;
732 1.1.1.1.76.1 pgoyette }
733 1.1.1.1.76.1 pgoyette
734 1.1.1.1.76.1 pgoyette /* Recover the interrupted operation op. First read foo.add for recovering an
735 1.1.1.1.76.1 pgoyette append or compress operation. Return -1 if there was an error reading or
736 1.1.1.1.76.1 pgoyette writing foo.gz or reading an existing foo.add, or -2 if there was a memory
737 1.1.1.1.76.1 pgoyette allocation failure. */
738 1.1.1.1.76.1 pgoyette local int log_recover(struct log *log, int op)
739 1.1.1.1.76.1 pgoyette {
740 1.1.1.1.76.1 pgoyette int fd, ret = 0;
741 1.1.1.1.76.1 pgoyette unsigned char *data = NULL;
742 1.1.1.1.76.1 pgoyette size_t len = 0;
743 1.1.1.1.76.1 pgoyette struct stat st;
744 1.1.1.1.76.1 pgoyette
745 1.1.1.1.76.1 pgoyette /* log recovery */
746 1.1.1.1.76.1 pgoyette log_log(log, op, "start");
747 1.1.1.1.76.1 pgoyette
748 1.1.1.1.76.1 pgoyette /* load foo.add file if expected and present */
749 1.1.1.1.76.1 pgoyette if (op == APPEND_OP || op == COMPRESS_OP) {
750 1.1.1.1.76.1 pgoyette strcpy(log->end, ".add");
751 1.1.1.1.76.1 pgoyette if (stat(log->path, &st) == 0 && st.st_size) {
752 1.1.1.1.76.1 pgoyette len = (size_t)(st.st_size);
753 1.1.1.1.76.1 pgoyette if ((off_t)len != st.st_size ||
754 1.1.1.1.76.1 pgoyette (data = malloc(st.st_size)) == NULL) {
755 1.1.1.1.76.1 pgoyette log_log(log, op, "allocation failure");
756 1.1.1.1.76.1 pgoyette return -2;
757 1.1.1.1.76.1 pgoyette }
758 1.1.1.1.76.1 pgoyette if ((fd = open(log->path, O_RDONLY, 0)) < 0) {
759 1.1.1.1.76.1 pgoyette log_log(log, op, ".add file read failure");
760 1.1.1.1.76.1 pgoyette return -1;
761 1.1.1.1.76.1 pgoyette }
762 1.1.1.1.76.1 pgoyette ret = (size_t)read(fd, data, len) != len;
763 1.1.1.1.76.1 pgoyette close(fd);
764 1.1.1.1.76.1 pgoyette if (ret) {
765 1.1.1.1.76.1 pgoyette log_log(log, op, ".add file read failure");
766 1.1.1.1.76.1 pgoyette return -1;
767 1.1.1.1.76.1 pgoyette }
768 1.1.1.1.76.1 pgoyette log_log(log, op, "loaded .add file");
769 1.1.1.1.76.1 pgoyette }
770 1.1.1.1.76.1 pgoyette else
771 1.1.1.1.76.1 pgoyette log_log(log, op, "missing .add file!");
772 1.1 christos }
773 1.1 christos
774 1.1.1.1.76.1 pgoyette /* recover the interrupted operation */
775 1.1.1.1.76.1 pgoyette switch (op) {
776 1.1.1.1.76.1 pgoyette case APPEND_OP:
777 1.1.1.1.76.1 pgoyette ret = log_append(log, data, len);
778 1.1.1.1.76.1 pgoyette break;
779 1.1.1.1.76.1 pgoyette case COMPRESS_OP:
780 1.1.1.1.76.1 pgoyette ret = log_compress(log, data, len);
781 1.1.1.1.76.1 pgoyette break;
782 1.1.1.1.76.1 pgoyette case REPLACE_OP:
783 1.1.1.1.76.1 pgoyette ret = log_replace(log);
784 1.1 christos }
785 1.1 christos
786 1.1.1.1.76.1 pgoyette /* log status */
787 1.1.1.1.76.1 pgoyette log_log(log, op, ret ? "failure" : "complete");
788 1.1.1.1.76.1 pgoyette
789 1.1.1.1.76.1 pgoyette /* clean up */
790 1.1.1.1.76.1 pgoyette if (data != NULL)
791 1.1.1.1.76.1 pgoyette free(data);
792 1.1.1.1.76.1 pgoyette return ret;
793 1.1 christos }
794 1.1 christos
795 1.1.1.1.76.1 pgoyette /* Close the foo.gz file (if open) and release the lock. */
796 1.1.1.1.76.1 pgoyette local void log_close(struct log *log)
797 1.1.1.1.76.1 pgoyette {
798 1.1.1.1.76.1 pgoyette if (log->fd >= 0)
799 1.1.1.1.76.1 pgoyette close(log->fd);
800 1.1.1.1.76.1 pgoyette log->fd = -1;
801 1.1.1.1.76.1 pgoyette log_unlock(log);
802 1.1.1.1.76.1 pgoyette }
803 1.1 christos
804 1.1.1.1.76.1 pgoyette /* Open foo.gz, verify the header, and load the extra field contents, after
805 1.1.1.1.76.1 pgoyette first creating the foo.lock file to gain exclusive access to the foo.*
806 1.1.1.1.76.1 pgoyette files. If foo.gz does not exist or is empty, then write the initial header,
807 1.1.1.1.76.1 pgoyette extra, and body content of an empty foo.gz log file. If there is an error
808 1.1.1.1.76.1 pgoyette creating the lock file due to access restrictions, or an error reading or
809 1.1.1.1.76.1 pgoyette writing the foo.gz file, or if the foo.gz file is not a proper log file for
810 1.1.1.1.76.1 pgoyette this object (e.g. not a gzip file or does not contain the expected extra
811 1.1.1.1.76.1 pgoyette field), then return true. If there is an error, the lock is released.
812 1.1.1.1.76.1 pgoyette Otherwise, the lock is left in place. */
813 1.1.1.1.76.1 pgoyette local int log_open(struct log *log)
814 1.1 christos {
815 1.1.1.1.76.1 pgoyette int op;
816 1.1 christos
817 1.1.1.1.76.1 pgoyette /* release open file resource if left over -- can occur if lock lost
818 1.1.1.1.76.1 pgoyette between gzlog_open() and gzlog_write() */
819 1.1.1.1.76.1 pgoyette if (log->fd >= 0)
820 1.1.1.1.76.1 pgoyette close(log->fd);
821 1.1.1.1.76.1 pgoyette log->fd = -1;
822 1.1 christos
823 1.1.1.1.76.1 pgoyette /* negotiate exclusive access */
824 1.1.1.1.76.1 pgoyette if (log_lock(log) < 0)
825 1.1.1.1.76.1 pgoyette return -1;
826 1.1.1.1.76.1 pgoyette
827 1.1.1.1.76.1 pgoyette /* open the log file, foo.gz */
828 1.1.1.1.76.1 pgoyette strcpy(log->end, ".gz");
829 1.1.1.1.76.1 pgoyette log->fd = open(log->path, O_RDWR | O_CREAT, 0644);
830 1.1.1.1.76.1 pgoyette if (log->fd < 0) {
831 1.1.1.1.76.1 pgoyette log_close(log);
832 1.1.1.1.76.1 pgoyette return -1;
833 1.1.1.1.76.1 pgoyette }
834 1.1.1.1.76.1 pgoyette
835 1.1.1.1.76.1 pgoyette /* if new, initialize foo.gz with an empty log, delete old dictionary */
836 1.1.1.1.76.1 pgoyette if (lseek(log->fd, 0, SEEK_END) == 0) {
837 1.1.1.1.76.1 pgoyette if (write(log->fd, log_gzhead, HEAD) != HEAD ||
838 1.1.1.1.76.1 pgoyette write(log->fd, log_gzext, EXTRA) != EXTRA ||
839 1.1.1.1.76.1 pgoyette write(log->fd, log_gzbody, BODY) != BODY) {
840 1.1.1.1.76.1 pgoyette log_close(log);
841 1.1.1.1.76.1 pgoyette return -1;
842 1.1 christos }
843 1.1.1.1.76.1 pgoyette strcpy(log->end, ".dict");
844 1.1.1.1.76.1 pgoyette unlink(log->path);
845 1.1.1.1.76.1 pgoyette }
846 1.1.1.1.76.1 pgoyette
847 1.1.1.1.76.1 pgoyette /* verify log file and load extra field information */
848 1.1.1.1.76.1 pgoyette if ((op = log_head(log)) < 0) {
849 1.1.1.1.76.1 pgoyette log_close(log);
850 1.1.1.1.76.1 pgoyette return -1;
851 1.1.1.1.76.1 pgoyette }
852 1.1.1.1.76.1 pgoyette
853 1.1.1.1.76.1 pgoyette /* check for interrupted process and if so, recover */
854 1.1.1.1.76.1 pgoyette if (op != NO_OP && log_recover(log, op)) {
855 1.1.1.1.76.1 pgoyette log_close(log);
856 1.1.1.1.76.1 pgoyette return -1;
857 1.1.1.1.76.1 pgoyette }
858 1.1.1.1.76.1 pgoyette
859 1.1.1.1.76.1 pgoyette /* touch the lock file to prevent another process from grabbing it */
860 1.1.1.1.76.1 pgoyette log_touch(log);
861 1.1 christos return 0;
862 1.1 christos }
863 1.1 christos
864 1.1.1.1.76.1 pgoyette /* See gzlog.h for the description of the external methods below */
865 1.1.1.1.76.1 pgoyette gzlog *gzlog_open(char *path)
866 1.1 christos {
867 1.1.1.1.76.1 pgoyette size_t n;
868 1.1.1.1.76.1 pgoyette struct log *log;
869 1.1.1.1.76.1 pgoyette
870 1.1.1.1.76.1 pgoyette /* check arguments */
871 1.1.1.1.76.1 pgoyette if (path == NULL || *path == 0)
872 1.1.1.1.76.1 pgoyette return NULL;
873 1.1.1.1.76.1 pgoyette
874 1.1.1.1.76.1 pgoyette /* allocate and initialize log structure */
875 1.1.1.1.76.1 pgoyette log = malloc(sizeof(struct log));
876 1.1.1.1.76.1 pgoyette if (log == NULL)
877 1.1.1.1.76.1 pgoyette return NULL;
878 1.1.1.1.76.1 pgoyette strcpy(log->id, LOGID);
879 1.1.1.1.76.1 pgoyette log->fd = -1;
880 1.1.1.1.76.1 pgoyette
881 1.1.1.1.76.1 pgoyette /* save path and end of path for name construction */
882 1.1.1.1.76.1 pgoyette n = strlen(path);
883 1.1.1.1.76.1 pgoyette log->path = malloc(n + 9); /* allow for ".repairs" */
884 1.1.1.1.76.1 pgoyette if (log->path == NULL) {
885 1.1.1.1.76.1 pgoyette free(log);
886 1.1.1.1.76.1 pgoyette return NULL;
887 1.1 christos }
888 1.1.1.1.76.1 pgoyette strcpy(log->path, path);
889 1.1.1.1.76.1 pgoyette log->end = log->path + n;
890 1.1 christos
891 1.1.1.1.76.1 pgoyette /* gain exclusive access and verify log file -- may perform a
892 1.1.1.1.76.1 pgoyette recovery operation if needed */
893 1.1.1.1.76.1 pgoyette if (log_open(log)) {
894 1.1.1.1.76.1 pgoyette free(log->path);
895 1.1.1.1.76.1 pgoyette free(log);
896 1.1.1.1.76.1 pgoyette return NULL;
897 1.1 christos }
898 1.1.1.1.76.1 pgoyette
899 1.1.1.1.76.1 pgoyette /* return pointer to log structure */
900 1.1.1.1.76.1 pgoyette return log;
901 1.1.1.1.76.1 pgoyette }
902 1.1.1.1.76.1 pgoyette
903 1.1.1.1.76.1 pgoyette /* gzlog_compress() return values:
904 1.1.1.1.76.1 pgoyette 0: all good
905 1.1.1.1.76.1 pgoyette -1: file i/o error (usually access issue)
906 1.1.1.1.76.1 pgoyette -2: memory allocation failure
907 1.1.1.1.76.1 pgoyette -3: invalid log pointer argument */
908 1.1.1.1.76.1 pgoyette int gzlog_compress(gzlog *logd)
909 1.1.1.1.76.1 pgoyette {
910 1.1.1.1.76.1 pgoyette int fd, ret;
911 1.1.1.1.76.1 pgoyette uint block;
912 1.1.1.1.76.1 pgoyette size_t len, next;
913 1.1.1.1.76.1 pgoyette unsigned char *data, buf[5];
914 1.1.1.1.76.1 pgoyette struct log *log = logd;
915 1.1.1.1.76.1 pgoyette
916 1.1.1.1.76.1 pgoyette /* check arguments */
917 1.1.1.1.76.1 pgoyette if (log == NULL || strcmp(log->id, LOGID))
918 1.1.1.1.76.1 pgoyette return -3;
919 1.1.1.1.76.1 pgoyette
920 1.1.1.1.76.1 pgoyette /* see if we lost the lock -- if so get it again and reload the extra
921 1.1.1.1.76.1 pgoyette field information (it probably changed), recover last operation if
922 1.1.1.1.76.1 pgoyette necessary */
923 1.1.1.1.76.1 pgoyette if (log_check(log) && log_open(log))
924 1.1.1.1.76.1 pgoyette return -1;
925 1.1.1.1.76.1 pgoyette
926 1.1.1.1.76.1 pgoyette /* create space for uncompressed data */
927 1.1.1.1.76.1 pgoyette len = ((size_t)(log->last - log->first) & ~(((size_t)1 << 10) - 1)) +
928 1.1.1.1.76.1 pgoyette log->stored;
929 1.1.1.1.76.1 pgoyette if ((data = malloc(len)) == NULL)
930 1.1.1.1.76.1 pgoyette return -2;
931 1.1.1.1.76.1 pgoyette
932 1.1.1.1.76.1 pgoyette /* do statement here is just a cheap trick for error handling */
933 1.1.1.1.76.1 pgoyette do {
934 1.1.1.1.76.1 pgoyette /* read in the uncompressed data */
935 1.1.1.1.76.1 pgoyette if (lseek(log->fd, log->first - 1, SEEK_SET) < 0)
936 1.1 christos break;
937 1.1.1.1.76.1 pgoyette next = 0;
938 1.1.1.1.76.1 pgoyette while (next < len) {
939 1.1.1.1.76.1 pgoyette if (read(log->fd, buf, 5) != 5)
940 1.1 christos break;
941 1.1.1.1.76.1 pgoyette block = PULL2(buf + 1);
942 1.1.1.1.76.1 pgoyette if (next + block > len ||
943 1.1.1.1.76.1 pgoyette read(log->fd, (char *)data + next, block) != block)
944 1.1 christos break;
945 1.1.1.1.76.1 pgoyette next += block;
946 1.1 christos }
947 1.1.1.1.76.1 pgoyette if (lseek(log->fd, 0, SEEK_CUR) != log->last + 4 + log->stored)
948 1.1.1.1.76.1 pgoyette break;
949 1.1.1.1.76.1 pgoyette log_touch(log);
950 1.1 christos
951 1.1.1.1.76.1 pgoyette /* write the uncompressed data to the .add file */
952 1.1.1.1.76.1 pgoyette strcpy(log->end, ".add");
953 1.1.1.1.76.1 pgoyette fd = open(log->path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
954 1.1.1.1.76.1 pgoyette if (fd < 0)
955 1.1.1.1.76.1 pgoyette break;
956 1.1.1.1.76.1 pgoyette ret = (size_t)write(fd, data, len) != len;
957 1.1.1.1.76.1 pgoyette if (ret | close(fd))
958 1.1.1.1.76.1 pgoyette break;
959 1.1.1.1.76.1 pgoyette log_touch(log);
960 1.1 christos
961 1.1.1.1.76.1 pgoyette /* write the dictionary for the next compress to the .temp file */
962 1.1.1.1.76.1 pgoyette strcpy(log->end, ".temp");
963 1.1.1.1.76.1 pgoyette fd = open(log->path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
964 1.1.1.1.76.1 pgoyette if (fd < 0)
965 1.1.1.1.76.1 pgoyette break;
966 1.1.1.1.76.1 pgoyette next = DICT > len ? len : DICT;
967 1.1.1.1.76.1 pgoyette ret = (size_t)write(fd, (char *)data + len - next, next) != next;
968 1.1.1.1.76.1 pgoyette if (ret | close(fd))
969 1.1.1.1.76.1 pgoyette break;
970 1.1.1.1.76.1 pgoyette log_touch(log);
971 1.1 christos
972 1.1.1.1.76.1 pgoyette /* roll back to compressed data, mark the compress in progress */
973 1.1.1.1.76.1 pgoyette log->last = log->first;
974 1.1.1.1.76.1 pgoyette log->stored = 0;
975 1.1.1.1.76.1 pgoyette if (log_mark(log, COMPRESS_OP))
976 1.1.1.1.76.1 pgoyette break;
977 1.1.1.1.76.1 pgoyette BAIL(7);
978 1.1 christos
979 1.1.1.1.76.1 pgoyette /* compress and append the data (clears mark) */
980 1.1.1.1.76.1 pgoyette ret = log_compress(log, data, len);
981 1.1.1.1.76.1 pgoyette free(data);
982 1.1.1.1.76.1 pgoyette return ret;
983 1.1.1.1.76.1 pgoyette } while (0);
984 1.1 christos
985 1.1.1.1.76.1 pgoyette /* broke out of do above on i/o error */
986 1.1.1.1.76.1 pgoyette free(data);
987 1.1.1.1.76.1 pgoyette return -1;
988 1.1.1.1.76.1 pgoyette }
989 1.1 christos
990 1.1.1.1.76.1 pgoyette /* gzlog_write() return values:
991 1.1.1.1.76.1 pgoyette 0: all good
992 1.1.1.1.76.1 pgoyette -1: file i/o error (usually access issue)
993 1.1.1.1.76.1 pgoyette -2: memory allocation failure
994 1.1.1.1.76.1 pgoyette -3: invalid log pointer argument */
995 1.1.1.1.76.1 pgoyette int gzlog_write(gzlog *logd, void *data, size_t len)
996 1.1 christos {
997 1.1.1.1.76.1 pgoyette int fd, ret;
998 1.1.1.1.76.1 pgoyette struct log *log = logd;
999 1.1 christos
1000 1.1.1.1.76.1 pgoyette /* check arguments */
1001 1.1.1.1.76.1 pgoyette if (log == NULL || strcmp(log->id, LOGID))
1002 1.1.1.1.76.1 pgoyette return -3;
1003 1.1.1.1.76.1 pgoyette if (data == NULL || len <= 0)
1004 1.1.1.1.76.1 pgoyette return 0;
1005 1.1.1.1.76.1 pgoyette
1006 1.1.1.1.76.1 pgoyette /* see if we lost the lock -- if so get it again and reload the extra
1007 1.1.1.1.76.1 pgoyette field information (it probably changed), recover last operation if
1008 1.1.1.1.76.1 pgoyette necessary */
1009 1.1.1.1.76.1 pgoyette if (log_check(log) && log_open(log))
1010 1.1.1.1.76.1 pgoyette return -1;
1011 1.1.1.1.76.1 pgoyette
1012 1.1.1.1.76.1 pgoyette /* create and write .add file */
1013 1.1.1.1.76.1 pgoyette strcpy(log->end, ".add");
1014 1.1.1.1.76.1 pgoyette fd = open(log->path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
1015 1.1.1.1.76.1 pgoyette if (fd < 0)
1016 1.1.1.1.76.1 pgoyette return -1;
1017 1.1.1.1.76.1 pgoyette ret = (size_t)write(fd, data, len) != len;
1018 1.1.1.1.76.1 pgoyette if (ret | close(fd))
1019 1.1.1.1.76.1 pgoyette return -1;
1020 1.1.1.1.76.1 pgoyette log_touch(log);
1021 1.1.1.1.76.1 pgoyette
1022 1.1.1.1.76.1 pgoyette /* mark log file with append in progress */
1023 1.1.1.1.76.1 pgoyette if (log_mark(log, APPEND_OP))
1024 1.1.1.1.76.1 pgoyette return -1;
1025 1.1.1.1.76.1 pgoyette BAIL(8);
1026 1.1.1.1.76.1 pgoyette
1027 1.1.1.1.76.1 pgoyette /* append data (clears mark) */
1028 1.1.1.1.76.1 pgoyette if (log_append(log, data, len))
1029 1.1.1.1.76.1 pgoyette return -1;
1030 1.1.1.1.76.1 pgoyette
1031 1.1.1.1.76.1 pgoyette /* check to see if it's time to compress -- if not, then done */
1032 1.1.1.1.76.1 pgoyette if (((log->last - log->first) >> 10) + (log->stored >> 10) < TRIGGER)
1033 1.1.1.1.76.1 pgoyette return 0;
1034 1.1 christos
1035 1.1.1.1.76.1 pgoyette /* time to compress */
1036 1.1.1.1.76.1 pgoyette return gzlog_compress(log);
1037 1.1.1.1.76.1 pgoyette }
1038 1.1 christos
1039 1.1.1.1.76.1 pgoyette /* gzlog_close() return values:
1040 1.1.1.1.76.1 pgoyette 0: ok
1041 1.1.1.1.76.1 pgoyette -3: invalid log pointer argument */
1042 1.1.1.1.76.1 pgoyette int gzlog_close(gzlog *logd)
1043 1.1.1.1.76.1 pgoyette {
1044 1.1.1.1.76.1 pgoyette struct log *log = logd;
1045 1.1 christos
1046 1.1.1.1.76.1 pgoyette /* check arguments */
1047 1.1.1.1.76.1 pgoyette if (log == NULL || strcmp(log->id, LOGID))
1048 1.1.1.1.76.1 pgoyette return -3;
1049 1.1.1.1.76.1 pgoyette
1050 1.1.1.1.76.1 pgoyette /* close the log file and release the lock */
1051 1.1.1.1.76.1 pgoyette log_close(log);
1052 1.1.1.1.76.1 pgoyette
1053 1.1.1.1.76.1 pgoyette /* free structure and return */
1054 1.1.1.1.76.1 pgoyette if (log->path != NULL)
1055 1.1.1.1.76.1 pgoyette free(log->path);
1056 1.1.1.1.76.1 pgoyette strcpy(log->id, "bad");
1057 1.1.1.1.76.1 pgoyette free(log);
1058 1.1 christos return 0;
1059 1.1 christos }
1060