strfile.c revision 1.15 1 1.15 jsm /* $NetBSD: strfile.c,v 1.15 1999/12/07 23:07:39 jsm Exp $ */
2 1.3 cgd
3 1.1 cgd /*-
4 1.3 cgd * Copyright (c) 1989, 1993
5 1.3 cgd * The Regents of the University of California. All rights reserved.
6 1.1 cgd *
7 1.1 cgd * This code is derived from software contributed to Berkeley by
8 1.1 cgd * Ken Arnold.
9 1.1 cgd *
10 1.1 cgd * Redistribution and use in source and binary forms, with or without
11 1.1 cgd * modification, are permitted provided that the following conditions
12 1.1 cgd * are met:
13 1.1 cgd * 1. Redistributions of source code must retain the above copyright
14 1.1 cgd * notice, this list of conditions and the following disclaimer.
15 1.1 cgd * 2. Redistributions in binary form must reproduce the above copyright
16 1.1 cgd * notice, this list of conditions and the following disclaimer in the
17 1.1 cgd * documentation and/or other materials provided with the distribution.
18 1.1 cgd * 3. All advertising materials mentioning features or use of this software
19 1.1 cgd * must display the following acknowledgement:
20 1.1 cgd * This product includes software developed by the University of
21 1.1 cgd * California, Berkeley and its contributors.
22 1.1 cgd * 4. Neither the name of the University nor the names of its contributors
23 1.1 cgd * may be used to endorse or promote products derived from this software
24 1.1 cgd * without specific prior written permission.
25 1.1 cgd *
26 1.1 cgd * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
27 1.1 cgd * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 1.1 cgd * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29 1.1 cgd * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
30 1.1 cgd * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31 1.1 cgd * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32 1.1 cgd * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33 1.1 cgd * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 1.1 cgd * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 1.1 cgd * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 1.1 cgd * SUCH DAMAGE.
37 1.1 cgd */
38 1.1 cgd
39 1.7 lukem #include <sys/cdefs.h>
40 1.1 cgd #ifndef lint
41 1.7 lukem __COPYRIGHT("@(#) Copyright (c) 1989, 1993\n\
42 1.7 lukem The Regents of the University of California. All rights reserved.\n");
43 1.1 cgd #endif /* not lint */
44 1.1 cgd
45 1.1 cgd #ifndef lint
46 1.3 cgd #if 0
47 1.3 cgd static char sccsid[] = "@(#)strfile.c 8.1 (Berkeley) 5/31/93";
48 1.3 cgd #else
49 1.15 jsm __RCSID("$NetBSD: strfile.c,v 1.15 1999/12/07 23:07:39 jsm Exp $");
50 1.3 cgd #endif
51 1.1 cgd #endif /* not lint */
52 1.1 cgd
53 1.5 christos # include <sys/types.h>
54 1.1 cgd # include <sys/param.h>
55 1.7 lukem # include <err.h>
56 1.7 lukem # include <ctype.h>
57 1.1 cgd # include <stdio.h>
58 1.7 lukem # include <stdlib.h>
59 1.4 cgd # include <string.h>
60 1.12 jsm # include <time.h>
61 1.7 lukem # include <unistd.h>
62 1.1 cgd # include "strfile.h"
63 1.1 cgd
64 1.1 cgd # ifndef MAXPATHLEN
65 1.1 cgd # define MAXPATHLEN 1024
66 1.1 cgd # endif /* MAXPATHLEN */
67 1.1 cgd
68 1.1 cgd /*
69 1.1 cgd * This program takes a file composed of strings seperated by
70 1.1 cgd * lines starting with two consecutive delimiting character (default
71 1.1 cgd * character is '%') and creates another file which consists of a table
72 1.1 cgd * describing the file (structure from "strfile.h"), a table of seek
73 1.1 cgd * pointers to the start of the strings, and the strings, each terminated
74 1.1 cgd * by a null byte. Usage:
75 1.1 cgd *
76 1.1 cgd * % strfile [-iorsx] [ -cC ] sourcefile [ datafile ]
77 1.1 cgd *
78 1.1 cgd * c - Change delimiting character from '%' to 'C'
79 1.1 cgd * s - Silent. Give no summary of data processed at the end of
80 1.1 cgd * the run.
81 1.1 cgd * o - order the strings in alphabetic order
82 1.1 cgd * i - if ordering, ignore case
83 1.1 cgd * r - randomize the order of the strings
84 1.1 cgd * x - set rotated bit
85 1.1 cgd *
86 1.1 cgd * Ken Arnold Sept. 7, 1978 --
87 1.1 cgd *
88 1.1 cgd * Added ordering options.
89 1.1 cgd */
90 1.1 cgd
91 1.1 cgd # define TRUE 1
92 1.1 cgd # define FALSE 0
93 1.1 cgd
94 1.1 cgd # define STORING_PTRS (Oflag || Rflag)
95 1.1 cgd # define CHUNKSIZE 512
96 1.1 cgd
97 1.15 jsm # define ALLOC(ptr,sz) do { \
98 1.1 cgd if (ptr == NULL) \
99 1.1 cgd ptr = malloc((unsigned int) (CHUNKSIZE * sizeof *ptr)); \
100 1.1 cgd else if (((sz) + 1) % CHUNKSIZE == 0) \
101 1.1 cgd ptr = realloc((void *) ptr, ((unsigned int) ((sz) + CHUNKSIZE) * sizeof *ptr)); \
102 1.7 lukem if (ptr == NULL) \
103 1.7 lukem errx(1, "out of space"); \
104 1.15 jsm } while (0)
105 1.1 cgd
106 1.1 cgd #ifdef NO_VOID
107 1.1 cgd # define void char
108 1.1 cgd #endif
109 1.1 cgd
110 1.1 cgd typedef struct {
111 1.1 cgd char first;
112 1.1 cgd off_t pos;
113 1.1 cgd } STR;
114 1.1 cgd
115 1.1 cgd char *Infile = NULL, /* input file name */
116 1.1 cgd Outfile[MAXPATHLEN] = "", /* output file name */
117 1.1 cgd Delimch = '%'; /* delimiting character */
118 1.1 cgd
119 1.1 cgd int Sflag = FALSE; /* silent run flag */
120 1.1 cgd int Oflag = FALSE; /* ordering flag */
121 1.1 cgd int Iflag = FALSE; /* ignore case flag */
122 1.1 cgd int Rflag = FALSE; /* randomize order flag */
123 1.1 cgd int Xflag = FALSE; /* set rotated bit */
124 1.1 cgd long Num_pts = 0; /* number of pointers/strings */
125 1.1 cgd
126 1.1 cgd off_t *Seekpts;
127 1.1 cgd
128 1.1 cgd FILE *Sort_1, *Sort_2; /* pointers for sorting */
129 1.1 cgd
130 1.1 cgd STRFILE Tbl; /* statistics table */
131 1.1 cgd
132 1.1 cgd STR *Firstch; /* first chars of each string */
133 1.1 cgd
134 1.7 lukem void add_offset __P((FILE *, off_t));
135 1.7 lukem int cmp_str __P((const void *, const void *));
136 1.7 lukem void do_order __P((void));
137 1.7 lukem void getargs __P((int, char *[]));
138 1.7 lukem int main __P((int, char *[]));
139 1.7 lukem void randomize __P((void));
140 1.7 lukem char *unctrl __P((char));
141 1.8 hubertf void usage __P((void)) __attribute__((__noreturn__));
142 1.1 cgd
143 1.1 cgd
144 1.1 cgd /*
145 1.1 cgd * main:
146 1.1 cgd * Drive the sucker. There are two main modes -- either we store
147 1.1 cgd * the seek pointers, if the table is to be sorted or randomized,
148 1.1 cgd * or we write the pointer directly to the file, if we are to stay
149 1.1 cgd * in file order. If the former, we allocate and re-allocate in
150 1.1 cgd * CHUNKSIZE blocks; if the latter, we just write each pointer,
151 1.1 cgd * and then seek back to the beginning to write in the table.
152 1.1 cgd */
153 1.7 lukem int
154 1.1 cgd main(ac, av)
155 1.7 lukem int ac;
156 1.7 lukem char *av[];
157 1.1 cgd {
158 1.7 lukem char *sp, dc;
159 1.7 lukem FILE *inf, *outf;
160 1.7 lukem off_t last_off, length, pos, *p;
161 1.7 lukem int first, cnt;
162 1.7 lukem char *nsp;
163 1.7 lukem STR *fp;
164 1.7 lukem static char string[257];
165 1.1 cgd
166 1.1 cgd getargs(ac, av); /* evalute arguments */
167 1.1 cgd dc = Delimch;
168 1.7 lukem if ((inf = fopen(Infile, "r")) == NULL)
169 1.7 lukem err(1, "open `%s'", Infile);
170 1.1 cgd
171 1.7 lukem if ((outf = fopen(Outfile, "w")) == NULL)
172 1.7 lukem err(1, "open `%s'", Outfile);
173 1.1 cgd if (!STORING_PTRS)
174 1.11 jsm (void) fseek(outf, sizeof Tbl, SEEK_SET);
175 1.1 cgd
176 1.1 cgd /*
177 1.1 cgd * Write the strings onto the file
178 1.1 cgd */
179 1.1 cgd
180 1.1 cgd Tbl.str_longlen = 0;
181 1.1 cgd Tbl.str_shortlen = (unsigned int) 0xffffffff;
182 1.1 cgd Tbl.str_delim = dc;
183 1.1 cgd Tbl.str_version = VERSION;
184 1.1 cgd first = Oflag;
185 1.1 cgd add_offset(outf, ftell(inf));
186 1.1 cgd last_off = 0;
187 1.1 cgd do {
188 1.1 cgd sp = fgets(string, 256, inf);
189 1.7 lukem if (sp == NULL || (sp[0] == dc && sp[1] == '\n')) {
190 1.1 cgd pos = ftell(inf);
191 1.1 cgd length = pos - last_off - (sp ? strlen(sp) : 0);
192 1.1 cgd last_off = pos;
193 1.1 cgd if (!length)
194 1.1 cgd continue;
195 1.1 cgd add_offset(outf, pos);
196 1.14 jsm if ((off_t)Tbl.str_longlen < length)
197 1.1 cgd Tbl.str_longlen = length;
198 1.14 jsm if ((off_t)Tbl.str_shortlen > length)
199 1.1 cgd Tbl.str_shortlen = length;
200 1.1 cgd first = Oflag;
201 1.1 cgd }
202 1.1 cgd else if (first) {
203 1.1 cgd for (nsp = sp; !isalnum(*nsp); nsp++)
204 1.1 cgd continue;
205 1.1 cgd ALLOC(Firstch, Num_pts);
206 1.1 cgd fp = &Firstch[Num_pts - 1];
207 1.1 cgd if (Iflag && isupper(*nsp))
208 1.1 cgd fp->first = tolower(*nsp);
209 1.1 cgd else
210 1.1 cgd fp->first = *nsp;
211 1.1 cgd fp->pos = Seekpts[Num_pts - 1];
212 1.1 cgd first = FALSE;
213 1.1 cgd }
214 1.1 cgd } while (sp != NULL);
215 1.1 cgd
216 1.1 cgd /*
217 1.1 cgd * write the tables in
218 1.1 cgd */
219 1.1 cgd
220 1.1 cgd (void) fclose(inf);
221 1.1 cgd
222 1.1 cgd if (Oflag)
223 1.1 cgd do_order();
224 1.1 cgd else if (Rflag)
225 1.1 cgd randomize();
226 1.1 cgd
227 1.1 cgd if (Xflag)
228 1.1 cgd Tbl.str_flags |= STR_ROTATED;
229 1.1 cgd
230 1.1 cgd if (!Sflag) {
231 1.1 cgd printf("\"%s\" created\n", Outfile);
232 1.1 cgd if (Num_pts == 2)
233 1.1 cgd puts("There was 1 string");
234 1.1 cgd else
235 1.7 lukem printf("There were %d strings\n", (int)(Num_pts - 1));
236 1.1 cgd printf("Longest string: %lu byte%s\n", Tbl.str_longlen,
237 1.1 cgd Tbl.str_longlen == 1 ? "" : "s");
238 1.1 cgd printf("Shortest string: %lu byte%s\n", Tbl.str_shortlen,
239 1.1 cgd Tbl.str_shortlen == 1 ? "" : "s");
240 1.1 cgd }
241 1.1 cgd
242 1.11 jsm (void) fseek(outf, (off_t) 0, SEEK_SET);
243 1.9 simonb HTOBE32(Tbl.str_version);
244 1.9 simonb Tbl.str_numstr = htobe32(Num_pts - 1);
245 1.9 simonb HTOBE32(Tbl.str_longlen);
246 1.9 simonb HTOBE32(Tbl.str_shortlen);
247 1.9 simonb HTOBE32(Tbl.str_flags);
248 1.1 cgd (void) fwrite((char *) &Tbl, sizeof Tbl, 1, outf);
249 1.1 cgd if (STORING_PTRS) {
250 1.1 cgd for (p = Seekpts, cnt = Num_pts; cnt--; ++p)
251 1.9 simonb HTOBE64(*p);
252 1.1 cgd (void) fwrite((char *) Seekpts, sizeof *Seekpts, (int) Num_pts, outf);
253 1.1 cgd }
254 1.13 jsm fflush(outf);
255 1.13 jsm if (ferror(outf))
256 1.13 jsm err(1, "fwrite %s", Outfile);
257 1.1 cgd (void) fclose(outf);
258 1.1 cgd exit(0);
259 1.1 cgd }
260 1.1 cgd
261 1.1 cgd /*
262 1.1 cgd * This routine evaluates arguments from the command line
263 1.1 cgd */
264 1.7 lukem void
265 1.1 cgd getargs(argc, argv)
266 1.7 lukem int argc;
267 1.7 lukem char **argv;
268 1.1 cgd {
269 1.1 cgd int ch;
270 1.1 cgd
271 1.6 lukem while ((ch = getopt(argc, argv, "c:iorsx")) != -1)
272 1.1 cgd switch(ch) {
273 1.1 cgd case 'c': /* new delimiting char */
274 1.1 cgd Delimch = *optarg;
275 1.1 cgd if (!isascii(Delimch)) {
276 1.1 cgd printf("bad delimiting character: '\\%o\n'",
277 1.1 cgd Delimch);
278 1.1 cgd }
279 1.1 cgd break;
280 1.1 cgd case 'i': /* ignore case in ordering */
281 1.1 cgd Iflag++;
282 1.1 cgd break;
283 1.1 cgd case 'o': /* order strings */
284 1.1 cgd Oflag++;
285 1.1 cgd break;
286 1.1 cgd case 'r': /* randomize pointers */
287 1.1 cgd Rflag++;
288 1.1 cgd break;
289 1.1 cgd case 's': /* silent */
290 1.1 cgd Sflag++;
291 1.1 cgd break;
292 1.1 cgd case 'x': /* set the rotated bit */
293 1.1 cgd Xflag++;
294 1.1 cgd break;
295 1.1 cgd case '?':
296 1.1 cgd default:
297 1.1 cgd usage();
298 1.1 cgd }
299 1.1 cgd argv += optind;
300 1.1 cgd
301 1.1 cgd if (*argv) {
302 1.1 cgd Infile = *argv;
303 1.1 cgd if (*++argv)
304 1.1 cgd (void) strcpy(Outfile, *argv);
305 1.1 cgd }
306 1.1 cgd if (!Infile) {
307 1.1 cgd puts("No input file name");
308 1.1 cgd usage();
309 1.1 cgd }
310 1.1 cgd if (*Outfile == '\0') {
311 1.1 cgd (void) strcpy(Outfile, Infile);
312 1.1 cgd (void) strcat(Outfile, ".dat");
313 1.1 cgd }
314 1.1 cgd }
315 1.1 cgd
316 1.7 lukem void
317 1.1 cgd usage()
318 1.1 cgd {
319 1.1 cgd (void) fprintf(stderr,
320 1.1 cgd "strfile [-iorsx] [-c char] sourcefile [datafile]\n");
321 1.1 cgd exit(1);
322 1.1 cgd }
323 1.1 cgd
324 1.1 cgd /*
325 1.1 cgd * add_offset:
326 1.1 cgd * Add an offset to the list, or write it out, as appropriate.
327 1.1 cgd */
328 1.7 lukem void
329 1.1 cgd add_offset(fp, off)
330 1.7 lukem FILE *fp;
331 1.7 lukem off_t off;
332 1.1 cgd {
333 1.1 cgd off_t net;
334 1.1 cgd
335 1.1 cgd if (!STORING_PTRS) {
336 1.9 simonb net = htobe64(off);
337 1.1 cgd fwrite(&net, 1, sizeof net, fp);
338 1.1 cgd } else {
339 1.1 cgd ALLOC(Seekpts, Num_pts + 1);
340 1.1 cgd Seekpts[Num_pts] = off;
341 1.1 cgd }
342 1.1 cgd Num_pts++;
343 1.1 cgd }
344 1.1 cgd
345 1.1 cgd /*
346 1.1 cgd * do_order:
347 1.1 cgd * Order the strings alphabetically (possibly ignoring case).
348 1.1 cgd */
349 1.7 lukem void
350 1.1 cgd do_order()
351 1.1 cgd {
352 1.7 lukem int i;
353 1.7 lukem off_t *lp;
354 1.7 lukem STR *fp;
355 1.1 cgd
356 1.1 cgd Sort_1 = fopen(Infile, "r");
357 1.1 cgd Sort_2 = fopen(Infile, "r");
358 1.1 cgd qsort((char *) Firstch, (int) Tbl.str_numstr, sizeof *Firstch, cmp_str);
359 1.1 cgd i = Tbl.str_numstr;
360 1.1 cgd lp = Seekpts;
361 1.1 cgd fp = Firstch;
362 1.1 cgd while (i--)
363 1.1 cgd *lp++ = fp++->pos;
364 1.1 cgd (void) fclose(Sort_1);
365 1.1 cgd (void) fclose(Sort_2);
366 1.1 cgd Tbl.str_flags |= STR_ORDERED;
367 1.1 cgd }
368 1.1 cgd
369 1.1 cgd /*
370 1.1 cgd * cmp_str:
371 1.1 cgd * Compare two strings in the file
372 1.1 cgd */
373 1.1 cgd char *
374 1.1 cgd unctrl(c)
375 1.7 lukem char c;
376 1.1 cgd {
377 1.1 cgd static char buf[3];
378 1.1 cgd
379 1.1 cgd if (isprint(c)) {
380 1.1 cgd buf[0] = c;
381 1.1 cgd buf[1] = '\0';
382 1.1 cgd }
383 1.1 cgd else if (c == 0177) {
384 1.1 cgd buf[0] = '^';
385 1.1 cgd buf[1] = '?';
386 1.1 cgd }
387 1.1 cgd else {
388 1.1 cgd buf[0] = '^';
389 1.1 cgd buf[1] = c + 'A' - 1;
390 1.1 cgd }
391 1.1 cgd return buf;
392 1.1 cgd }
393 1.1 cgd
394 1.7 lukem int
395 1.7 lukem cmp_str(vp1, vp2)
396 1.7 lukem const void *vp1, *vp2;
397 1.1 cgd {
398 1.10 jsm const STR *p1, *p2;
399 1.7 lukem int c1, c2;
400 1.7 lukem int n1, n2;
401 1.7 lukem
402 1.10 jsm p1 = (const STR *)vp1;
403 1.10 jsm p2 = (const STR *)vp2;
404 1.1 cgd
405 1.1 cgd # define SET_N(nf,ch) (nf = (ch == '\n'))
406 1.1 cgd # define IS_END(ch,nf) (ch == Delimch && nf)
407 1.1 cgd
408 1.1 cgd c1 = p1->first;
409 1.1 cgd c2 = p2->first;
410 1.1 cgd if (c1 != c2)
411 1.1 cgd return c1 - c2;
412 1.1 cgd
413 1.11 jsm (void) fseek(Sort_1, p1->pos, SEEK_SET);
414 1.11 jsm (void) fseek(Sort_2, p2->pos, SEEK_SET);
415 1.1 cgd
416 1.1 cgd n1 = FALSE;
417 1.1 cgd n2 = FALSE;
418 1.1 cgd while (!isalnum(c1 = getc(Sort_1)) && c1 != '\0')
419 1.1 cgd SET_N(n1, c1);
420 1.1 cgd while (!isalnum(c2 = getc(Sort_2)) && c2 != '\0')
421 1.1 cgd SET_N(n2, c2);
422 1.1 cgd
423 1.1 cgd while (!IS_END(c1, n1) && !IS_END(c2, n2)) {
424 1.1 cgd if (Iflag) {
425 1.1 cgd if (isupper(c1))
426 1.1 cgd c1 = tolower(c1);
427 1.1 cgd if (isupper(c2))
428 1.1 cgd c2 = tolower(c2);
429 1.1 cgd }
430 1.1 cgd if (c1 != c2)
431 1.1 cgd return c1 - c2;
432 1.1 cgd SET_N(n1, c1);
433 1.1 cgd SET_N(n2, c2);
434 1.1 cgd c1 = getc(Sort_1);
435 1.1 cgd c2 = getc(Sort_2);
436 1.1 cgd }
437 1.1 cgd if (IS_END(c1, n1))
438 1.1 cgd c1 = 0;
439 1.1 cgd if (IS_END(c2, n2))
440 1.1 cgd c2 = 0;
441 1.1 cgd return c1 - c2;
442 1.1 cgd }
443 1.1 cgd
444 1.1 cgd /*
445 1.1 cgd * randomize:
446 1.1 cgd * Randomize the order of the string table. We must be careful
447 1.1 cgd * not to randomize across delimiter boundaries. All
448 1.1 cgd * randomization is done within each block.
449 1.1 cgd */
450 1.7 lukem void
451 1.1 cgd randomize()
452 1.1 cgd {
453 1.7 lukem int cnt, i;
454 1.7 lukem off_t tmp;
455 1.7 lukem off_t *sp;
456 1.1 cgd
457 1.1 cgd srandom((int)(time((time_t *) NULL) + getpid()));
458 1.1 cgd
459 1.1 cgd Tbl.str_flags |= STR_RANDOM;
460 1.1 cgd cnt = Tbl.str_numstr;
461 1.1 cgd
462 1.1 cgd /*
463 1.1 cgd * move things around randomly
464 1.1 cgd */
465 1.1 cgd
466 1.1 cgd for (sp = Seekpts; cnt > 0; cnt--, sp++) {
467 1.1 cgd i = random() % cnt;
468 1.1 cgd tmp = sp[0];
469 1.1 cgd sp[0] = sp[i];
470 1.1 cgd sp[i] = tmp;
471 1.1 cgd }
472 1.1 cgd }
473