makemandb.c revision 1.58 1 /* $NetBSD: makemandb.c,v 1.58 2019/03/11 00:14:44 christos Exp $ */
2 /*
3 * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay (at) gmail.com>
4 * Copyright (c) 2011 Kristaps Dzonsons <kristaps (at) bsd.lv>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19 #include <sys/cdefs.h>
20 __RCSID("$NetBSD: makemandb.c,v 1.58 2019/03/11 00:14:44 christos Exp $");
21
22 #include <sys/stat.h>
23 #include <sys/types.h>
24
25 #include <assert.h>
26 #include <dirent.h>
27 #include <err.h>
28 #include <archive.h>
29 #include <libgen.h>
30 #include <md5.h>
31 #include <stdio.h>
32 #include <stdlib.h>
33 #include <string.h>
34 #include <unistd.h>
35 #include <util.h>
36
37 #include "apropos-utils.h"
38 #include "dist/man.h"
39 #include "dist/mandoc.h"
40 #include "dist/mdoc.h"
41 #include "dist/roff.h"
42 #include "dist/roff_int.h"
43 #include "dist/mandoc_parse.h"
44
45 #define BUFLEN 1024
46 #define MDOC 0 //If the page is of mdoc(7) type
47 #define MAN 1 //If the page is of man(7) type
48
49 /*
50 * A data structure for holding section specific data.
51 */
52 typedef struct secbuff {
53 char *data;
54 size_t buflen; //Total length of buffer allocated initially
55 size_t offset; // Current offset in the buffer.
56 } secbuff;
57
58 typedef struct makemandb_flags {
59 int optimize;
60 int limit; // limit the indexing to only NAME section
61 int recreate; // Database was created from scratch
62 int verbosity; // 0: quiet, 1: default, 2: verbose
63 } makemandb_flags;
64
65 typedef struct roff_mandb_rec {
66 /* Fields for mandb table */
67 char *name; // for storing the name of the man page
68 char *name_desc; // for storing the one line description (.Nd)
69 secbuff desc; // for storing the DESCRIPTION section
70 secbuff lib; // for the LIBRARY section
71 secbuff return_vals; // RETURN VALUES
72 secbuff env; // ENVIRONMENT
73 secbuff files; // FILES
74 secbuff exit_status; // EXIT STATUS
75 secbuff diagnostics; // DIAGNOSTICS
76 secbuff errors; // ERRORS
77 char *section;
78
79 int xr_found; // To track whether a .Xr was seen when parsing a section
80
81 /* Fields for mandb_meta table */
82 char *md5_hash;
83 dev_t device;
84 ino_t inode;
85 time_t mtime;
86
87 /* Fields for mandb_links table */
88 char *machine;
89 char *links; //all the links to a page in a space separated form
90 char *file_path;
91
92 /* Non-db fields */
93 int page_type; //Indicates the type of page: mdoc or man
94 } mandb_rec;
95
96 typedef void (*proff_nf)(const struct roff_node *n, mandb_rec *);
97
98 static void append(secbuff *sbuff, const char *src);
99 static void init_secbuffs(mandb_rec *);
100 static void free_secbuffs(mandb_rec *);
101 static int check_md5(const char *, sqlite3 *, char **, void *, size_t);
102 static void cleanup(mandb_rec *);
103 static void set_section(const struct roff_meta *, mandb_rec *);
104 static void set_machine(const struct roff_meta *, mandb_rec *);
105 static int insert_into_db(sqlite3 *, mandb_rec *);
106 static void begin_parse(const char *, struct mparse *, mandb_rec *, int);
107 static void proff_node(const struct roff_node *, mandb_rec *,
108 struct roff_meta *, const proff_nf *);
109 static void pmdoc_Nm(const struct roff_node *, mandb_rec *);
110 static void pmdoc_Nd(const struct roff_node *, mandb_rec *);
111 static void pmdoc_Sh(const struct roff_node *, mandb_rec *);
112 static void mdoc_parse_Sh(const struct roff_node *, mandb_rec *);
113 static void pmdoc_Xr(const struct roff_node *, mandb_rec *);
114 static void pmdoc_Pp(const struct roff_node *, mandb_rec *);
115 static void pmdoc_macro_handler(const struct roff_node *, mandb_rec *, int);
116 static void pman_parse_node(const struct roff_node *, secbuff *);
117 static void pman_parse_name(const struct roff_node *, mandb_rec *);
118 static void pman_sh(const struct roff_node *, mandb_rec *);
119 static void pman_block(const struct roff_node *, mandb_rec *);
120 static void traversedir(const char *, const char *, sqlite3 *, struct mparse *);
121 static void mdoc_parse_section(enum roff_sec, const char *, mandb_rec *);
122 static void man_parse_section(enum man_sec, const struct roff_node *, mandb_rec *);
123 static void build_file_cache(sqlite3 *, const char *, const char *,
124 struct stat *);
125 static void update_db(sqlite3 *, struct mparse *, mandb_rec *);
126 __dead static void usage(void);
127 static void optimize(sqlite3 *);
128 static char *parse_escape(const char *);
129 static void replace_hyph(char *);
130 static makemandb_flags mflags = { .verbosity = 1 };
131
132 static const proff_nf mdocs[MDOC_MAX - MDOC_Dd] = {
133 NULL, /* Dd */
134 NULL, /* Dt */
135 NULL, /* Os */
136 pmdoc_Sh, /* Sh */
137 NULL, /* Ss */
138 pmdoc_Pp, /* Pp */
139 NULL, /* D1 */
140 NULL, /* Dl */
141 NULL, /* Bd */
142 NULL, /* Ed */
143 NULL, /* Bl */
144 NULL, /* El */
145 NULL, /* It */
146 NULL, /* Ad */
147 NULL, /* An */
148 NULL, /* Ap */
149 NULL, /* Ar */
150 NULL, /* Cd */
151 NULL, /* Cm */
152 NULL, /* Dv */
153 NULL, /* Er */
154 NULL, /* Ev */
155 NULL, /* Ex */
156 NULL, /* Fa */
157 NULL, /* Fd */
158 NULL, /* Fl */
159 NULL, /* Fn */
160 NULL, /* Ft */
161 NULL, /* Ic */
162 NULL, /* In */
163 NULL, /* Li */
164 pmdoc_Nd, /* Nd */
165 pmdoc_Nm, /* Nm */
166 NULL, /* Op */
167 NULL, /* Ot */
168 NULL, /* Pa */
169 NULL, /* Rv */
170 NULL, /* St */
171 NULL, /* Va */
172 NULL, /* Vt */
173 pmdoc_Xr, /* Xr */
174 NULL, /* %A */
175 NULL, /* %B */
176 NULL, /* %D */
177 NULL, /* %I */
178 NULL, /* %J */
179 NULL, /* %N */
180 NULL, /* %O */
181 NULL, /* %P */
182 NULL, /* %R */
183 NULL, /* %T */
184 NULL, /* %V */
185 NULL, /* Ac */
186 NULL, /* Ao */
187 NULL, /* Aq */
188 NULL, /* At */
189 NULL, /* Bc */
190 NULL, /* Bf */
191 NULL, /* Bo */
192 NULL, /* Bq */
193 NULL, /* Bsx */
194 NULL, /* Bx */
195 NULL, /* Db */
196 NULL, /* Dc */
197 NULL, /* Do */
198 NULL, /* Dq */
199 NULL, /* Ec */
200 NULL, /* Ef */
201 NULL, /* Em */
202 NULL, /* Eo */
203 NULL, /* Fx */
204 NULL, /* Ms */
205 NULL, /* No */
206 NULL, /* Ns */
207 NULL, /* Nx */
208 NULL, /* Ox */
209 NULL, /* Pc */
210 NULL, /* Pf */
211 NULL, /* Po */
212 NULL, /* Pq */
213 NULL, /* Qc */
214 NULL, /* Ql */
215 NULL, /* Qo */
216 NULL, /* Qq */
217 NULL, /* Re */
218 NULL, /* Rs */
219 NULL, /* Sc */
220 NULL, /* So */
221 NULL, /* Sq */
222 NULL, /* Sm */
223 NULL, /* Sx */
224 NULL, /* Sy */
225 NULL, /* Tn */
226 NULL, /* Ux */
227 NULL, /* Xc */
228 NULL, /* Xo */
229 NULL, /* Fo */
230 NULL, /* Fc */
231 NULL, /* Oo */
232 NULL, /* Oc */
233 NULL, /* Bk */
234 NULL, /* Ek */
235 NULL, /* Bt */
236 NULL, /* Hf */
237 NULL, /* Fr */
238 NULL, /* Ud */
239 NULL, /* Lb */
240 NULL, /* Lp */
241 NULL, /* Lk */
242 NULL, /* Mt */
243 NULL, /* Brq */
244 NULL, /* Bro */
245 NULL, /* Brc */
246 NULL, /* %C */
247 NULL, /* Es */
248 NULL, /* En */
249 NULL, /* Dx */
250 NULL, /* %Q */
251 NULL, /* %U */
252 NULL /* Ta */
253 };
254
255 static const proff_nf mans[MAN_MAX - MAN_TH] = {
256 NULL, //TH
257 pman_sh, //SH
258 NULL, //SS
259 NULL, //TP
260 NULL, //LP
261 NULL, //PP
262 NULL, //P
263 NULL, //IP
264 NULL, //HP
265 NULL, //SM
266 NULL, //SB
267 NULL, //BI
268 NULL, //IB
269 NULL, //BR
270 NULL, //RB
271 NULL, //R
272 pman_block, //B
273 NULL, //I
274 NULL, //IR
275 NULL, //RI
276 NULL, //nf
277 NULL, //fi
278 NULL, //RE
279 NULL, //RS
280 NULL, //DT
281 NULL, //UC
282 NULL, //PD
283 NULL, //AT
284 NULL, //in
285 NULL, //OP
286 NULL, //EX
287 NULL, //EE
288 NULL, //UR
289 NULL, //UE
290 NULL, //MT
291 NULL //ME
292 };
293
294 int
295 main(int argc, char *argv[])
296 {
297 FILE *file;
298 const char *sqlstr, *manconf = NULL;
299 char *line, *command;
300 char *errmsg;
301 int ch;
302 struct mparse *mp;
303 sqlite3 *db;
304 ssize_t len;
305 size_t linesize;
306 struct roff_mandb_rec rec;
307
308 while ((ch = getopt(argc, argv, "C:floQqv")) != -1) {
309 switch (ch) {
310 case 'C':
311 manconf = optarg;
312 break;
313 case 'f':
314 mflags.recreate = 1;
315 break;
316 case 'l':
317 mflags.limit = 1;
318 break;
319 case 'o':
320 mflags.optimize = 1;
321 break;
322 case 'Q':
323 mflags.verbosity = 0;
324 break;
325 case 'q':
326 mflags.verbosity = 1;
327 break;
328 case 'v':
329 mflags.verbosity = 2;
330 break;
331 default:
332 usage();
333 }
334 }
335
336 memset(&rec, 0, sizeof(rec));
337
338 init_secbuffs(&rec);
339 mchars_alloc();
340 mp = mparse_alloc(MPARSE_SO | MPARSE_UTF8 | MPARSE_LATIN1 |
341 MPARSE_VALIDATE, MANDOC_OS_OTHER, NULL);
342
343 if (manconf) {
344 char *arg;
345 size_t command_len = shquote(manconf, NULL, 0) + 1;
346 arg = emalloc(command_len);
347 shquote(manconf, arg, command_len);
348 easprintf(&command, "man -p -C %s", arg);
349 free(arg);
350 } else {
351 command = estrdup("man -p");
352 manconf = MANCONF;
353 }
354
355 if (mflags.recreate) {
356 char *dbp = get_dbpath(manconf);
357 /* No error here, it will fail in init_db in the same call */
358 if (dbp != NULL)
359 remove(dbp);
360 }
361
362 if ((db = init_db(MANDB_CREATE, manconf)) == NULL)
363 exit(EXIT_FAILURE);
364
365 sqlite3_exec(db, "PRAGMA synchronous = 0", NULL, NULL, &errmsg);
366 if (errmsg != NULL) {
367 warnx("%s", errmsg);
368 free(errmsg);
369 close_db(db);
370 exit(EXIT_FAILURE);
371 }
372
373 sqlite3_exec(db, "ATTACH DATABASE \':memory:\' AS metadb", NULL, NULL,
374 &errmsg);
375 if (errmsg != NULL) {
376 warnx("%s", errmsg);
377 free(errmsg);
378 close_db(db);
379 exit(EXIT_FAILURE);
380 }
381
382
383 /* Call man -p to get the list of man page dirs */
384 if ((file = popen(command, "r")) == NULL) {
385 close_db(db);
386 err(EXIT_FAILURE, "fopen failed");
387 }
388 free(command);
389
390 /* Begin the transaction for indexing the pages */
391 sqlite3_exec(db, "BEGIN", NULL, NULL, &errmsg);
392 if (errmsg != NULL) {
393 warnx("%s", errmsg);
394 free(errmsg);
395 close_db(db);
396 exit(EXIT_FAILURE);
397 }
398
399 sqlstr = "CREATE TABLE metadb.file_cache(device, inode, mtime, parent,"
400 " file PRIMARY KEY);"
401 "CREATE UNIQUE INDEX metadb.index_file_cache_dev"
402 " ON file_cache (device, inode)";
403
404 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
405 if (errmsg != NULL) {
406 warnx("%s", errmsg);
407 free(errmsg);
408 close_db(db);
409 exit(EXIT_FAILURE);
410 }
411
412 if (mflags.verbosity)
413 printf("Building temporary file cache\n");
414 line = NULL;
415 linesize = 0;
416 while ((len = getline(&line, &linesize, file)) != -1) {
417 /* Replace the new line character at the end of string with '\0' */
418 line[len - 1] = '\0';
419 char *pdir = estrdup(dirname(line));
420 /* Traverse the man page directories and parse the pages */
421 traversedir(pdir, line, db, mp);
422 free(pdir);
423 }
424 free(line);
425
426 if (pclose(file) == -1) {
427 close_db(db);
428 cleanup(&rec);
429 free_secbuffs(&rec);
430 err(EXIT_FAILURE, "pclose error");
431 }
432
433 if (mflags.verbosity)
434 printf("Performing index update\n");
435 update_db(db, mp, &rec);
436 mparse_free(mp);
437 mchars_free();
438 free_secbuffs(&rec);
439
440 /* Commit the transaction */
441 sqlite3_exec(db, "COMMIT", NULL, NULL, &errmsg);
442 if (errmsg != NULL) {
443 warnx("%s", errmsg);
444 free(errmsg);
445 close_db(db);
446 exit(EXIT_FAILURE);
447 }
448
449 if (mflags.optimize)
450 optimize(db);
451
452 close_db(db);
453 return 0;
454 }
455
456 /*
457 * traversedir --
458 * Traverses the given directory recursively and passes all the man page files
459 * in the way to build_file_cache()
460 */
461 static void
462 traversedir(const char *parent, const char *file, sqlite3 *db,
463 struct mparse *mp)
464 {
465 struct stat sb;
466 struct dirent *dirp;
467 DIR *dp;
468 char *buf;
469
470 if (stat(file, &sb) < 0) {
471 if (mflags.verbosity)
472 warn("stat failed: %s", file);
473 return;
474 }
475
476 /* If it is a directory, traverse it recursively */
477 if (S_ISDIR(sb.st_mode)) {
478 if ((dp = opendir(file)) == NULL) {
479 if (mflags.verbosity)
480 warn("opendir error: %s", file);
481 return;
482 }
483
484 while ((dirp = readdir(dp)) != NULL) {
485 /* Avoid . and .. entries in a directory */
486 if (dirp->d_name[0] != '.') {
487 easprintf(&buf, "%s/%s", file, dirp->d_name);
488 traversedir(parent, buf, db, mp);
489 free(buf);
490 }
491 }
492 closedir(dp);
493 return;
494 }
495
496 if (!S_ISREG(sb.st_mode))
497 return;
498
499 if (sb.st_size == 0) {
500 if (mflags.verbosity)
501 warnx("Empty file: %s", file);
502 return;
503 }
504 build_file_cache(db, parent, file, &sb);
505 }
506
507 /* build_file_cache --
508 * This function generates an md5 hash of the file passed as its 2nd parameter
509 * and stores it in a temporary table file_cache along with the full file path.
510 * This is done to support incremental updation of the database.
511 * The temporary table file_cache is dropped thereafter in the function
512 * update_db(), once the database has been updated.
513 */
514 static void
515 build_file_cache(sqlite3 *db, const char *parent, const char *file,
516 struct stat *sb)
517 {
518 const char *sqlstr;
519 sqlite3_stmt *stmt = NULL;
520 int rc, idx;
521 assert(file != NULL);
522 dev_t device_cache = sb->st_dev;
523 ino_t inode_cache = sb->st_ino;
524 time_t mtime_cache = sb->st_mtime;
525
526 sqlstr = "INSERT INTO metadb.file_cache VALUES (:device, :inode,"
527 " :mtime, :parent, :file)";
528 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
529 if (rc != SQLITE_OK) {
530 if (mflags.verbosity)
531 warnx("%s", sqlite3_errmsg(db));
532 return;
533 }
534
535 idx = sqlite3_bind_parameter_index(stmt, ":device");
536 rc = sqlite3_bind_int64(stmt, idx, device_cache);
537 if (rc != SQLITE_OK) {
538 if (mflags.verbosity)
539 warnx("%s", sqlite3_errmsg(db));
540 sqlite3_finalize(stmt);
541 return;
542 }
543
544 idx = sqlite3_bind_parameter_index(stmt, ":inode");
545 rc = sqlite3_bind_int64(stmt, idx, inode_cache);
546 if (rc != SQLITE_OK) {
547 if (mflags.verbosity)
548 warnx("%s", sqlite3_errmsg(db));
549 sqlite3_finalize(stmt);
550 return;
551 }
552
553 idx = sqlite3_bind_parameter_index(stmt, ":mtime");
554 rc = sqlite3_bind_int64(stmt, idx, mtime_cache);
555 if (rc != SQLITE_OK) {
556 if (mflags.verbosity)
557 warnx("%s", sqlite3_errmsg(db));
558 sqlite3_finalize(stmt);
559 return;
560 }
561
562 idx = sqlite3_bind_parameter_index(stmt, ":parent");
563 rc = sqlite3_bind_text(stmt, idx, parent, -1, NULL);
564 if (rc != SQLITE_OK) {
565 if (mflags.verbosity)
566 warnx("%s", sqlite3_errmsg(db));
567 sqlite3_finalize(stmt);
568 return;
569 }
570
571 idx = sqlite3_bind_parameter_index(stmt, ":file");
572 rc = sqlite3_bind_text(stmt, idx, file, -1, NULL);
573 if (rc != SQLITE_OK) {
574 if (mflags.verbosity)
575 warnx("%s", sqlite3_errmsg(db));
576 sqlite3_finalize(stmt);
577 return;
578 }
579
580 sqlite3_step(stmt);
581 sqlite3_finalize(stmt);
582 }
583
584 /* read_and_decompress --
585 * Reads the given file into memory. If it is compressed, decompress
586 * it before returning to the caller.
587 */
588 static int
589 read_and_decompress(const char *file, void **bufp, size_t *len)
590 {
591 size_t off;
592 ssize_t r;
593 struct archive *a;
594 struct archive_entry *ae;
595 char *buf;
596
597 if ((a = archive_read_new()) == NULL)
598 errx(EXIT_FAILURE, "memory allocation failed");
599
600 *bufp = NULL;
601 if (archive_read_support_filter_all(a) != ARCHIVE_OK ||
602 archive_read_support_format_raw(a) != ARCHIVE_OK ||
603 archive_read_open_filename(a, file, 65536) != ARCHIVE_OK ||
604 archive_read_next_header(a, &ae) != ARCHIVE_OK)
605 goto archive_error;
606 *len = 65536;
607 buf = emalloc(*len);
608 off = 0;
609 for (;;) {
610 r = archive_read_data(a, buf + off, *len - off);
611 if (r == ARCHIVE_OK) {
612 archive_read_free(a);
613 *bufp = buf;
614 *len = off;
615 return 0;
616 }
617 if (r <= 0) {
618 free(buf);
619 break;
620 }
621 off += r;
622 if (off == *len) {
623 *len *= 2;
624 if (*len < off) {
625 if (mflags.verbosity)
626 warnx("File too large: %s", file);
627 free(buf);
628 archive_read_free(a);
629 return -1;
630 }
631 buf = erealloc(buf, *len);
632 }
633 }
634
635 archive_error:
636 warnx("Error while reading `%s': %s", file, archive_error_string(a));
637 archive_read_free(a);
638 return -1;
639 }
640
641 static void
642 update_existing_entry(sqlite3 *db, const char *file, const char *hash,
643 mandb_rec *rec, int *new_count, int *link_count, int *err_count)
644 {
645 int update_count, rc, idx;
646 const char *inner_sqlstr;
647 sqlite3_stmt *inner_stmt;
648
649 update_count = sqlite3_total_changes(db);
650 inner_sqlstr = "UPDATE mandb_meta SET device = :device,"
651 " inode = :inode, mtime = :mtime WHERE"
652 " md5_hash = :md5 AND file = :file AND"
653 " (device <> :device2 OR inode <> "
654 " :inode2 OR mtime <> :mtime2)";
655 rc = sqlite3_prepare_v2(db, inner_sqlstr, -1, &inner_stmt, NULL);
656 if (rc != SQLITE_OK) {
657 if (mflags.verbosity)
658 warnx("%s", sqlite3_errmsg(db));
659 return;
660 }
661 idx = sqlite3_bind_parameter_index(inner_stmt, ":device");
662 sqlite3_bind_int64(inner_stmt, idx, rec->device);
663 idx = sqlite3_bind_parameter_index(inner_stmt, ":inode");
664 sqlite3_bind_int64(inner_stmt, idx, rec->inode);
665 idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime");
666 sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
667 idx = sqlite3_bind_parameter_index(inner_stmt, ":md5");
668 sqlite3_bind_text(inner_stmt, idx, hash, -1, NULL);
669 idx = sqlite3_bind_parameter_index(inner_stmt, ":file");
670 sqlite3_bind_text(inner_stmt, idx, file, -1, NULL);
671 idx = sqlite3_bind_parameter_index(inner_stmt, ":device2");
672 sqlite3_bind_int64(inner_stmt, idx, rec->device);
673 idx = sqlite3_bind_parameter_index(inner_stmt, ":inode2");
674 sqlite3_bind_int64(inner_stmt, idx, rec->inode);
675 idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime2");
676 sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
677
678 rc = sqlite3_step(inner_stmt);
679 if (rc == SQLITE_DONE) {
680 /* Check if an update has been performed. */
681 if (update_count != sqlite3_total_changes(db)) {
682 if (mflags.verbosity == 2)
683 printf("Updated %s\n", file);
684 (*new_count)++;
685 } else {
686 /* Otherwise it was a hardlink. */
687 (*link_count)++;
688 }
689 } else {
690 if (mflags.verbosity == 2)
691 warnx("Could not update the meta data for %s", file);
692 (*err_count)++;
693 }
694 sqlite3_finalize(inner_stmt);
695 }
696
697 /* update_db --
698 * Does an incremental updation of the database by checking the file_cache.
699 * It parses and adds the pages which are present in file_cache,
700 * but not in the database.
701 * It also removes the pages which are present in the databse,
702 * but not in the file_cache.
703 */
704 static void
705 update_db(sqlite3 *db, struct mparse *mp, mandb_rec *rec)
706 {
707 const char *sqlstr;
708 sqlite3_stmt *stmt = NULL;
709 char *file;
710 char *parent;
711 char *errmsg = NULL;
712 char *md5sum;
713 void *buf;
714 size_t buflen;
715 struct sql_row {
716 struct sql_row *next;
717 dev_t device;
718 ino_t inode;
719 time_t mtime;
720 char *parent;
721 char *file;
722 } *rows, *row;
723 int new_count = 0; /* Counter for newly indexed/updated pages */
724 int total_count = 0; /* Counter for total number of pages */
725 int err_count = 0; /* Counter for number of failed pages */
726 int link_count = 0; /* Counter for number of hard/sym links */
727 int md5_status;
728 int rc;
729
730 sqlstr = "SELECT device, inode, mtime, parent, file"
731 " FROM metadb.file_cache fc"
732 " WHERE NOT EXISTS(SELECT 1 FROM mandb_meta WHERE"
733 " device = fc.device AND inode = fc.inode AND "
734 " mtime = fc.mtime AND file = fc.file)";
735
736 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
737 if (rc != SQLITE_OK) {
738 if (mflags.verbosity)
739 warnx("%s", sqlite3_errmsg(db));
740 close_db(db);
741 errx(EXIT_FAILURE, "Could not query file cache");
742 }
743
744 buf = NULL;
745 rows = NULL;
746 while (sqlite3_step(stmt) == SQLITE_ROW) {
747 row = emalloc(sizeof(struct sql_row));
748 row->device = sqlite3_column_int64(stmt, 0);
749 row->inode = sqlite3_column_int64(stmt, 1);
750 row->mtime = sqlite3_column_int64(stmt, 2);
751 row->parent = estrdup((const char *) sqlite3_column_text(stmt, 3));
752 row->file = estrdup((const char *) sqlite3_column_text(stmt, 4));
753 row->next = rows;
754 rows = row;
755 total_count++;
756 }
757 sqlite3_finalize(stmt);
758
759 for ( ; rows != NULL; free(parent), free(file), free(buf)) {
760 int fd;
761
762 row = rows;
763 rows = rows->next;
764
765 rec->device = row->device;
766 rec->inode = row->inode;
767 rec->mtime = row->mtime;
768 parent = row->parent;
769 file = row->file;
770 free(row);
771
772 // XXX: reading twice!
773 if (read_and_decompress(file, &buf, &buflen)) {
774 err_count++;
775 continue;
776 }
777 if ((fd = mparse_open(mp, file)) == -1) {
778 err_count++;
779 continue;
780 }
781
782 md5_status = check_md5(file, db, &md5sum, buf, buflen);
783 assert(md5sum != NULL);
784 if (md5_status == -1) {
785 if (mflags.verbosity)
786 warnx("An error occurred in checking md5 value"
787 " for file %s", file);
788 err_count++;
789 close(fd);
790 continue;
791 }
792
793 if (md5_status == 0) {
794 /*
795 * The MD5 hash is already present in the database,
796 * so simply update the metadata.
797 */
798 update_existing_entry(db, file, md5sum, rec,
799 &new_count, &link_count, &err_count);
800 free(md5sum);
801 close(fd);
802 continue;
803 }
804
805 if (md5_status == 1) {
806 /*
807 * The MD5 hash was not present in the database.
808 * This means is either a new file or an updated file.
809 * We should go ahead with parsing.
810 */
811 if (chdir(parent) == -1) {
812 if (mflags.verbosity)
813 warn("chdir failed for `%s', could "
814 "not index `%s'", parent, file);
815 err_count++;
816 free(md5sum);
817 close(fd);
818 continue;
819 }
820
821 if (mflags.verbosity == 2)
822 printf("Parsing: %s\n", file);
823 rec->md5_hash = md5sum;
824 rec->file_path = estrdup(file);
825 // file_path is freed by insert_into_db itself.
826 begin_parse(file, mp, rec, fd);
827 if (insert_into_db(db, rec) < 0) {
828 if (mflags.verbosity)
829 warnx("Error in indexing `%s'", file);
830 err_count++;
831 } else {
832 new_count++;
833 }
834 }
835 close(fd);
836 }
837
838 if (mflags.verbosity == 2) {
839 printf("Number of new or updated pages encountered: %d\n"
840 "Number of hard links found: %d\n"
841 "Number of pages that were successfully"
842 " indexed or updated: %d\n"
843 "Number of pages that could not be indexed"
844 " due to errors: %d\n",
845 total_count - link_count, link_count, new_count, err_count);
846 }
847
848 if (mflags.recreate)
849 return;
850
851 if (mflags.verbosity == 2)
852 printf("Deleting stale index entries\n");
853
854 sqlstr = "DELETE FROM mandb_meta WHERE file NOT IN"
855 " (SELECT file FROM metadb.file_cache);"
856 "DELETE FROM mandb_links WHERE md5_hash NOT IN"
857 " (SELECT md5_hash from mandb_meta);"
858 "DROP TABLE metadb.file_cache;"
859 "DELETE FROM mandb WHERE rowid NOT IN"
860 " (SELECT id FROM mandb_meta);";
861
862 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
863 if (errmsg != NULL) {
864 warnx("Removing old entries failed: %s", errmsg);
865 warnx("Please rebuild database from scratch with -f.");
866 free(errmsg);
867 return;
868 }
869 }
870
871 /*
872 * begin_parse --
873 * parses the man page using libmandoc
874 */
875 static void
876 begin_parse(const char *file, struct mparse *mp, mandb_rec *rec, int fd)
877 {
878 struct roff_meta *roff;
879 mparse_reset(mp);
880
881 rec->xr_found = 0;
882
883 mparse_readfd(mp, fd, file);
884 roff = mparse_result(mp);
885 if (roff == NULL) {
886 if (mflags.verbosity == 2)
887 warnx("Not a roff(7) page");
888 return;
889 }
890
891 if (roff->macroset == MACROSET_MDOC) {
892 rec->page_type = MDOC;
893 proff_node(roff->first->child, rec, roff, mdocs);
894 } else if (roff->macroset == MACROSET_MAN) {
895 rec->page_type = MAN;
896 proff_node(roff->first->child, rec, roff, mans);
897 } else
898 warnx("Unknown macroset %d", roff->macroset);
899 set_machine(roff, rec);
900 set_section(roff, rec);
901 }
902
903 /*
904 * set_section --
905 * Extracts the section number and normalizes it to only the numeric part
906 * (Which should be the first character of the string).
907 */
908 static void
909 set_section(const struct roff_meta *rm, mandb_rec *rec)
910 {
911 if (!rm)
912 return;
913 const char *s = rm->msec == NULL ? "?" : rm->msec;
914 easprintf(&rec->section, "%s", s);
915 if (rec->section[0] == '?' && mflags.verbosity == 2)
916 warnx("%s: Missing section number", rec->file_path);
917 }
918
919 /*
920 * get_machine --
921 * Extracts the machine architecture information if available.
922 */
923 static void
924 set_machine(const struct roff_meta *rm, mandb_rec *rec)
925 {
926 if (rm == NULL)
927 return;
928 if (rm->arch)
929 rec->machine = estrdup(rm->arch);
930 }
931
932 /*
933 * pmdoc_Nm --
934 * Extracts the Name of the manual page from the .Nm macro
935 */
936 static void
937 pmdoc_Nm(const struct roff_node *n, mandb_rec *rec)
938 {
939 if (n->sec != SEC_NAME)
940 return;
941
942 for (n = n->child; n; n = n->next) {
943 if (n->type == ROFFT_TEXT) {
944 char *escaped_name = parse_escape(n->string);
945 concat(&rec->name, escaped_name);
946 free(escaped_name);
947 }
948 }
949 }
950
951 /*
952 * pmdoc_Nd --
953 * Extracts the one line description of the man page from the .Nd macro
954 */
955 static void
956 pmdoc_Nd(const struct roff_node *n, mandb_rec *rec)
957 {
958 if (n->type == ROFFT_BODY)
959 deroff(&rec->name_desc, n);
960 if (rec->name_desc)
961 replace_hyph(rec->name_desc);
962
963 }
964
965 /*
966 * pmdoc_macro_handler--
967 * This function is a single point of handling all the special macros that we
968 * want to handle especially. For example the .Xr macro for properly parsing
969 * the referenced page name along with the section number, or the .Pp macro
970 * for adding a new line whenever we encounter it.
971 */
972 static void
973 pmdoc_macro_handler(const struct roff_node *n, mandb_rec *rec, int doct)
974 {
975 const struct roff_node *sn;
976 assert(n);
977
978 switch (doct) {
979 /* Parse the man page references.
980 * Basically the .Xr macros are used like:
981 * .Xr ls 1
982 * and formatted like this:
983 * ls(1)
984 * Prepare a buffer to format the data like the above example and call
985 * pmdoc_parse_section to append it.
986 */
987 case MDOC_Xr:
988 n = n->child;
989 while (n->type != ROFFT_TEXT && n->next)
990 n = n->next;
991
992 if (n && n->type != ROFFT_TEXT)
993 return;
994 sn = n;
995 if (n->next)
996 n = n->next;
997
998 while (n->type != ROFFT_TEXT && n->next)
999 n = n->next;
1000
1001 if (n && n->type == ROFFT_TEXT) {
1002 char *buf;
1003 easprintf(&buf, "%s(%s)", sn->string, n->string);
1004 mdoc_parse_section(n->sec, buf, rec);
1005 free(buf);
1006 }
1007
1008 break;
1009
1010 /* Parse the .Pp macro to add a new line */
1011 case MDOC_Pp:
1012 if (n->type == ROFFT_TEXT)
1013 mdoc_parse_section(n->sec, "\n", rec);
1014 break;
1015 default:
1016 break;
1017 }
1018
1019 }
1020
1021 /*
1022 * pmdoc_Xr, pmdoc_Pp--
1023 * Empty stubs.
1024 * The parser calls these functions each time it encounters
1025 * a .Xr or .Pp macro. We are parsing all the data from
1026 * the pmdoc_Sh function, so don't do anything here.
1027 * (See if else blocks in pmdoc_Sh.)
1028 */
1029 static void
1030 pmdoc_Xr(const struct roff_node *n, mandb_rec *rec)
1031 {
1032 }
1033
1034 static void
1035 pmdoc_Pp(const struct roff_node *n, mandb_rec *rec)
1036 {
1037 }
1038
1039 /*
1040 * pmdoc_Sh --
1041 * Called when a .Sh macro is encountered and tries to parse its body
1042 */
1043 static void
1044 pmdoc_Sh(const struct roff_node *n, mandb_rec *rec)
1045 {
1046 if (n == NULL)
1047 return;
1048
1049 switch (n->sec) {
1050 case SEC_NAME:
1051 case SEC_SYNOPSIS:
1052 case SEC_EXAMPLES:
1053 case SEC_STANDARDS:
1054 case SEC_HISTORY:
1055 case SEC_AUTHORS:
1056 case SEC_BUGS:
1057 /*
1058 * We don't care about text from these sections
1059 */
1060 return;
1061 default:
1062 break;
1063 }
1064
1065 if (n->type == ROFFT_BLOCK)
1066 mdoc_parse_Sh(n->body, rec);
1067 }
1068
1069 /*
1070 * Called from pmdoc_Sh to parse body of a .Sh macro. It calls
1071 * mdoc_parse_section to append the data to the section specific buffer.
1072 * The .Xr macro needs special handling, thus the separate if branch for it.
1073 */
1074 static void
1075 mdoc_parse_Sh(const struct roff_node *n, mandb_rec *rec)
1076 {
1077 if (n == NULL || (n->type != ROFFT_TEXT && n->tok == MDOC_MAX))
1078 return;
1079 int xr_found = 0;
1080
1081 if (n->type == ROFFT_TEXT) {
1082 mdoc_parse_section(n->sec, n->string, rec);
1083 } else if (mdocs[n->tok] == pmdoc_Xr) {
1084 /*
1085 * When encountering other inline macros,
1086 * call pmdoc_macro_handler.
1087 */
1088 pmdoc_macro_handler(n, rec, MDOC_Xr);
1089 xr_found = 1;
1090 } else if (mdocs[n->tok] == pmdoc_Pp) {
1091 pmdoc_macro_handler(n, rec, MDOC_Pp);
1092 }
1093
1094 /*
1095 * If an Xr macro was encountered then the child node has
1096 * already been explored by pmdoc_macro_handler.
1097 */
1098 if (xr_found == 0)
1099 mdoc_parse_Sh(n->child, rec);
1100 mdoc_parse_Sh(n->next, rec);
1101 }
1102
1103 /*
1104 * mdoc_parse_section--
1105 * Utility function for parsing sections of the mdoc type pages.
1106 * Takes two params:
1107 * 1. sec is an enum which indicates the section in which we are present
1108 * 2. string is the string which we need to append to the secbuff for this
1109 * particular section.
1110 * The function appends string to the global section buffer and returns.
1111 */
1112 static void
1113 mdoc_parse_section(enum roff_sec sec, const char *string, mandb_rec *rec)
1114 {
1115 /*
1116 * If the user specified the 'l' flag, then parse and store only the
1117 * NAME section. Ignore the rest.
1118 */
1119 if (mflags.limit)
1120 return;
1121
1122 switch (sec) {
1123 case SEC_LIBRARY:
1124 append(&rec->lib, string);
1125 break;
1126 case SEC_RETURN_VALUES:
1127 append(&rec->return_vals, string);
1128 break;
1129 case SEC_ENVIRONMENT:
1130 append(&rec->env, string);
1131 break;
1132 case SEC_FILES:
1133 append(&rec->files, string);
1134 break;
1135 case SEC_EXIT_STATUS:
1136 append(&rec->exit_status, string);
1137 break;
1138 case SEC_DIAGNOSTICS:
1139 append(&rec->diagnostics, string);
1140 break;
1141 case SEC_ERRORS:
1142 append(&rec->errors, string);
1143 break;
1144 default:
1145 append(&rec->desc, string);
1146 break;
1147 }
1148 }
1149
1150 static void
1151 proff_node(const struct roff_node * n, mandb_rec * rec,
1152 struct roff_meta * roff, const proff_nf * func)
1153 {
1154 if (n == NULL)
1155 return;
1156
1157 int tok_idx;
1158
1159 switch (n->type) {
1160 case (ROFFT_BODY):
1161 /* FALLTHROUGH */
1162 case (ROFFT_BLOCK):
1163 /* FALLTHROUGH */
1164 case (ROFFT_ELEM):
1165 if (roff->macroset == MACROSET_MAN)
1166 tok_idx = n->tok - MAN_TH;
1167 else if (roff->macroset == MACROSET_MDOC)
1168 tok_idx = n->tok - MDOC_Dd;
1169 else
1170 tok_idx = -1;
1171 if (tok_idx >= 0 && func[tok_idx] != NULL)
1172 (*func[tok_idx]) (n, rec);
1173 break;
1174 default:
1175 break;
1176 }
1177
1178 proff_node(n->child, rec, roff, func);
1179 proff_node(n->next, rec, roff, func);
1180 }
1181
1182 /*
1183 * pman_parse_name --
1184 * Parses the NAME section and puts the complete content in the name_desc
1185 * variable.
1186 */
1187 static void
1188 pman_parse_name(const struct roff_node *n, mandb_rec *rec)
1189 {
1190 if (n == NULL)
1191 return;
1192
1193 if (n->type == ROFFT_TEXT) {
1194 char *tmp = parse_escape(n->string);
1195 concat(&rec->name_desc, tmp);
1196 free(tmp);
1197 }
1198
1199 if (n->child)
1200 pman_parse_name(n->child, rec);
1201
1202 if(n->next)
1203 pman_parse_name(n->next, rec);
1204 }
1205
1206 /*
1207 * A stub function to be able to parse the macros like .B embedded inside
1208 * a section.
1209 */
1210 static void
1211 pman_block(const struct roff_node *n, mandb_rec *rec)
1212 {
1213 }
1214
1215 /*
1216 * pman_sh --
1217 * This function does one of the two things:
1218 * 1. If the present section is NAME, then it will:
1219 * (a) Extract the name of the page (in case of multiple comma separated
1220 * names, it will pick up the first one).
1221 * (b) Build a space spearated list of all the symlinks/hardlinks to
1222 * this page and store in the buffer 'links'. These are extracted from
1223 * the comma separated list of names in the NAME section as well.
1224 * (c) Move on to the one line description section, which is after the list
1225 * of names in the NAME section.
1226 * 2. Otherwise, it will check the section name and call the man_parse_section
1227 * function, passing the enum corresponding to that section.
1228 */
1229 static void
1230 pman_sh(const struct roff_node *n, mandb_rec *rec)
1231 {
1232 static const struct {
1233 enum man_sec section;
1234 const char *header;
1235 } mapping[] = {
1236 { MANSEC_DESCRIPTION, "DESCRIPTION" },
1237 { MANSEC_SYNOPSIS, "SYNOPSIS" },
1238 { MANSEC_LIBRARY, "LIBRARY" },
1239 { MANSEC_ERRORS, "ERRORS" },
1240 { MANSEC_FILES, "FILES" },
1241 { MANSEC_RETURN_VALUES, "RETURN VALUE" },
1242 { MANSEC_RETURN_VALUES, "RETURN VALUES" },
1243 { MANSEC_EXIT_STATUS, "EXIT STATUS" },
1244 { MANSEC_EXAMPLES, "EXAMPLES" },
1245 { MANSEC_EXAMPLES, "EXAMPLE" },
1246 { MANSEC_STANDARDS, "STANDARDS" },
1247 { MANSEC_HISTORY, "HISTORY" },
1248 { MANSEC_BUGS, "BUGS" },
1249 { MANSEC_AUTHORS, "AUTHORS" },
1250 { MANSEC_COPYRIGHT, "COPYRIGHT" },
1251 };
1252 const struct roff_node *head;
1253 char *name_desc;
1254 size_t sz;
1255 size_t i;
1256
1257 if ((head = n->parent->head) == NULL || (head = head->child) == NULL ||
1258 head->type != ROFFT_TEXT)
1259 return;
1260
1261 /*
1262 * Check if this section should be extracted and
1263 * where it should be stored. Handled the trival cases first.
1264 */
1265 for (i = 0; i < sizeof(mapping) / sizeof(mapping[0]); ++i) {
1266 if (strcmp(head->string, mapping[i].header) == 0) {
1267 man_parse_section(mapping[i].section, n, rec);
1268 return;
1269 }
1270 }
1271
1272 if (strcmp(head->string, "NAME") == 0) {
1273 /*
1274 * We are in the NAME section.
1275 * pman_parse_name will put the complete content in name_desc.
1276 */
1277 pman_parse_name(n, rec);
1278
1279 name_desc = rec->name_desc;
1280 if (name_desc == NULL)
1281 return;
1282
1283 /* Remove any leading spaces. */
1284 while (name_desc[0] == ' ')
1285 name_desc++;
1286
1287 /* If the line begins with a "\&", avoid those */
1288 if (name_desc[0] == '\\' && name_desc[1] == '&')
1289 name_desc += 2;
1290
1291 /* Now name_desc should be left with a comma-space
1292 * separated list of names and the one line description
1293 * of the page:
1294 * "a, b, c \- sample description"
1295 * Take out the first name, before the first comma
1296 * (or space) and store it in rec->name.
1297 * If the page has aliases then they should be
1298 * in the form of a comma separated list.
1299 * Keep looping while there is a comma in name_desc,
1300 * extract the alias name and store in rec->links.
1301 * When there are no more commas left, break out.
1302 */
1303 int has_alias = 0; // Any more aliases left?
1304 while (*name_desc) {
1305 /* Remove any leading spaces or hyphens. */
1306 if (name_desc[0] == ' ' || name_desc[0] == '-') {
1307 name_desc++;
1308 continue;
1309 }
1310 sz = strcspn(name_desc, ", ");
1311
1312 /* Extract the first term and store it in rec->name. */
1313 if (rec->name == NULL) {
1314 if (name_desc[sz] == ',')
1315 has_alias = 1;
1316 rec->name = estrndup(name_desc, sz);
1317 /* XXX This would only happen with a poorly
1318 * written man page, maybe warn? */
1319 if (name_desc[sz] == '\0')
1320 break;
1321 name_desc += sz + 1;
1322 continue;
1323 }
1324
1325 /*
1326 * Once rec->name is set, rest of the names
1327 * are to be treated as links or aliases.
1328 */
1329 if (rec->name && has_alias) {
1330 if (name_desc[sz] != ',') {
1331 /* No more commas left --> no more
1332 * aliases to take out */
1333 has_alias = 0;
1334 }
1335 concat2(&rec->links, name_desc, sz);
1336 /* XXX This would only happen with a poorly
1337 * written man page, maybe warn? */
1338 if (name_desc[sz] == '\0')
1339 break;
1340 name_desc += sz + 1;
1341 continue;
1342 }
1343 break;
1344 }
1345
1346 /* Parse any escape sequences that might be there */
1347 char *temp = parse_escape(name_desc);
1348 free(rec->name_desc);
1349 rec->name_desc = temp;
1350 temp = parse_escape(rec->name);
1351 free(rec->name);
1352 rec->name = temp;
1353 return;
1354 }
1355
1356 /* The RETURN VALUE section might be specified in multiple ways */
1357 if (strcmp(head->string, "RETURN") == 0 &&
1358 head->next != NULL && head->next->type == ROFFT_TEXT &&
1359 (strcmp(head->next->string, "VALUE") == 0 ||
1360 strcmp(head->next->string, "VALUES") == 0)) {
1361 man_parse_section(MANSEC_RETURN_VALUES, n, rec);
1362 return;
1363 }
1364
1365 /*
1366 * EXIT STATUS section can also be specified all on one line or on two
1367 * separate lines.
1368 */
1369 if (strcmp(head->string, "EXIT") == 0 &&
1370 head->next != NULL && head->next->type == ROFFT_TEXT &&
1371 strcmp(head->next->string, "STATUS") == 0) {
1372 man_parse_section(MANSEC_EXIT_STATUS, n, rec);
1373 return;
1374 }
1375
1376 /* Store the rest of the content in desc. */
1377 man_parse_section(MANSEC_NONE, n, rec);
1378 }
1379
1380 /*
1381 * pman_parse_node --
1382 * Generic function to iterate through a node. Usually called from
1383 * man_parse_section to parse a particular section of the man page.
1384 */
1385 static void
1386 pman_parse_node(const struct roff_node *n, secbuff *s)
1387 {
1388 if (n == NULL)
1389 return;
1390
1391 if (n->type == ROFFT_TEXT)
1392 append(s, n->string);
1393
1394 pman_parse_node(n->child, s);
1395 pman_parse_node(n->next, s);
1396 }
1397
1398 /*
1399 * man_parse_section --
1400 * Takes two parameters:
1401 * sec: Tells which section we are present in
1402 * n: Is the present node of the AST.
1403 * Depending on the section, we call pman_parse_node to parse that section and
1404 * concatenate the content from that section into the buffer for that section.
1405 */
1406 static void
1407 man_parse_section(enum man_sec sec, const struct roff_node *n, mandb_rec *rec)
1408 {
1409 /*
1410 * If the user sepecified the 'l' flag then just parse
1411 * the NAME section, ignore the rest.
1412 */
1413 if (mflags.limit)
1414 return;
1415
1416 switch (sec) {
1417 case MANSEC_LIBRARY:
1418 pman_parse_node(n, &rec->lib);
1419 break;
1420 case MANSEC_RETURN_VALUES:
1421 pman_parse_node(n, &rec->return_vals);
1422 break;
1423 case MANSEC_ENVIRONMENT:
1424 pman_parse_node(n, &rec->env);
1425 break;
1426 case MANSEC_FILES:
1427 pman_parse_node(n, &rec->files);
1428 break;
1429 case MANSEC_EXIT_STATUS:
1430 pman_parse_node(n, &rec->exit_status);
1431 break;
1432 case MANSEC_DIAGNOSTICS:
1433 pman_parse_node(n, &rec->diagnostics);
1434 break;
1435 case MANSEC_ERRORS:
1436 pman_parse_node(n, &rec->errors);
1437 break;
1438 case MANSEC_NAME:
1439 case MANSEC_SYNOPSIS:
1440 case MANSEC_EXAMPLES:
1441 case MANSEC_STANDARDS:
1442 case MANSEC_HISTORY:
1443 case MANSEC_BUGS:
1444 case MANSEC_AUTHORS:
1445 case MANSEC_COPYRIGHT:
1446 break;
1447 default:
1448 pman_parse_node(n, &rec->desc);
1449 break;
1450 }
1451
1452 }
1453
1454 /*
1455 * insert_into_db --
1456 * Inserts the parsed data of the man page in the Sqlite databse.
1457 * If any of the values is NULL, then we cleanup and return -1 indicating
1458 * an error.
1459 * Otherwise, store the data in the database and return 0.
1460 */
1461 static int
1462 insert_into_db(sqlite3 *db, mandb_rec *rec)
1463 {
1464 int rc = 0;
1465 int idx = -1;
1466 const char *sqlstr = NULL;
1467 sqlite3_stmt *stmt = NULL;
1468 char *ln = NULL;
1469 char *errmsg = NULL;
1470 long int mandb_rowid;
1471
1472 /*
1473 * At the very minimum we want to make sure that we store
1474 * the following data:
1475 * Name, one line description, and the MD5 hash
1476 */
1477 if (rec->name == NULL || rec->name_desc == NULL ||
1478 rec->md5_hash == NULL) {
1479 cleanup(rec);
1480 return -1;
1481 }
1482
1483 /* Write null byte at the end of all the sec_buffs */
1484 rec->desc.data[rec->desc.offset] = 0;
1485 rec->lib.data[rec->lib.offset] = 0;
1486 rec->env.data[rec->env.offset] = 0;
1487 rec->return_vals.data[rec->return_vals.offset] = 0;
1488 rec->exit_status.data[rec->exit_status.offset] = 0;
1489 rec->files.data[rec->files.offset] = 0;
1490 rec->diagnostics.data[rec->diagnostics.offset] = 0;
1491 rec->errors.data[rec->errors.offset] = 0;
1492
1493 /*
1494 * In case of a mdoc page: (sorry, no better place to put this code)
1495 * parse the comma separated list of names of man pages,
1496 * the first name will be stored in the mandb table, rest will be
1497 * treated as links and put in the mandb_links table.
1498 */
1499 if (rec->page_type == MDOC) {
1500 char *tmp;
1501 rec->links = estrdup(rec->name);
1502 free(rec->name);
1503 size_t sz = strcspn(rec->links, " \0");
1504 rec->name = emalloc(sz + 1);
1505 memcpy(rec->name, rec->links, sz);
1506 if(rec->name[sz - 1] == ',')
1507 rec->name[sz - 1] = 0;
1508 else
1509 rec->name[sz] = 0;
1510 while (rec->links[sz] == ' ')
1511 ++sz;
1512 tmp = estrdup(rec->links + sz);
1513 free(rec->links);
1514 rec->links = tmp;
1515 }
1516
1517 /*------------------------ Populate the mandb table---------------------------*/
1518 sqlstr = "INSERT INTO mandb VALUES (:section, :name, :name_desc, :desc,"
1519 " :lib, :return_vals, :env, :files, :exit_status,"
1520 " :diagnostics, :errors, :md5_hash, :machine)";
1521
1522 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1523 if (rc != SQLITE_OK)
1524 goto Out;
1525
1526 idx = sqlite3_bind_parameter_index(stmt, ":name");
1527 rc = sqlite3_bind_text(stmt, idx, rec->name, -1, NULL);
1528 if (rc != SQLITE_OK) {
1529 sqlite3_finalize(stmt);
1530 goto Out;
1531 }
1532
1533 idx = sqlite3_bind_parameter_index(stmt, ":section");
1534 rc = sqlite3_bind_text(stmt, idx, rec->section, -1, NULL);
1535 if (rc != SQLITE_OK) {
1536 sqlite3_finalize(stmt);
1537 goto Out;
1538 }
1539
1540 idx = sqlite3_bind_parameter_index(stmt, ":name_desc");
1541 rc = sqlite3_bind_text(stmt, idx, rec->name_desc, -1, NULL);
1542 if (rc != SQLITE_OK) {
1543 sqlite3_finalize(stmt);
1544 goto Out;
1545 }
1546
1547 idx = sqlite3_bind_parameter_index(stmt, ":desc");
1548 rc = sqlite3_bind_text(stmt, idx, rec->desc.data,
1549 rec->desc.offset + 1, NULL);
1550 if (rc != SQLITE_OK) {
1551 sqlite3_finalize(stmt);
1552 goto Out;
1553 }
1554
1555 idx = sqlite3_bind_parameter_index(stmt, ":lib");
1556 rc = sqlite3_bind_text(stmt, idx, rec->lib.data,
1557 rec->lib.offset + 1, NULL);
1558 if (rc != SQLITE_OK) {
1559 sqlite3_finalize(stmt);
1560 goto Out;
1561 }
1562
1563 idx = sqlite3_bind_parameter_index(stmt, ":return_vals");
1564 rc = sqlite3_bind_text(stmt, idx, rec->return_vals.data,
1565 rec->return_vals.offset + 1, NULL);
1566 if (rc != SQLITE_OK) {
1567 sqlite3_finalize(stmt);
1568 goto Out;
1569 }
1570
1571 idx = sqlite3_bind_parameter_index(stmt, ":env");
1572 rc = sqlite3_bind_text(stmt, idx, rec->env.data,
1573 rec->env.offset + 1, NULL);
1574 if (rc != SQLITE_OK) {
1575 sqlite3_finalize(stmt);
1576 goto Out;
1577 }
1578
1579 idx = sqlite3_bind_parameter_index(stmt, ":files");
1580 rc = sqlite3_bind_text(stmt, idx, rec->files.data,
1581 rec->files.offset + 1, NULL);
1582 if (rc != SQLITE_OK) {
1583 sqlite3_finalize(stmt);
1584 goto Out;
1585 }
1586
1587 idx = sqlite3_bind_parameter_index(stmt, ":exit_status");
1588 rc = sqlite3_bind_text(stmt, idx, rec->exit_status.data,
1589 rec->exit_status.offset + 1, NULL);
1590 if (rc != SQLITE_OK) {
1591 sqlite3_finalize(stmt);
1592 goto Out;
1593 }
1594
1595 idx = sqlite3_bind_parameter_index(stmt, ":diagnostics");
1596 rc = sqlite3_bind_text(stmt, idx, rec->diagnostics.data,
1597 rec->diagnostics.offset + 1, NULL);
1598 if (rc != SQLITE_OK) {
1599 sqlite3_finalize(stmt);
1600 goto Out;
1601 }
1602
1603 idx = sqlite3_bind_parameter_index(stmt, ":errors");
1604 rc = sqlite3_bind_text(stmt, idx, rec->errors.data,
1605 rec->errors.offset + 1, NULL);
1606 if (rc != SQLITE_OK) {
1607 sqlite3_finalize(stmt);
1608 goto Out;
1609 }
1610
1611 idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1612 rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1613 if (rc != SQLITE_OK) {
1614 sqlite3_finalize(stmt);
1615 goto Out;
1616 }
1617
1618 idx = sqlite3_bind_parameter_index(stmt, ":machine");
1619 if (rec->machine)
1620 rc = sqlite3_bind_text(stmt, idx, rec->machine, -1, NULL);
1621 else
1622 rc = sqlite3_bind_null(stmt, idx);
1623 if (rc != SQLITE_OK) {
1624 sqlite3_finalize(stmt);
1625 goto Out;
1626 }
1627
1628 rc = sqlite3_step(stmt);
1629 if (rc != SQLITE_DONE) {
1630 sqlite3_finalize(stmt);
1631 goto Out;
1632 }
1633
1634 sqlite3_finalize(stmt);
1635
1636 /* Get the row id of the last inserted row */
1637 mandb_rowid = sqlite3_last_insert_rowid(db);
1638
1639 /*------------------------Populate the mandb_meta table-----------------------*/
1640 sqlstr = "INSERT INTO mandb_meta VALUES (:device, :inode, :mtime,"
1641 " :file, :md5_hash, :id)";
1642 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1643 if (rc != SQLITE_OK)
1644 goto Out;
1645
1646 idx = sqlite3_bind_parameter_index(stmt, ":device");
1647 rc = sqlite3_bind_int64(stmt, idx, rec->device);
1648 if (rc != SQLITE_OK) {
1649 sqlite3_finalize(stmt);
1650 goto Out;
1651 }
1652
1653 idx = sqlite3_bind_parameter_index(stmt, ":inode");
1654 rc = sqlite3_bind_int64(stmt, idx, rec->inode);
1655 if (rc != SQLITE_OK) {
1656 sqlite3_finalize(stmt);
1657 goto Out;
1658 }
1659
1660 idx = sqlite3_bind_parameter_index(stmt, ":mtime");
1661 rc = sqlite3_bind_int64(stmt, idx, rec->mtime);
1662 if (rc != SQLITE_OK) {
1663 sqlite3_finalize(stmt);
1664 goto Out;
1665 }
1666
1667 idx = sqlite3_bind_parameter_index(stmt, ":file");
1668 rc = sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
1669 if (rc != SQLITE_OK) {
1670 sqlite3_finalize(stmt);
1671 goto Out;
1672 }
1673
1674 idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1675 rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1676 if (rc != SQLITE_OK) {
1677 sqlite3_finalize(stmt);
1678 goto Out;
1679 }
1680
1681 idx = sqlite3_bind_parameter_index(stmt, ":id");
1682 rc = sqlite3_bind_int64(stmt, idx, mandb_rowid);
1683 if (rc != SQLITE_OK) {
1684 sqlite3_finalize(stmt);
1685 goto Out;
1686 }
1687
1688 rc = sqlite3_step(stmt);
1689 sqlite3_finalize(stmt);
1690 if (rc == SQLITE_CONSTRAINT_UNIQUE) {
1691 /* The *most* probable reason for reaching here is that
1692 * the UNIQUE contraint on the file column of the mandb_meta
1693 * table was violated.
1694 * This can happen when a file was updated/modified.
1695 * To fix this we need to do two things:
1696 * 1. Delete the row for the older version of this file
1697 * from mandb table.
1698 * 2. Run an UPDATE query to update the row for this file
1699 * in the mandb_meta table.
1700 */
1701 warnx("Trying to update index for %s", rec->file_path);
1702 char *sql = sqlite3_mprintf("DELETE FROM mandb "
1703 "WHERE rowid = (SELECT id"
1704 " FROM mandb_meta"
1705 " WHERE file = %Q)",
1706 rec->file_path);
1707 sqlite3_exec(db, sql, NULL, NULL, &errmsg);
1708 sqlite3_free(sql);
1709 if (errmsg != NULL) {
1710 if (mflags.verbosity)
1711 warnx("%s", errmsg);
1712 free(errmsg);
1713 }
1714 sqlstr = "UPDATE mandb_meta SET device = :device,"
1715 " inode = :inode, mtime = :mtime, id = :id,"
1716 " md5_hash = :md5 WHERE file = :file";
1717 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1718 if (rc != SQLITE_OK) {
1719 if (mflags.verbosity)
1720 warnx("Update failed with error: %s",
1721 sqlite3_errmsg(db));
1722 close_db(db);
1723 cleanup(rec);
1724 errx(EXIT_FAILURE,
1725 "Consider running makemandb with -f option");
1726 }
1727
1728 idx = sqlite3_bind_parameter_index(stmt, ":device");
1729 sqlite3_bind_int64(stmt, idx, rec->device);
1730 idx = sqlite3_bind_parameter_index(stmt, ":inode");
1731 sqlite3_bind_int64(stmt, idx, rec->inode);
1732 idx = sqlite3_bind_parameter_index(stmt, ":mtime");
1733 sqlite3_bind_int64(stmt, idx, rec->mtime);
1734 idx = sqlite3_bind_parameter_index(stmt, ":id");
1735 sqlite3_bind_int64(stmt, idx, mandb_rowid);
1736 idx = sqlite3_bind_parameter_index(stmt, ":md5");
1737 sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1738 idx = sqlite3_bind_parameter_index(stmt, ":file");
1739 sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
1740 rc = sqlite3_step(stmt);
1741 sqlite3_finalize(stmt);
1742
1743 if (rc != SQLITE_DONE) {
1744 if (mflags.verbosity)
1745 warnx("%s", sqlite3_errmsg(db));
1746 close_db(db);
1747 cleanup(rec);
1748 errx(EXIT_FAILURE,
1749 "Consider running makemandb with -f option");
1750 }
1751 } else if (rc != SQLITE_DONE) {
1752 /* Otherwise make this error fatal */
1753 warnx("Failed at %s\n%s", rec->file_path, sqlite3_errmsg(db));
1754 cleanup(rec);
1755 close_db(db);
1756 exit(EXIT_FAILURE);
1757 }
1758
1759 /*------------------------ Populate the mandb_links table---------------------*/
1760 char *str = NULL;
1761 char *links;
1762 if (rec->links && strlen(rec->links)) {
1763 links = rec->links;
1764 for(ln = strtok(links, " "); ln; ln = strtok(NULL, " ")) {
1765 if (ln[0] == ',')
1766 ln++;
1767 if(ln[strlen(ln) - 1] == ',')
1768 ln[strlen(ln) - 1] = 0;
1769
1770 str = sqlite3_mprintf("INSERT INTO mandb_links"
1771 " VALUES (%Q, %Q, %Q, %Q, %Q)",
1772 ln, rec->name, rec->section,
1773 rec->machine, rec->md5_hash);
1774 sqlite3_exec(db, str, NULL, NULL, &errmsg);
1775 sqlite3_free(str);
1776 if (errmsg != NULL) {
1777 warnx("%s", errmsg);
1778 cleanup(rec);
1779 free(errmsg);
1780 return -1;
1781 }
1782 }
1783 }
1784
1785 cleanup(rec);
1786 return 0;
1787
1788 Out:
1789 if (mflags.verbosity)
1790 warnx("%s", sqlite3_errmsg(db));
1791 cleanup(rec);
1792 return -1;
1793 }
1794
1795 /*
1796 * check_md5--
1797 * Generates the md5 hash of the file and checks if it already doesn't exist
1798 * in the table.
1799 * This function is being used to avoid hardlinks.
1800 * On successful completion it will also set the value of the fourth parameter
1801 * to the md5 hash of the file (computed previously). It is the responsibility
1802 * of the caller to free this buffer.
1803 * Return values:
1804 * -1: If an error occurs somewhere and sets the md5 return buffer to NULL.
1805 * 0: If the md5 hash does not exist in the table.
1806 * 1: If the hash exists in the database.
1807 */
1808 static int
1809 check_md5(const char *file, sqlite3 *db, char **md5, void *buf, size_t buflen)
1810 {
1811 int rc = 0;
1812 int idx = -1;
1813 char *sqlstr = NULL;
1814 char *mymd5;
1815 sqlite3_stmt *stmt = NULL;
1816 *md5 = NULL;
1817
1818 assert(file != NULL);
1819 if ((mymd5 = MD5Data(buf, buflen, NULL)) == NULL) {
1820 if (mflags.verbosity)
1821 warn("md5 failed: %s", file);
1822 return -1;
1823 }
1824
1825 easprintf(&sqlstr, "SELECT * FROM mandb_meta WHERE md5_hash = :md5_hash");
1826 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1827 if (rc != SQLITE_OK) {
1828 free(sqlstr);
1829 free(mymd5);
1830 return -1;
1831 }
1832
1833 idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1834 rc = sqlite3_bind_text(stmt, idx, mymd5, -1, NULL);
1835 if (rc != SQLITE_OK) {
1836 if (mflags.verbosity)
1837 warnx("%s", sqlite3_errmsg(db));
1838 sqlite3_finalize(stmt);
1839 free(sqlstr);
1840 free(mymd5);
1841 return -1;
1842 }
1843
1844 *md5 = mymd5;
1845 if (sqlite3_step(stmt) == SQLITE_ROW) {
1846 sqlite3_finalize(stmt);
1847 free(sqlstr);
1848 return 0;
1849 }
1850
1851 sqlite3_finalize(stmt);
1852 free(sqlstr);
1853 return 1;
1854 }
1855
1856 /* Optimize the index for faster search */
1857 static void
1858 optimize(sqlite3 *db)
1859 {
1860 const char *sqlstr;
1861 char *errmsg = NULL;
1862
1863 if (mflags.verbosity == 2)
1864 printf("Optimizing the database index\n");
1865 sqlstr = "INSERT INTO mandb(mandb) VALUES (\'optimize\');"
1866 "VACUUM";
1867 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
1868 if (errmsg != NULL) {
1869 if (mflags.verbosity)
1870 warnx("%s", errmsg);
1871 free(errmsg);
1872 return;
1873 }
1874 }
1875
1876 /*
1877 * cleanup --
1878 * cleans up the global buffers
1879 */
1880 static void
1881 cleanup(mandb_rec *rec)
1882 {
1883 rec->desc.offset = 0;
1884 rec->lib.offset = 0;
1885 rec->return_vals.offset = 0;
1886 rec->env.offset = 0;
1887 rec->exit_status.offset = 0;
1888 rec->diagnostics.offset = 0;
1889 rec->errors.offset = 0;
1890 rec->files.offset = 0;
1891
1892 free(rec->machine);
1893 rec->machine = NULL;
1894
1895 free(rec->links);
1896 rec->links = NULL;
1897
1898 free(rec->file_path);
1899 rec->file_path = NULL;
1900
1901 free(rec->name);
1902 rec->name = NULL;
1903
1904 free(rec->name_desc);
1905 rec->name_desc = NULL;
1906
1907 free(rec->md5_hash);
1908 rec->md5_hash = NULL;
1909
1910 free(rec->section);
1911 rec->section = NULL;
1912 }
1913
1914 /*
1915 * init_secbuffs--
1916 * Sets the value of buflen for all the sec_buff field of rec. And then
1917 * allocate memory to each sec_buff member of rec.
1918 */
1919 static void
1920 init_secbuffs(mandb_rec *rec)
1921 {
1922 /*
1923 * Some sec_buff might need more memory, for example desc,
1924 * which stores the data of the DESCRIPTION section,
1925 * while some might need very small amount of memory.
1926 * Therefore explicitly setting the value of buflen field for
1927 * each sec_buff.
1928 */
1929 rec->desc.buflen = 10 * BUFLEN;
1930 rec->desc.data = emalloc(rec->desc.buflen);
1931 rec->desc.offset = 0;
1932
1933 rec->lib.buflen = BUFLEN / 2;
1934 rec->lib.data = emalloc(rec->lib.buflen);
1935 rec->lib.offset = 0;
1936
1937 rec->return_vals.buflen = BUFLEN;
1938 rec->return_vals.data = emalloc(rec->return_vals.buflen);
1939 rec->return_vals.offset = 0;
1940
1941 rec->exit_status.buflen = BUFLEN;
1942 rec->exit_status.data = emalloc(rec->exit_status.buflen);
1943 rec->exit_status.offset = 0;
1944
1945 rec->env.buflen = BUFLEN;
1946 rec->env.data = emalloc(rec->env.buflen);
1947 rec->env.offset = 0;
1948
1949 rec->files.buflen = BUFLEN;
1950 rec->files.data = emalloc(rec->files.buflen);
1951 rec->files.offset = 0;
1952
1953 rec->diagnostics.buflen = BUFLEN;
1954 rec->diagnostics.data = emalloc(rec->diagnostics.buflen);
1955 rec->diagnostics.offset = 0;
1956
1957 rec->errors.buflen = BUFLEN;
1958 rec->errors.data = emalloc(rec->errors.buflen);
1959 rec->errors.offset = 0;
1960 }
1961
1962 /*
1963 * free_secbuffs--
1964 * This function should be called at the end, when all the pages have been
1965 * parsed.
1966 * It frees the memory allocated to sec_buffs by init_secbuffs in the starting.
1967 */
1968 static void
1969 free_secbuffs(mandb_rec *rec)
1970 {
1971 free(rec->desc.data);
1972 free(rec->lib.data);
1973 free(rec->return_vals.data);
1974 free(rec->exit_status.data);
1975 free(rec->env.data);
1976 free(rec->files.data);
1977 free(rec->diagnostics.data);
1978 free(rec->errors.data);
1979 }
1980
1981 static void
1982 replace_hyph(char *str)
1983 {
1984 char *iter = str;
1985 while ((iter = strchr(iter, ASCII_HYPH)) != NULL)
1986 *iter = '-';
1987
1988 iter = str;
1989 while ((iter = strchr(iter, ASCII_NBRSP)) != NULL)
1990 *iter = '-';
1991 }
1992
1993 static char *
1994 parse_escape(const char *str)
1995 {
1996 const char *backslash, *last_backslash;
1997 char *result, *iter;
1998 size_t len;
1999
2000 assert(str);
2001
2002 last_backslash = str;
2003 backslash = strchr(str, '\\');
2004 if (backslash == NULL) {
2005 result = estrdup(str);
2006 replace_hyph(result);
2007 return result;
2008 }
2009
2010 result = emalloc(strlen(str) + 1);
2011 iter = result;
2012
2013 do {
2014 len = backslash - last_backslash;
2015 memcpy(iter, last_backslash, len);
2016 iter += len;
2017 if (backslash[1] == '-' || backslash[1] == ' ') {
2018 *iter++ = backslash[1];
2019 last_backslash = backslash + 2;
2020 backslash = strchr(last_backslash, '\\');
2021 } else {
2022 ++backslash;
2023 mandoc_escape(&backslash, NULL, NULL);
2024 last_backslash = backslash;
2025 if (backslash == NULL)
2026 break;
2027 backslash = strchr(last_backslash, '\\');
2028 }
2029 } while (backslash != NULL);
2030 if (last_backslash != NULL)
2031 strcpy(iter, last_backslash);
2032
2033 replace_hyph(result);
2034 return result;
2035 }
2036
2037 /*
2038 * append--
2039 * Concatenates a space and src at the end of sbuff->data (much like concat in
2040 * apropos-utils.c).
2041 * Rather than reallocating space for writing data, it uses the value of the
2042 * offset field of sec_buff to write new data at the free space left in the
2043 * buffer.
2044 * In case the size of the data to be appended exceeds the number of bytes left
2045 * in the buffer, it reallocates buflen number of bytes and then continues.
2046 * Value of offset field should be adjusted as new data is written.
2047 *
2048 * NOTE: This function does not write the null byte at the end of the buffers,
2049 * write a null byte at the position pointed to by offset before inserting data
2050 * in the db.
2051 */
2052 static void
2053 append(secbuff *sbuff, const char *src)
2054 {
2055 short flag = 0;
2056 size_t srclen, newlen;
2057 char *temp;
2058
2059 assert(src != NULL);
2060 temp = parse_escape(src);
2061 srclen = strlen(temp);
2062
2063 if (sbuff->data == NULL) {
2064 sbuff->data = emalloc(sbuff->buflen);
2065 sbuff->offset = 0;
2066 }
2067
2068 newlen = sbuff->offset + srclen + 2;
2069 if (newlen >= sbuff->buflen) {
2070 while (sbuff->buflen < newlen)
2071 sbuff->buflen += sbuff->buflen;
2072 sbuff->data = erealloc(sbuff->data, sbuff->buflen);
2073 flag = 1;
2074 }
2075
2076 /* Append a space at the end of the buffer. */
2077 if (sbuff->offset || flag)
2078 sbuff->data[sbuff->offset++] = ' ';
2079 /* Now, copy src at the end of the buffer. */
2080 memcpy(sbuff->data + sbuff->offset, temp, srclen);
2081 sbuff->offset += srclen;
2082 free(temp);
2083 }
2084
2085 static void
2086 usage(void)
2087 {
2088 fprintf(stderr, "Usage: %s [-floQqv] [-C path]\n", getprogname());
2089 exit(1);
2090 }
2091