makemandb.c revision 1.60 1 /* $NetBSD: makemandb.c,v 1.60 2019/05/18 07:56:43 abhinav Exp $ */
2 /*
3 * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay (at) gmail.com>
4 * Copyright (c) 2011 Kristaps Dzonsons <kristaps (at) bsd.lv>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19 #include <sys/cdefs.h>
20 __RCSID("$NetBSD: makemandb.c,v 1.60 2019/05/18 07:56:43 abhinav Exp $");
21
22 #include <sys/stat.h>
23 #include <sys/types.h>
24
25 #include <assert.h>
26 #include <dirent.h>
27 #include <err.h>
28 #include <archive.h>
29 #include <libgen.h>
30 #include <md5.h>
31 #include <stdio.h>
32 #include <stdlib.h>
33 #include <string.h>
34 #include <unistd.h>
35 #include <util.h>
36
37 #include "apropos-utils.h"
38 #include "dist/man.h"
39 #include "dist/mandoc.h"
40 #include "dist/mdoc.h"
41 #include "dist/roff.h"
42 #include "dist/mandoc_parse.h"
43
44 #define BUFLEN 1024
45 #define MDOC 0 //If the page is of mdoc(7) type
46 #define MAN 1 //If the page is of man(7) type
47
48 /*
49 * A data structure for holding section specific data.
50 */
51 typedef struct secbuff {
52 char *data;
53 size_t buflen; //Total length of buffer allocated initially
54 size_t offset; // Current offset in the buffer.
55 } secbuff;
56
57 typedef struct makemandb_flags {
58 int optimize;
59 int limit; // limit the indexing to only NAME section
60 int recreate; // Database was created from scratch
61 int verbosity; // 0: quiet, 1: default, 2: verbose
62 } makemandb_flags;
63
64 typedef struct roff_mandb_rec {
65 /* Fields for mandb table */
66 char *name; // for storing the name of the man page
67 char *name_desc; // for storing the one line description (.Nd)
68 secbuff desc; // for storing the DESCRIPTION section
69 secbuff lib; // for the LIBRARY section
70 secbuff return_vals; // RETURN VALUES
71 secbuff env; // ENVIRONMENT
72 secbuff files; // FILES
73 secbuff exit_status; // EXIT STATUS
74 secbuff diagnostics; // DIAGNOSTICS
75 secbuff errors; // ERRORS
76 char *section;
77
78 int xr_found; // To track whether a .Xr was seen when parsing a section
79
80 /* Fields for mandb_meta table */
81 char *md5_hash;
82 dev_t device;
83 ino_t inode;
84 time_t mtime;
85
86 /* Fields for mandb_links table */
87 char *machine;
88 char *links; //all the links to a page in a space separated form
89 char *file_path;
90
91 /* Non-db fields */
92 int page_type; //Indicates the type of page: mdoc or man
93 } mandb_rec;
94
95 typedef void (*proff_nf)(const struct roff_node *n, mandb_rec *);
96
97 static void append(secbuff *sbuff, const char *src);
98 static void init_secbuffs(mandb_rec *);
99 static void free_secbuffs(mandb_rec *);
100 static int check_md5(const char *, sqlite3 *, char **, void *, size_t);
101 static void cleanup(mandb_rec *);
102 static void set_section(const struct roff_meta *, mandb_rec *);
103 static void set_machine(const struct roff_meta *, mandb_rec *);
104 static int insert_into_db(sqlite3 *, mandb_rec *);
105 static void begin_parse(const char *, struct mparse *, mandb_rec *, int);
106 static void proff_node(const struct roff_node *, mandb_rec *,
107 struct roff_meta *, const proff_nf *);
108 static void pmdoc_Nm(const struct roff_node *, mandb_rec *);
109 static void pmdoc_Nd(const struct roff_node *, mandb_rec *);
110 static void pmdoc_Sh(const struct roff_node *, mandb_rec *);
111 static void mdoc_parse_Sh(const struct roff_node *, mandb_rec *);
112 static void pmdoc_Xr(const struct roff_node *, mandb_rec *);
113 static void pmdoc_Pp(const struct roff_node *, mandb_rec *);
114 static void pmdoc_macro_handler(const struct roff_node *, mandb_rec *, int);
115 static void pman_parse_node(const struct roff_node *, secbuff *);
116 static void pman_parse_name(const struct roff_node *, mandb_rec *);
117 static void pman_sh(const struct roff_node *, mandb_rec *);
118 static void pman_block(const struct roff_node *, mandb_rec *);
119 static void traversedir(const char *, const char *, sqlite3 *, struct mparse *);
120 static void mdoc_parse_section(enum roff_sec, const char *, mandb_rec *);
121 static void man_parse_section(enum man_sec, const struct roff_node *, mandb_rec *);
122 static void build_file_cache(sqlite3 *, const char *, const char *,
123 struct stat *);
124 static void update_db(sqlite3 *, struct mparse *, mandb_rec *);
125 __dead static void usage(void);
126 static void optimize(sqlite3 *);
127 static char *parse_escape(const char *);
128 static void replace_hyph(char *);
129 static makemandb_flags mflags = { .verbosity = 1 };
130
131 static const proff_nf mdocs[MDOC_MAX - MDOC_Dd] = {
132 NULL, /* Dd */
133 NULL, /* Dt */
134 NULL, /* Os */
135 pmdoc_Sh, /* Sh */
136 NULL, /* Ss */
137 pmdoc_Pp, /* Pp */
138 NULL, /* D1 */
139 NULL, /* Dl */
140 NULL, /* Bd */
141 NULL, /* Ed */
142 NULL, /* Bl */
143 NULL, /* El */
144 NULL, /* It */
145 NULL, /* Ad */
146 NULL, /* An */
147 NULL, /* Ap */
148 NULL, /* Ar */
149 NULL, /* Cd */
150 NULL, /* Cm */
151 NULL, /* Dv */
152 NULL, /* Er */
153 NULL, /* Ev */
154 NULL, /* Ex */
155 NULL, /* Fa */
156 NULL, /* Fd */
157 NULL, /* Fl */
158 NULL, /* Fn */
159 NULL, /* Ft */
160 NULL, /* Ic */
161 NULL, /* In */
162 NULL, /* Li */
163 pmdoc_Nd, /* Nd */
164 pmdoc_Nm, /* Nm */
165 NULL, /* Op */
166 NULL, /* Ot */
167 NULL, /* Pa */
168 NULL, /* Rv */
169 NULL, /* St */
170 NULL, /* Va */
171 NULL, /* Vt */
172 pmdoc_Xr, /* Xr */
173 NULL, /* %A */
174 NULL, /* %B */
175 NULL, /* %D */
176 NULL, /* %I */
177 NULL, /* %J */
178 NULL, /* %N */
179 NULL, /* %O */
180 NULL, /* %P */
181 NULL, /* %R */
182 NULL, /* %T */
183 NULL, /* %V */
184 NULL, /* Ac */
185 NULL, /* Ao */
186 NULL, /* Aq */
187 NULL, /* At */
188 NULL, /* Bc */
189 NULL, /* Bf */
190 NULL, /* Bo */
191 NULL, /* Bq */
192 NULL, /* Bsx */
193 NULL, /* Bx */
194 NULL, /* Db */
195 NULL, /* Dc */
196 NULL, /* Do */
197 NULL, /* Dq */
198 NULL, /* Ec */
199 NULL, /* Ef */
200 NULL, /* Em */
201 NULL, /* Eo */
202 NULL, /* Fx */
203 NULL, /* Ms */
204 NULL, /* No */
205 NULL, /* Ns */
206 NULL, /* Nx */
207 NULL, /* Ox */
208 NULL, /* Pc */
209 NULL, /* Pf */
210 NULL, /* Po */
211 NULL, /* Pq */
212 NULL, /* Qc */
213 NULL, /* Ql */
214 NULL, /* Qo */
215 NULL, /* Qq */
216 NULL, /* Re */
217 NULL, /* Rs */
218 NULL, /* Sc */
219 NULL, /* So */
220 NULL, /* Sq */
221 NULL, /* Sm */
222 NULL, /* Sx */
223 NULL, /* Sy */
224 NULL, /* Tn */
225 NULL, /* Ux */
226 NULL, /* Xc */
227 NULL, /* Xo */
228 NULL, /* Fo */
229 NULL, /* Fc */
230 NULL, /* Oo */
231 NULL, /* Oc */
232 NULL, /* Bk */
233 NULL, /* Ek */
234 NULL, /* Bt */
235 NULL, /* Hf */
236 NULL, /* Fr */
237 NULL, /* Ud */
238 NULL, /* Lb */
239 NULL, /* Lp */
240 NULL, /* Lk */
241 NULL, /* Mt */
242 NULL, /* Brq */
243 NULL, /* Bro */
244 NULL, /* Brc */
245 NULL, /* %C */
246 NULL, /* Es */
247 NULL, /* En */
248 NULL, /* Dx */
249 NULL, /* %Q */
250 NULL, /* %U */
251 NULL /* Ta */
252 };
253
254 static const proff_nf mans[MAN_MAX - MAN_TH] = {
255 NULL, //TH
256 pman_sh, //SH
257 NULL, //SS
258 NULL, //TP
259 NULL, //LP
260 NULL, //PP
261 NULL, //P
262 NULL, //IP
263 NULL, //HP
264 NULL, //SM
265 NULL, //SB
266 NULL, //BI
267 NULL, //IB
268 NULL, //BR
269 NULL, //RB
270 NULL, //R
271 pman_block, //B
272 NULL, //I
273 NULL, //IR
274 NULL, //RI
275 NULL, //nf
276 NULL, //fi
277 NULL, //RE
278 NULL, //RS
279 NULL, //DT
280 NULL, //UC
281 NULL, //PD
282 NULL, //AT
283 NULL, //in
284 NULL, //OP
285 NULL, //EX
286 NULL, //EE
287 NULL, //UR
288 NULL, //UE
289 NULL, //MT
290 NULL //ME
291 };
292
293 int
294 main(int argc, char *argv[])
295 {
296 FILE *file;
297 const char *sqlstr, *manconf = NULL;
298 char *line, *command;
299 char *errmsg;
300 int ch;
301 struct mparse *mp;
302 sqlite3 *db;
303 ssize_t len;
304 size_t linesize;
305 struct roff_mandb_rec rec;
306
307 while ((ch = getopt(argc, argv, "C:floQqv")) != -1) {
308 switch (ch) {
309 case 'C':
310 manconf = optarg;
311 break;
312 case 'f':
313 mflags.recreate = 1;
314 break;
315 case 'l':
316 mflags.limit = 1;
317 break;
318 case 'o':
319 mflags.optimize = 1;
320 break;
321 case 'Q':
322 mflags.verbosity = 0;
323 break;
324 case 'q':
325 mflags.verbosity = 1;
326 break;
327 case 'v':
328 mflags.verbosity = 2;
329 break;
330 default:
331 usage();
332 }
333 }
334
335 memset(&rec, 0, sizeof(rec));
336
337 init_secbuffs(&rec);
338 mchars_alloc();
339 mp = mparse_alloc(MPARSE_SO | MPARSE_UTF8 | MPARSE_LATIN1 |
340 MPARSE_VALIDATE, MANDOC_OS_OTHER, NULL);
341
342 if (manconf) {
343 char *arg;
344 size_t command_len = shquote(manconf, NULL, 0) + 1;
345 arg = emalloc(command_len);
346 shquote(manconf, arg, command_len);
347 easprintf(&command, "man -p -C %s", arg);
348 free(arg);
349 } else {
350 command = estrdup("man -p");
351 manconf = MANCONF;
352 }
353
354 if (mflags.recreate) {
355 char *dbp = get_dbpath(manconf);
356 /* No error here, it will fail in init_db in the same call */
357 if (dbp != NULL)
358 remove(dbp);
359 }
360
361 if ((db = init_db(MANDB_CREATE, manconf)) == NULL)
362 exit(EXIT_FAILURE);
363
364 sqlite3_exec(db, "PRAGMA synchronous = 0", NULL, NULL, &errmsg);
365 if (errmsg != NULL) {
366 warnx("%s", errmsg);
367 free(errmsg);
368 close_db(db);
369 exit(EXIT_FAILURE);
370 }
371
372 sqlite3_exec(db, "ATTACH DATABASE \':memory:\' AS metadb", NULL, NULL,
373 &errmsg);
374 if (errmsg != NULL) {
375 warnx("%s", errmsg);
376 free(errmsg);
377 close_db(db);
378 exit(EXIT_FAILURE);
379 }
380
381
382 /* Call man -p to get the list of man page dirs */
383 if ((file = popen(command, "r")) == NULL) {
384 close_db(db);
385 err(EXIT_FAILURE, "fopen failed");
386 }
387 free(command);
388
389 /* Begin the transaction for indexing the pages */
390 sqlite3_exec(db, "BEGIN", NULL, NULL, &errmsg);
391 if (errmsg != NULL) {
392 warnx("%s", errmsg);
393 free(errmsg);
394 close_db(db);
395 exit(EXIT_FAILURE);
396 }
397
398 sqlstr = "CREATE TABLE metadb.file_cache(device, inode, mtime, parent,"
399 " file PRIMARY KEY);"
400 "CREATE UNIQUE INDEX metadb.index_file_cache_dev"
401 " ON file_cache (device, inode)";
402
403 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
404 if (errmsg != NULL) {
405 warnx("%s", errmsg);
406 free(errmsg);
407 close_db(db);
408 exit(EXIT_FAILURE);
409 }
410
411 if (mflags.verbosity)
412 printf("Building temporary file cache\n");
413 line = NULL;
414 linesize = 0;
415 while ((len = getline(&line, &linesize, file)) != -1) {
416 /* Replace the new line character at the end of string with '\0' */
417 line[len - 1] = '\0';
418 char *pdir = estrdup(dirname(line));
419 /* Traverse the man page directories and parse the pages */
420 traversedir(pdir, line, db, mp);
421 free(pdir);
422 }
423 free(line);
424
425 if (pclose(file) == -1) {
426 close_db(db);
427 cleanup(&rec);
428 free_secbuffs(&rec);
429 err(EXIT_FAILURE, "pclose error");
430 }
431
432 if (mflags.verbosity)
433 printf("Performing index update\n");
434 update_db(db, mp, &rec);
435 mparse_free(mp);
436 mchars_free();
437 free_secbuffs(&rec);
438
439 /* Commit the transaction */
440 sqlite3_exec(db, "COMMIT", NULL, NULL, &errmsg);
441 if (errmsg != NULL) {
442 warnx("%s", errmsg);
443 free(errmsg);
444 close_db(db);
445 exit(EXIT_FAILURE);
446 }
447
448 if (mflags.optimize)
449 optimize(db);
450
451 close_db(db);
452 return 0;
453 }
454
455 /*
456 * traversedir --
457 * Traverses the given directory recursively and passes all the man page files
458 * in the way to build_file_cache()
459 */
460 static void
461 traversedir(const char *parent, const char *file, sqlite3 *db,
462 struct mparse *mp)
463 {
464 struct stat sb;
465 struct dirent *dirp;
466 DIR *dp;
467 char *buf;
468
469 if (stat(file, &sb) < 0) {
470 if (mflags.verbosity)
471 warn("stat failed: %s", file);
472 return;
473 }
474
475 /* If it is a directory, traverse it recursively */
476 if (S_ISDIR(sb.st_mode)) {
477 if ((dp = opendir(file)) == NULL) {
478 if (mflags.verbosity)
479 warn("opendir error: %s", file);
480 return;
481 }
482
483 while ((dirp = readdir(dp)) != NULL) {
484 /* Avoid . and .. entries in a directory */
485 if (dirp->d_name[0] != '.') {
486 easprintf(&buf, "%s/%s", file, dirp->d_name);
487 traversedir(parent, buf, db, mp);
488 free(buf);
489 }
490 }
491 closedir(dp);
492 return;
493 }
494
495 if (!S_ISREG(sb.st_mode))
496 return;
497
498 if (sb.st_size == 0) {
499 if (mflags.verbosity)
500 warnx("Empty file: %s", file);
501 return;
502 }
503 build_file_cache(db, parent, file, &sb);
504 }
505
506 /* build_file_cache --
507 * This function generates an md5 hash of the file passed as its 2nd parameter
508 * and stores it in a temporary table file_cache along with the full file path.
509 * This is done to support incremental updation of the database.
510 * The temporary table file_cache is dropped thereafter in the function
511 * update_db(), once the database has been updated.
512 */
513 static void
514 build_file_cache(sqlite3 *db, const char *parent, const char *file,
515 struct stat *sb)
516 {
517 const char *sqlstr;
518 sqlite3_stmt *stmt = NULL;
519 int rc, idx;
520 assert(file != NULL);
521 dev_t device_cache = sb->st_dev;
522 ino_t inode_cache = sb->st_ino;
523 time_t mtime_cache = sb->st_mtime;
524
525 sqlstr = "INSERT INTO metadb.file_cache VALUES (:device, :inode,"
526 " :mtime, :parent, :file)";
527 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
528 if (rc != SQLITE_OK) {
529 if (mflags.verbosity)
530 warnx("%s", sqlite3_errmsg(db));
531 return;
532 }
533
534 idx = sqlite3_bind_parameter_index(stmt, ":device");
535 rc = sqlite3_bind_int64(stmt, idx, device_cache);
536 if (rc != SQLITE_OK) {
537 if (mflags.verbosity)
538 warnx("%s", sqlite3_errmsg(db));
539 sqlite3_finalize(stmt);
540 return;
541 }
542
543 idx = sqlite3_bind_parameter_index(stmt, ":inode");
544 rc = sqlite3_bind_int64(stmt, idx, inode_cache);
545 if (rc != SQLITE_OK) {
546 if (mflags.verbosity)
547 warnx("%s", sqlite3_errmsg(db));
548 sqlite3_finalize(stmt);
549 return;
550 }
551
552 idx = sqlite3_bind_parameter_index(stmt, ":mtime");
553 rc = sqlite3_bind_int64(stmt, idx, mtime_cache);
554 if (rc != SQLITE_OK) {
555 if (mflags.verbosity)
556 warnx("%s", sqlite3_errmsg(db));
557 sqlite3_finalize(stmt);
558 return;
559 }
560
561 idx = sqlite3_bind_parameter_index(stmt, ":parent");
562 rc = sqlite3_bind_text(stmt, idx, parent, -1, NULL);
563 if (rc != SQLITE_OK) {
564 if (mflags.verbosity)
565 warnx("%s", sqlite3_errmsg(db));
566 sqlite3_finalize(stmt);
567 return;
568 }
569
570 idx = sqlite3_bind_parameter_index(stmt, ":file");
571 rc = sqlite3_bind_text(stmt, idx, file, -1, NULL);
572 if (rc != SQLITE_OK) {
573 if (mflags.verbosity)
574 warnx("%s", sqlite3_errmsg(db));
575 sqlite3_finalize(stmt);
576 return;
577 }
578
579 sqlite3_step(stmt);
580 sqlite3_finalize(stmt);
581 }
582
583 /* read_and_decompress --
584 * Reads the given file into memory. If it is compressed, decompress
585 * it before returning to the caller.
586 */
587 static int
588 read_and_decompress(const char *file, void **bufp, size_t *len)
589 {
590 size_t off;
591 ssize_t r;
592 struct archive *a;
593 struct archive_entry *ae;
594 char *buf;
595
596 if ((a = archive_read_new()) == NULL)
597 errx(EXIT_FAILURE, "memory allocation failed");
598
599 *bufp = NULL;
600 if (archive_read_support_filter_all(a) != ARCHIVE_OK ||
601 archive_read_support_format_raw(a) != ARCHIVE_OK ||
602 archive_read_open_filename(a, file, 65536) != ARCHIVE_OK ||
603 archive_read_next_header(a, &ae) != ARCHIVE_OK)
604 goto archive_error;
605 *len = 65536;
606 buf = emalloc(*len);
607 off = 0;
608 for (;;) {
609 r = archive_read_data(a, buf + off, *len - off);
610 if (r == ARCHIVE_OK) {
611 archive_read_free(a);
612 *bufp = buf;
613 *len = off;
614 return 0;
615 }
616 if (r <= 0) {
617 free(buf);
618 break;
619 }
620 off += r;
621 if (off == *len) {
622 *len *= 2;
623 if (*len < off) {
624 if (mflags.verbosity)
625 warnx("File too large: %s", file);
626 free(buf);
627 archive_read_free(a);
628 return -1;
629 }
630 buf = erealloc(buf, *len);
631 }
632 }
633
634 archive_error:
635 warnx("Error while reading `%s': %s", file, archive_error_string(a));
636 archive_read_free(a);
637 return -1;
638 }
639
640 static void
641 update_existing_entry(sqlite3 *db, const char *file, const char *hash,
642 mandb_rec *rec, int *new_count, int *link_count, int *err_count)
643 {
644 int update_count, rc, idx;
645 const char *inner_sqlstr;
646 sqlite3_stmt *inner_stmt;
647
648 update_count = sqlite3_total_changes(db);
649 inner_sqlstr = "UPDATE mandb_meta SET device = :device,"
650 " inode = :inode, mtime = :mtime WHERE"
651 " md5_hash = :md5 AND file = :file AND"
652 " (device <> :device2 OR inode <> "
653 " :inode2 OR mtime <> :mtime2)";
654 rc = sqlite3_prepare_v2(db, inner_sqlstr, -1, &inner_stmt, NULL);
655 if (rc != SQLITE_OK) {
656 if (mflags.verbosity)
657 warnx("%s", sqlite3_errmsg(db));
658 return;
659 }
660 idx = sqlite3_bind_parameter_index(inner_stmt, ":device");
661 sqlite3_bind_int64(inner_stmt, idx, rec->device);
662 idx = sqlite3_bind_parameter_index(inner_stmt, ":inode");
663 sqlite3_bind_int64(inner_stmt, idx, rec->inode);
664 idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime");
665 sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
666 idx = sqlite3_bind_parameter_index(inner_stmt, ":md5");
667 sqlite3_bind_text(inner_stmt, idx, hash, -1, NULL);
668 idx = sqlite3_bind_parameter_index(inner_stmt, ":file");
669 sqlite3_bind_text(inner_stmt, idx, file, -1, NULL);
670 idx = sqlite3_bind_parameter_index(inner_stmt, ":device2");
671 sqlite3_bind_int64(inner_stmt, idx, rec->device);
672 idx = sqlite3_bind_parameter_index(inner_stmt, ":inode2");
673 sqlite3_bind_int64(inner_stmt, idx, rec->inode);
674 idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime2");
675 sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
676
677 rc = sqlite3_step(inner_stmt);
678 if (rc == SQLITE_DONE) {
679 /* Check if an update has been performed. */
680 if (update_count != sqlite3_total_changes(db)) {
681 if (mflags.verbosity == 2)
682 printf("Updated %s\n", file);
683 (*new_count)++;
684 } else {
685 /* Otherwise it was a hardlink. */
686 (*link_count)++;
687 }
688 } else {
689 if (mflags.verbosity == 2)
690 warnx("Could not update the meta data for %s", file);
691 (*err_count)++;
692 }
693 sqlite3_finalize(inner_stmt);
694 }
695
696 /* update_db --
697 * Does an incremental updation of the database by checking the file_cache.
698 * It parses and adds the pages which are present in file_cache,
699 * but not in the database.
700 * It also removes the pages which are present in the databse,
701 * but not in the file_cache.
702 */
703 static void
704 update_db(sqlite3 *db, struct mparse *mp, mandb_rec *rec)
705 {
706 const char *sqlstr;
707 sqlite3_stmt *stmt = NULL;
708 char *file;
709 char *parent;
710 char *errmsg = NULL;
711 char *md5sum;
712 void *buf;
713 size_t buflen;
714 struct sql_row {
715 struct sql_row *next;
716 dev_t device;
717 ino_t inode;
718 time_t mtime;
719 char *parent;
720 char *file;
721 } *rows, *row;
722 int new_count = 0; /* Counter for newly indexed/updated pages */
723 int total_count = 0; /* Counter for total number of pages */
724 int err_count = 0; /* Counter for number of failed pages */
725 int link_count = 0; /* Counter for number of hard/sym links */
726 int md5_status;
727 int rc;
728
729 sqlstr = "SELECT device, inode, mtime, parent, file"
730 " FROM metadb.file_cache fc"
731 " WHERE NOT EXISTS(SELECT 1 FROM mandb_meta WHERE"
732 " device = fc.device AND inode = fc.inode AND "
733 " mtime = fc.mtime AND file = fc.file)";
734
735 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
736 if (rc != SQLITE_OK) {
737 if (mflags.verbosity)
738 warnx("%s", sqlite3_errmsg(db));
739 close_db(db);
740 errx(EXIT_FAILURE, "Could not query file cache");
741 }
742
743 buf = NULL;
744 rows = NULL;
745 while (sqlite3_step(stmt) == SQLITE_ROW) {
746 row = emalloc(sizeof(struct sql_row));
747 row->device = sqlite3_column_int64(stmt, 0);
748 row->inode = sqlite3_column_int64(stmt, 1);
749 row->mtime = sqlite3_column_int64(stmt, 2);
750 row->parent = estrdup((const char *) sqlite3_column_text(stmt, 3));
751 row->file = estrdup((const char *) sqlite3_column_text(stmt, 4));
752 row->next = rows;
753 rows = row;
754 total_count++;
755 }
756 sqlite3_finalize(stmt);
757
758 for ( ; rows != NULL; free(parent), free(file), free(buf)) {
759 int fd;
760
761 row = rows;
762 rows = rows->next;
763
764 rec->device = row->device;
765 rec->inode = row->inode;
766 rec->mtime = row->mtime;
767 parent = row->parent;
768 file = row->file;
769 free(row);
770
771 // XXX: reading twice!
772 if (read_and_decompress(file, &buf, &buflen)) {
773 err_count++;
774 continue;
775 }
776 if ((fd = mparse_open(mp, file)) == -1) {
777 err_count++;
778 continue;
779 }
780
781 md5_status = check_md5(file, db, &md5sum, buf, buflen);
782 assert(md5sum != NULL);
783 if (md5_status == -1) {
784 if (mflags.verbosity)
785 warnx("An error occurred in checking md5 value"
786 " for file %s", file);
787 err_count++;
788 close(fd);
789 continue;
790 }
791
792 if (md5_status == 0) {
793 /*
794 * The MD5 hash is already present in the database,
795 * so simply update the metadata.
796 */
797 update_existing_entry(db, file, md5sum, rec,
798 &new_count, &link_count, &err_count);
799 free(md5sum);
800 close(fd);
801 continue;
802 }
803
804 if (md5_status == 1) {
805 /*
806 * The MD5 hash was not present in the database.
807 * This means is either a new file or an updated file.
808 * We should go ahead with parsing.
809 */
810 if (chdir(parent) == -1) {
811 if (mflags.verbosity)
812 warn("chdir failed for `%s', could "
813 "not index `%s'", parent, file);
814 err_count++;
815 free(md5sum);
816 close(fd);
817 continue;
818 }
819
820 if (mflags.verbosity == 2)
821 printf("Parsing: %s\n", file);
822 rec->md5_hash = md5sum;
823 rec->file_path = estrdup(file);
824 // file_path is freed by insert_into_db itself.
825 begin_parse(file, mp, rec, fd);
826 if (insert_into_db(db, rec) < 0) {
827 if (mflags.verbosity)
828 warnx("Error in indexing `%s'", file);
829 err_count++;
830 } else {
831 new_count++;
832 }
833 }
834 close(fd);
835 }
836
837 if (mflags.verbosity == 2) {
838 printf("Number of new or updated pages encountered: %d\n"
839 "Number of hard links found: %d\n"
840 "Number of pages that were successfully"
841 " indexed or updated: %d\n"
842 "Number of pages that could not be indexed"
843 " due to errors: %d\n",
844 total_count - link_count, link_count, new_count, err_count);
845 }
846
847 if (mflags.recreate)
848 return;
849
850 if (mflags.verbosity == 2)
851 printf("Deleting stale index entries\n");
852
853 sqlstr = "DELETE FROM mandb_meta WHERE file NOT IN"
854 " (SELECT file FROM metadb.file_cache);"
855 "DELETE FROM mandb_links WHERE md5_hash NOT IN"
856 " (SELECT md5_hash from mandb_meta);"
857 "DROP TABLE metadb.file_cache;"
858 "DELETE FROM mandb WHERE rowid NOT IN"
859 " (SELECT id FROM mandb_meta);";
860
861 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
862 if (errmsg != NULL) {
863 warnx("Removing old entries failed: %s", errmsg);
864 warnx("Please rebuild database from scratch with -f.");
865 free(errmsg);
866 return;
867 }
868 }
869
870 /*
871 * begin_parse --
872 * parses the man page using libmandoc
873 */
874 static void
875 begin_parse(const char *file, struct mparse *mp, mandb_rec *rec, int fd)
876 {
877 struct roff_meta *roff;
878 mparse_reset(mp);
879
880 rec->xr_found = 0;
881
882 mparse_readfd(mp, fd, file);
883 roff = mparse_result(mp);
884 if (roff == NULL) {
885 if (mflags.verbosity == 2)
886 warnx("Not a roff(7) page");
887 return;
888 }
889
890 if (roff->macroset == MACROSET_MDOC) {
891 rec->page_type = MDOC;
892 proff_node(roff->first->child, rec, roff, mdocs);
893 } else if (roff->macroset == MACROSET_MAN) {
894 rec->page_type = MAN;
895 proff_node(roff->first->child, rec, roff, mans);
896 } else
897 warnx("Unknown macroset %d", roff->macroset);
898 set_machine(roff, rec);
899 set_section(roff, rec);
900 }
901
902 /*
903 * set_section --
904 * Extracts the section number and normalizes it to only the numeric part
905 * (Which should be the first character of the string).
906 */
907 static void
908 set_section(const struct roff_meta *rm, mandb_rec *rec)
909 {
910 if (!rm)
911 return;
912 const char *s = rm->msec == NULL ? "?" : rm->msec;
913 easprintf(&rec->section, "%s", s);
914 if (rec->section[0] == '?' && mflags.verbosity == 2)
915 warnx("%s: Missing section number", rec->file_path);
916 }
917
918 /*
919 * get_machine --
920 * Extracts the machine architecture information if available.
921 */
922 static void
923 set_machine(const struct roff_meta *rm, mandb_rec *rec)
924 {
925 if (rm == NULL)
926 return;
927 if (rm->arch)
928 rec->machine = estrdup(rm->arch);
929 }
930
931 /*
932 * pmdoc_Nm --
933 * Extracts the Name of the manual page from the .Nm macro
934 */
935 static void
936 pmdoc_Nm(const struct roff_node *n, mandb_rec *rec)
937 {
938 if (n->sec != SEC_NAME)
939 return;
940
941 for (n = n->child; n; n = n->next) {
942 if (n->type == ROFFT_TEXT) {
943 char *escaped_name = parse_escape(n->string);
944 concat(&rec->name, escaped_name);
945 free(escaped_name);
946 }
947 }
948 }
949
950 /*
951 * pmdoc_Nd --
952 * Extracts the one line description of the man page from the .Nd macro
953 */
954 static void
955 pmdoc_Nd(const struct roff_node *n, mandb_rec *rec)
956 {
957 if (n->type == ROFFT_BODY)
958 deroff(&rec->name_desc, n);
959 if (rec->name_desc)
960 replace_hyph(rec->name_desc);
961
962 }
963
964 /*
965 * pmdoc_macro_handler--
966 * This function is a single point of handling all the special macros that we
967 * want to handle especially. For example the .Xr macro for properly parsing
968 * the referenced page name along with the section number, or the .Pp macro
969 * for adding a new line whenever we encounter it.
970 */
971 static void
972 pmdoc_macro_handler(const struct roff_node *n, mandb_rec *rec, int doct)
973 {
974 const struct roff_node *sn;
975 assert(n);
976
977 switch (doct) {
978 /* Parse the man page references.
979 * Basically the .Xr macros are used like:
980 * .Xr ls 1
981 * and formatted like this:
982 * ls(1)
983 * Prepare a buffer to format the data like the above example and call
984 * pmdoc_parse_section to append it.
985 */
986 case MDOC_Xr:
987 n = n->child;
988 while (n->type != ROFFT_TEXT && n->next)
989 n = n->next;
990
991 if (n && n->type != ROFFT_TEXT)
992 return;
993 sn = n;
994 if (n->next)
995 n = n->next;
996
997 while (n->type != ROFFT_TEXT && n->next)
998 n = n->next;
999
1000 if (n && n->type == ROFFT_TEXT) {
1001 char *buf;
1002 easprintf(&buf, "%s(%s)", sn->string, n->string);
1003 mdoc_parse_section(n->sec, buf, rec);
1004 free(buf);
1005 }
1006
1007 break;
1008
1009 /* Parse the .Pp macro to add a new line */
1010 case MDOC_Pp:
1011 if (n->type == ROFFT_TEXT)
1012 mdoc_parse_section(n->sec, "\n", rec);
1013 break;
1014 default:
1015 break;
1016 }
1017
1018 }
1019
1020 /*
1021 * pmdoc_Xr, pmdoc_Pp--
1022 * Empty stubs.
1023 * The parser calls these functions each time it encounters
1024 * a .Xr or .Pp macro. We are parsing all the data from
1025 * the pmdoc_Sh function, so don't do anything here.
1026 * (See if else blocks in pmdoc_Sh.)
1027 */
1028 static void
1029 pmdoc_Xr(const struct roff_node *n, mandb_rec *rec)
1030 {
1031 }
1032
1033 static void
1034 pmdoc_Pp(const struct roff_node *n, mandb_rec *rec)
1035 {
1036 }
1037
1038 /*
1039 * pmdoc_Sh --
1040 * Called when a .Sh macro is encountered and tries to parse its body
1041 */
1042 static void
1043 pmdoc_Sh(const struct roff_node *n, mandb_rec *rec)
1044 {
1045 if (n == NULL)
1046 return;
1047
1048 switch (n->sec) {
1049 case SEC_NAME:
1050 case SEC_SYNOPSIS:
1051 case SEC_EXAMPLES:
1052 case SEC_STANDARDS:
1053 case SEC_HISTORY:
1054 case SEC_AUTHORS:
1055 case SEC_BUGS:
1056 /*
1057 * We don't care about text from these sections
1058 */
1059 return;
1060 default:
1061 break;
1062 }
1063
1064 if (n->type == ROFFT_BLOCK)
1065 mdoc_parse_Sh(n->body, rec);
1066 }
1067
1068 /*
1069 * Called from pmdoc_Sh to parse body of a .Sh macro. It calls
1070 * mdoc_parse_section to append the data to the section specific buffer.
1071 * The .Xr macro needs special handling, thus the separate if branch for it.
1072 */
1073 static void
1074 mdoc_parse_Sh(const struct roff_node *n, mandb_rec *rec)
1075 {
1076 if (n == NULL || (n->type != ROFFT_TEXT && n->tok == MDOC_MAX))
1077 return;
1078 int xr_found = 0;
1079
1080 if (n->type == ROFFT_TEXT) {
1081 mdoc_parse_section(n->sec, n->string, rec);
1082 } else if (mdocs[n->tok] == pmdoc_Xr) {
1083 /*
1084 * When encountering other inline macros,
1085 * call pmdoc_macro_handler.
1086 */
1087 pmdoc_macro_handler(n, rec, MDOC_Xr);
1088 xr_found = 1;
1089 } else if (mdocs[n->tok] == pmdoc_Pp) {
1090 pmdoc_macro_handler(n, rec, MDOC_Pp);
1091 }
1092
1093 /*
1094 * If an Xr macro was encountered then the child node has
1095 * already been explored by pmdoc_macro_handler.
1096 */
1097 if (xr_found == 0)
1098 mdoc_parse_Sh(n->child, rec);
1099 mdoc_parse_Sh(n->next, rec);
1100 }
1101
1102 /*
1103 * mdoc_parse_section--
1104 * Utility function for parsing sections of the mdoc type pages.
1105 * Takes two params:
1106 * 1. sec is an enum which indicates the section in which we are present
1107 * 2. string is the string which we need to append to the secbuff for this
1108 * particular section.
1109 * The function appends string to the global section buffer and returns.
1110 */
1111 static void
1112 mdoc_parse_section(enum roff_sec sec, const char *string, mandb_rec *rec)
1113 {
1114 /*
1115 * If the user specified the 'l' flag, then parse and store only the
1116 * NAME section. Ignore the rest.
1117 */
1118 if (mflags.limit)
1119 return;
1120
1121 switch (sec) {
1122 case SEC_LIBRARY:
1123 append(&rec->lib, string);
1124 break;
1125 case SEC_RETURN_VALUES:
1126 append(&rec->return_vals, string);
1127 break;
1128 case SEC_ENVIRONMENT:
1129 append(&rec->env, string);
1130 break;
1131 case SEC_FILES:
1132 append(&rec->files, string);
1133 break;
1134 case SEC_EXIT_STATUS:
1135 append(&rec->exit_status, string);
1136 break;
1137 case SEC_DIAGNOSTICS:
1138 append(&rec->diagnostics, string);
1139 break;
1140 case SEC_ERRORS:
1141 append(&rec->errors, string);
1142 break;
1143 default:
1144 append(&rec->desc, string);
1145 break;
1146 }
1147 }
1148
1149 static void
1150 proff_node(const struct roff_node * n, mandb_rec * rec,
1151 struct roff_meta * roff, const proff_nf * func)
1152 {
1153 if (n == NULL)
1154 return;
1155
1156 int tok_idx;
1157
1158 switch (n->type) {
1159 case (ROFFT_BODY):
1160 /* FALLTHROUGH */
1161 case (ROFFT_BLOCK):
1162 /* FALLTHROUGH */
1163 case (ROFFT_ELEM):
1164 if (roff->macroset == MACROSET_MAN)
1165 tok_idx = n->tok - MAN_TH;
1166 else if (roff->macroset == MACROSET_MDOC)
1167 tok_idx = n->tok - MDOC_Dd;
1168 else
1169 tok_idx = -1;
1170 if (tok_idx >= 0 && func[tok_idx] != NULL)
1171 (*func[tok_idx]) (n, rec);
1172 break;
1173 default:
1174 break;
1175 }
1176
1177 proff_node(n->child, rec, roff, func);
1178 proff_node(n->next, rec, roff, func);
1179 }
1180
1181 /*
1182 * pman_parse_name --
1183 * Parses the NAME section and puts the complete content in the name_desc
1184 * variable.
1185 */
1186 static void
1187 pman_parse_name(const struct roff_node *n, mandb_rec *rec)
1188 {
1189 if (n == NULL)
1190 return;
1191
1192 if (n->type == ROFFT_TEXT) {
1193 char *tmp = parse_escape(n->string);
1194 concat(&rec->name_desc, tmp);
1195 free(tmp);
1196 }
1197
1198 if (n->child)
1199 pman_parse_name(n->child, rec);
1200
1201 if(n->next)
1202 pman_parse_name(n->next, rec);
1203 }
1204
1205 /*
1206 * A stub function to be able to parse the macros like .B embedded inside
1207 * a section.
1208 */
1209 static void
1210 pman_block(const struct roff_node *n, mandb_rec *rec)
1211 {
1212 }
1213
1214 /*
1215 * pman_sh --
1216 * This function does one of the two things:
1217 * 1. If the present section is NAME, then it will:
1218 * (a) Extract the name of the page (in case of multiple comma separated
1219 * names, it will pick up the first one).
1220 * (b) Build a space spearated list of all the symlinks/hardlinks to
1221 * this page and store in the buffer 'links'. These are extracted from
1222 * the comma separated list of names in the NAME section as well.
1223 * (c) Move on to the one line description section, which is after the list
1224 * of names in the NAME section.
1225 * 2. Otherwise, it will check the section name and call the man_parse_section
1226 * function, passing the enum corresponding to that section.
1227 */
1228 static void
1229 pman_sh(const struct roff_node *n, mandb_rec *rec)
1230 {
1231 static const struct {
1232 enum man_sec section;
1233 const char *header;
1234 } mapping[] = {
1235 { MANSEC_DESCRIPTION, "DESCRIPTION" },
1236 { MANSEC_SYNOPSIS, "SYNOPSIS" },
1237 { MANSEC_LIBRARY, "LIBRARY" },
1238 { MANSEC_ERRORS, "ERRORS" },
1239 { MANSEC_FILES, "FILES" },
1240 { MANSEC_RETURN_VALUES, "RETURN VALUE" },
1241 { MANSEC_RETURN_VALUES, "RETURN VALUES" },
1242 { MANSEC_EXIT_STATUS, "EXIT STATUS" },
1243 { MANSEC_EXAMPLES, "EXAMPLES" },
1244 { MANSEC_EXAMPLES, "EXAMPLE" },
1245 { MANSEC_STANDARDS, "STANDARDS" },
1246 { MANSEC_HISTORY, "HISTORY" },
1247 { MANSEC_BUGS, "BUGS" },
1248 { MANSEC_AUTHORS, "AUTHORS" },
1249 { MANSEC_COPYRIGHT, "COPYRIGHT" },
1250 };
1251 const struct roff_node *head;
1252 char *name_desc;
1253 size_t sz;
1254 size_t i;
1255
1256 if ((head = n->parent->head) == NULL || (head = head->child) == NULL ||
1257 head->type != ROFFT_TEXT)
1258 return;
1259
1260 /*
1261 * Check if this section should be extracted and
1262 * where it should be stored. Handled the trival cases first.
1263 */
1264 for (i = 0; i < sizeof(mapping) / sizeof(mapping[0]); ++i) {
1265 if (strcmp(head->string, mapping[i].header) == 0) {
1266 man_parse_section(mapping[i].section, n, rec);
1267 return;
1268 }
1269 }
1270
1271 if (strcmp(head->string, "NAME") == 0) {
1272 /*
1273 * We are in the NAME section.
1274 * pman_parse_name will put the complete content in name_desc.
1275 */
1276 pman_parse_name(n, rec);
1277
1278 name_desc = rec->name_desc;
1279 if (name_desc == NULL)
1280 return;
1281
1282 /* Remove any leading spaces. */
1283 while (name_desc[0] == ' ')
1284 name_desc++;
1285
1286 /* If the line begins with a "\&", avoid those */
1287 if (name_desc[0] == '\\' && name_desc[1] == '&')
1288 name_desc += 2;
1289
1290 /* Now name_desc should be left with a comma-space
1291 * separated list of names and the one line description
1292 * of the page:
1293 * "a, b, c \- sample description"
1294 * Take out the first name, before the first comma
1295 * (or space) and store it in rec->name.
1296 * If the page has aliases then they should be
1297 * in the form of a comma separated list.
1298 * Keep looping while there is a comma in name_desc,
1299 * extract the alias name and store in rec->links.
1300 * When there are no more commas left, break out.
1301 */
1302 int has_alias = 0; // Any more aliases left?
1303 while (*name_desc) {
1304 /* Remove any leading spaces or hyphens. */
1305 if (name_desc[0] == ' ' || name_desc[0] == '-') {
1306 name_desc++;
1307 continue;
1308 }
1309 sz = strcspn(name_desc, ", ");
1310
1311 /* Extract the first term and store it in rec->name. */
1312 if (rec->name == NULL) {
1313 if (name_desc[sz] == ',')
1314 has_alias = 1;
1315 rec->name = estrndup(name_desc, sz);
1316 /* XXX This would only happen with a poorly
1317 * written man page, maybe warn? */
1318 if (name_desc[sz] == '\0')
1319 break;
1320 name_desc += sz + 1;
1321 continue;
1322 }
1323
1324 /*
1325 * Once rec->name is set, rest of the names
1326 * are to be treated as links or aliases.
1327 */
1328 if (rec->name && has_alias) {
1329 if (name_desc[sz] != ',') {
1330 /* No more commas left --> no more
1331 * aliases to take out */
1332 has_alias = 0;
1333 }
1334 concat2(&rec->links, name_desc, sz);
1335 /* XXX This would only happen with a poorly
1336 * written man page, maybe warn? */
1337 if (name_desc[sz] == '\0')
1338 break;
1339 name_desc += sz + 1;
1340 continue;
1341 }
1342 break;
1343 }
1344
1345 /* Parse any escape sequences that might be there */
1346 char *temp = parse_escape(name_desc);
1347 free(rec->name_desc);
1348 rec->name_desc = temp;
1349 temp = parse_escape(rec->name);
1350 free(rec->name);
1351 rec->name = temp;
1352 return;
1353 }
1354
1355 /* The RETURN VALUE section might be specified in multiple ways */
1356 if (strcmp(head->string, "RETURN") == 0 &&
1357 head->next != NULL && head->next->type == ROFFT_TEXT &&
1358 (strcmp(head->next->string, "VALUE") == 0 ||
1359 strcmp(head->next->string, "VALUES") == 0)) {
1360 man_parse_section(MANSEC_RETURN_VALUES, n, rec);
1361 return;
1362 }
1363
1364 /*
1365 * EXIT STATUS section can also be specified all on one line or on two
1366 * separate lines.
1367 */
1368 if (strcmp(head->string, "EXIT") == 0 &&
1369 head->next != NULL && head->next->type == ROFFT_TEXT &&
1370 strcmp(head->next->string, "STATUS") == 0) {
1371 man_parse_section(MANSEC_EXIT_STATUS, n, rec);
1372 return;
1373 }
1374
1375 /* Store the rest of the content in desc. */
1376 man_parse_section(MANSEC_NONE, n, rec);
1377 }
1378
1379 /*
1380 * pman_parse_node --
1381 * Generic function to iterate through a node. Usually called from
1382 * man_parse_section to parse a particular section of the man page.
1383 */
1384 static void
1385 pman_parse_node(const struct roff_node *n, secbuff *s)
1386 {
1387 if (n == NULL)
1388 return;
1389
1390 if (n->type == ROFFT_TEXT)
1391 append(s, n->string);
1392
1393 pman_parse_node(n->child, s);
1394 pman_parse_node(n->next, s);
1395 }
1396
1397 /*
1398 * man_parse_section --
1399 * Takes two parameters:
1400 * sec: Tells which section we are present in
1401 * n: Is the present node of the AST.
1402 * Depending on the section, we call pman_parse_node to parse that section and
1403 * concatenate the content from that section into the buffer for that section.
1404 */
1405 static void
1406 man_parse_section(enum man_sec sec, const struct roff_node *n, mandb_rec *rec)
1407 {
1408 /*
1409 * If the user sepecified the 'l' flag then just parse
1410 * the NAME section, ignore the rest.
1411 */
1412 if (mflags.limit)
1413 return;
1414
1415 switch (sec) {
1416 case MANSEC_LIBRARY:
1417 pman_parse_node(n, &rec->lib);
1418 break;
1419 case MANSEC_RETURN_VALUES:
1420 pman_parse_node(n, &rec->return_vals);
1421 break;
1422 case MANSEC_ENVIRONMENT:
1423 pman_parse_node(n, &rec->env);
1424 break;
1425 case MANSEC_FILES:
1426 pman_parse_node(n, &rec->files);
1427 break;
1428 case MANSEC_EXIT_STATUS:
1429 pman_parse_node(n, &rec->exit_status);
1430 break;
1431 case MANSEC_DIAGNOSTICS:
1432 pman_parse_node(n, &rec->diagnostics);
1433 break;
1434 case MANSEC_ERRORS:
1435 pman_parse_node(n, &rec->errors);
1436 break;
1437 case MANSEC_NAME:
1438 case MANSEC_SYNOPSIS:
1439 case MANSEC_EXAMPLES:
1440 case MANSEC_STANDARDS:
1441 case MANSEC_HISTORY:
1442 case MANSEC_BUGS:
1443 case MANSEC_AUTHORS:
1444 case MANSEC_COPYRIGHT:
1445 break;
1446 default:
1447 pman_parse_node(n, &rec->desc);
1448 break;
1449 }
1450
1451 }
1452
1453 /*
1454 * insert_into_db --
1455 * Inserts the parsed data of the man page in the Sqlite databse.
1456 * If any of the values is NULL, then we cleanup and return -1 indicating
1457 * an error.
1458 * Otherwise, store the data in the database and return 0.
1459 */
1460 static int
1461 insert_into_db(sqlite3 *db, mandb_rec *rec)
1462 {
1463 int rc = 0;
1464 int idx = -1;
1465 const char *sqlstr = NULL;
1466 sqlite3_stmt *stmt = NULL;
1467 char *ln = NULL;
1468 char *errmsg = NULL;
1469 long int mandb_rowid;
1470
1471 /*
1472 * At the very minimum we want to make sure that we store
1473 * the following data:
1474 * Name, one line description, and the MD5 hash
1475 */
1476 if (rec->name == NULL || rec->name_desc == NULL ||
1477 rec->md5_hash == NULL) {
1478 cleanup(rec);
1479 return -1;
1480 }
1481
1482 /* Write null byte at the end of all the sec_buffs */
1483 rec->desc.data[rec->desc.offset] = 0;
1484 rec->lib.data[rec->lib.offset] = 0;
1485 rec->env.data[rec->env.offset] = 0;
1486 rec->return_vals.data[rec->return_vals.offset] = 0;
1487 rec->exit_status.data[rec->exit_status.offset] = 0;
1488 rec->files.data[rec->files.offset] = 0;
1489 rec->diagnostics.data[rec->diagnostics.offset] = 0;
1490 rec->errors.data[rec->errors.offset] = 0;
1491
1492 /*
1493 * In case of a mdoc page: (sorry, no better place to put this code)
1494 * parse the comma separated list of names of man pages,
1495 * the first name will be stored in the mandb table, rest will be
1496 * treated as links and put in the mandb_links table.
1497 */
1498 if (rec->page_type == MDOC) {
1499 char *tmp;
1500 rec->links = estrdup(rec->name);
1501 free(rec->name);
1502 size_t sz = strcspn(rec->links, " \0");
1503 rec->name = emalloc(sz + 1);
1504 memcpy(rec->name, rec->links, sz);
1505 if(rec->name[sz - 1] == ',')
1506 rec->name[sz - 1] = 0;
1507 else
1508 rec->name[sz] = 0;
1509 while (rec->links[sz] == ' ')
1510 ++sz;
1511 tmp = estrdup(rec->links + sz);
1512 free(rec->links);
1513 rec->links = tmp;
1514 }
1515
1516 /*------------------------ Populate the mandb table---------------------------*/
1517 sqlstr = "INSERT INTO mandb VALUES (:section, :name, :name_desc, :desc,"
1518 " :lib, :return_vals, :env, :files, :exit_status,"
1519 " :diagnostics, :errors, :md5_hash, :machine)";
1520
1521 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1522 if (rc != SQLITE_OK)
1523 goto Out;
1524
1525 idx = sqlite3_bind_parameter_index(stmt, ":name");
1526 rc = sqlite3_bind_text(stmt, idx, rec->name, -1, NULL);
1527 if (rc != SQLITE_OK) {
1528 sqlite3_finalize(stmt);
1529 goto Out;
1530 }
1531
1532 idx = sqlite3_bind_parameter_index(stmt, ":section");
1533 rc = sqlite3_bind_text(stmt, idx, rec->section, -1, NULL);
1534 if (rc != SQLITE_OK) {
1535 sqlite3_finalize(stmt);
1536 goto Out;
1537 }
1538
1539 idx = sqlite3_bind_parameter_index(stmt, ":name_desc");
1540 rc = sqlite3_bind_text(stmt, idx, rec->name_desc, -1, NULL);
1541 if (rc != SQLITE_OK) {
1542 sqlite3_finalize(stmt);
1543 goto Out;
1544 }
1545
1546 idx = sqlite3_bind_parameter_index(stmt, ":desc");
1547 rc = sqlite3_bind_text(stmt, idx, rec->desc.data,
1548 rec->desc.offset + 1, NULL);
1549 if (rc != SQLITE_OK) {
1550 sqlite3_finalize(stmt);
1551 goto Out;
1552 }
1553
1554 idx = sqlite3_bind_parameter_index(stmt, ":lib");
1555 rc = sqlite3_bind_text(stmt, idx, rec->lib.data,
1556 rec->lib.offset + 1, NULL);
1557 if (rc != SQLITE_OK) {
1558 sqlite3_finalize(stmt);
1559 goto Out;
1560 }
1561
1562 idx = sqlite3_bind_parameter_index(stmt, ":return_vals");
1563 rc = sqlite3_bind_text(stmt, idx, rec->return_vals.data,
1564 rec->return_vals.offset + 1, NULL);
1565 if (rc != SQLITE_OK) {
1566 sqlite3_finalize(stmt);
1567 goto Out;
1568 }
1569
1570 idx = sqlite3_bind_parameter_index(stmt, ":env");
1571 rc = sqlite3_bind_text(stmt, idx, rec->env.data,
1572 rec->env.offset + 1, NULL);
1573 if (rc != SQLITE_OK) {
1574 sqlite3_finalize(stmt);
1575 goto Out;
1576 }
1577
1578 idx = sqlite3_bind_parameter_index(stmt, ":files");
1579 rc = sqlite3_bind_text(stmt, idx, rec->files.data,
1580 rec->files.offset + 1, NULL);
1581 if (rc != SQLITE_OK) {
1582 sqlite3_finalize(stmt);
1583 goto Out;
1584 }
1585
1586 idx = sqlite3_bind_parameter_index(stmt, ":exit_status");
1587 rc = sqlite3_bind_text(stmt, idx, rec->exit_status.data,
1588 rec->exit_status.offset + 1, NULL);
1589 if (rc != SQLITE_OK) {
1590 sqlite3_finalize(stmt);
1591 goto Out;
1592 }
1593
1594 idx = sqlite3_bind_parameter_index(stmt, ":diagnostics");
1595 rc = sqlite3_bind_text(stmt, idx, rec->diagnostics.data,
1596 rec->diagnostics.offset + 1, NULL);
1597 if (rc != SQLITE_OK) {
1598 sqlite3_finalize(stmt);
1599 goto Out;
1600 }
1601
1602 idx = sqlite3_bind_parameter_index(stmt, ":errors");
1603 rc = sqlite3_bind_text(stmt, idx, rec->errors.data,
1604 rec->errors.offset + 1, NULL);
1605 if (rc != SQLITE_OK) {
1606 sqlite3_finalize(stmt);
1607 goto Out;
1608 }
1609
1610 idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1611 rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1612 if (rc != SQLITE_OK) {
1613 sqlite3_finalize(stmt);
1614 goto Out;
1615 }
1616
1617 idx = sqlite3_bind_parameter_index(stmt, ":machine");
1618 if (rec->machine)
1619 rc = sqlite3_bind_text(stmt, idx, rec->machine, -1, NULL);
1620 else
1621 rc = sqlite3_bind_null(stmt, idx);
1622 if (rc != SQLITE_OK) {
1623 sqlite3_finalize(stmt);
1624 goto Out;
1625 }
1626
1627 rc = sqlite3_step(stmt);
1628 if (rc != SQLITE_DONE) {
1629 sqlite3_finalize(stmt);
1630 goto Out;
1631 }
1632
1633 sqlite3_finalize(stmt);
1634
1635 /* Get the row id of the last inserted row */
1636 mandb_rowid = sqlite3_last_insert_rowid(db);
1637
1638 /*------------------------Populate the mandb_meta table-----------------------*/
1639 sqlstr = "INSERT INTO mandb_meta VALUES (:device, :inode, :mtime,"
1640 " :file, :md5_hash, :id)";
1641 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1642 if (rc != SQLITE_OK)
1643 goto Out;
1644
1645 idx = sqlite3_bind_parameter_index(stmt, ":device");
1646 rc = sqlite3_bind_int64(stmt, idx, rec->device);
1647 if (rc != SQLITE_OK) {
1648 sqlite3_finalize(stmt);
1649 goto Out;
1650 }
1651
1652 idx = sqlite3_bind_parameter_index(stmt, ":inode");
1653 rc = sqlite3_bind_int64(stmt, idx, rec->inode);
1654 if (rc != SQLITE_OK) {
1655 sqlite3_finalize(stmt);
1656 goto Out;
1657 }
1658
1659 idx = sqlite3_bind_parameter_index(stmt, ":mtime");
1660 rc = sqlite3_bind_int64(stmt, idx, rec->mtime);
1661 if (rc != SQLITE_OK) {
1662 sqlite3_finalize(stmt);
1663 goto Out;
1664 }
1665
1666 idx = sqlite3_bind_parameter_index(stmt, ":file");
1667 rc = sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
1668 if (rc != SQLITE_OK) {
1669 sqlite3_finalize(stmt);
1670 goto Out;
1671 }
1672
1673 idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1674 rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1675 if (rc != SQLITE_OK) {
1676 sqlite3_finalize(stmt);
1677 goto Out;
1678 }
1679
1680 idx = sqlite3_bind_parameter_index(stmt, ":id");
1681 rc = sqlite3_bind_int64(stmt, idx, mandb_rowid);
1682 if (rc != SQLITE_OK) {
1683 sqlite3_finalize(stmt);
1684 goto Out;
1685 }
1686
1687 rc = sqlite3_step(stmt);
1688 sqlite3_finalize(stmt);
1689 if (rc == SQLITE_CONSTRAINT_UNIQUE) {
1690 /* The *most* probable reason for reaching here is that
1691 * the UNIQUE contraint on the file column of the mandb_meta
1692 * table was violated.
1693 * This can happen when a file was updated/modified.
1694 * To fix this we need to do two things:
1695 * 1. Delete the row for the older version of this file
1696 * from mandb table.
1697 * 2. Run an UPDATE query to update the row for this file
1698 * in the mandb_meta table.
1699 */
1700 warnx("Trying to update index for %s", rec->file_path);
1701 char *sql = sqlite3_mprintf("DELETE FROM mandb "
1702 "WHERE rowid = (SELECT id"
1703 " FROM mandb_meta"
1704 " WHERE file = %Q)",
1705 rec->file_path);
1706 sqlite3_exec(db, sql, NULL, NULL, &errmsg);
1707 sqlite3_free(sql);
1708 if (errmsg != NULL) {
1709 if (mflags.verbosity)
1710 warnx("%s", errmsg);
1711 free(errmsg);
1712 }
1713 sqlstr = "UPDATE mandb_meta SET device = :device,"
1714 " inode = :inode, mtime = :mtime, id = :id,"
1715 " md5_hash = :md5 WHERE file = :file";
1716 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1717 if (rc != SQLITE_OK) {
1718 if (mflags.verbosity)
1719 warnx("Update failed with error: %s",
1720 sqlite3_errmsg(db));
1721 close_db(db);
1722 cleanup(rec);
1723 errx(EXIT_FAILURE,
1724 "Consider running makemandb with -f option");
1725 }
1726
1727 idx = sqlite3_bind_parameter_index(stmt, ":device");
1728 sqlite3_bind_int64(stmt, idx, rec->device);
1729 idx = sqlite3_bind_parameter_index(stmt, ":inode");
1730 sqlite3_bind_int64(stmt, idx, rec->inode);
1731 idx = sqlite3_bind_parameter_index(stmt, ":mtime");
1732 sqlite3_bind_int64(stmt, idx, rec->mtime);
1733 idx = sqlite3_bind_parameter_index(stmt, ":id");
1734 sqlite3_bind_int64(stmt, idx, mandb_rowid);
1735 idx = sqlite3_bind_parameter_index(stmt, ":md5");
1736 sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1737 idx = sqlite3_bind_parameter_index(stmt, ":file");
1738 sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
1739 rc = sqlite3_step(stmt);
1740 sqlite3_finalize(stmt);
1741
1742 if (rc != SQLITE_DONE) {
1743 if (mflags.verbosity)
1744 warnx("%s", sqlite3_errmsg(db));
1745 close_db(db);
1746 cleanup(rec);
1747 errx(EXIT_FAILURE,
1748 "Consider running makemandb with -f option");
1749 }
1750 } else if (rc != SQLITE_DONE) {
1751 /* Otherwise make this error fatal */
1752 warnx("Failed at %s\n%s", rec->file_path, sqlite3_errmsg(db));
1753 cleanup(rec);
1754 close_db(db);
1755 exit(EXIT_FAILURE);
1756 }
1757
1758 /*------------------------ Populate the mandb_links table---------------------*/
1759 char *str = NULL;
1760 char *links;
1761 if (rec->links && strlen(rec->links)) {
1762 links = rec->links;
1763 for(ln = strtok(links, " "); ln; ln = strtok(NULL, " ")) {
1764 if (ln[0] == ',')
1765 ln++;
1766 if(ln[strlen(ln) - 1] == ',')
1767 ln[strlen(ln) - 1] = 0;
1768
1769 str = sqlite3_mprintf("INSERT INTO mandb_links"
1770 " VALUES (%Q, %Q, %Q, %Q, %Q, %Q)",
1771 ln, rec->name, rec->section,
1772 rec->machine, rec->md5_hash, rec->name_desc);
1773 sqlite3_exec(db, str, NULL, NULL, &errmsg);
1774 sqlite3_free(str);
1775 if (errmsg != NULL) {
1776 warnx("%s", errmsg);
1777 cleanup(rec);
1778 free(errmsg);
1779 return -1;
1780 }
1781 }
1782 }
1783
1784 cleanup(rec);
1785 return 0;
1786
1787 Out:
1788 if (mflags.verbosity)
1789 warnx("%s", sqlite3_errmsg(db));
1790 cleanup(rec);
1791 return -1;
1792 }
1793
1794 /*
1795 * check_md5--
1796 * Generates the md5 hash of the file and checks if it already doesn't exist
1797 * in the table.
1798 * This function is being used to avoid hardlinks.
1799 * On successful completion it will also set the value of the fourth parameter
1800 * to the md5 hash of the file (computed previously). It is the responsibility
1801 * of the caller to free this buffer.
1802 * Return values:
1803 * -1: If an error occurs somewhere and sets the md5 return buffer to NULL.
1804 * 0: If the md5 hash does not exist in the table.
1805 * 1: If the hash exists in the database.
1806 */
1807 static int
1808 check_md5(const char *file, sqlite3 *db, char **md5, void *buf, size_t buflen)
1809 {
1810 int rc = 0;
1811 int idx = -1;
1812 char *sqlstr = NULL;
1813 char *mymd5;
1814 sqlite3_stmt *stmt = NULL;
1815 *md5 = NULL;
1816
1817 assert(file != NULL);
1818 if ((mymd5 = MD5Data(buf, buflen, NULL)) == NULL) {
1819 if (mflags.verbosity)
1820 warn("md5 failed: %s", file);
1821 return -1;
1822 }
1823
1824 easprintf(&sqlstr, "SELECT * FROM mandb_meta WHERE md5_hash = :md5_hash");
1825 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1826 if (rc != SQLITE_OK) {
1827 free(sqlstr);
1828 free(mymd5);
1829 return -1;
1830 }
1831
1832 idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1833 rc = sqlite3_bind_text(stmt, idx, mymd5, -1, NULL);
1834 if (rc != SQLITE_OK) {
1835 if (mflags.verbosity)
1836 warnx("%s", sqlite3_errmsg(db));
1837 sqlite3_finalize(stmt);
1838 free(sqlstr);
1839 free(mymd5);
1840 return -1;
1841 }
1842
1843 *md5 = mymd5;
1844 if (sqlite3_step(stmt) == SQLITE_ROW) {
1845 sqlite3_finalize(stmt);
1846 free(sqlstr);
1847 return 0;
1848 }
1849
1850 sqlite3_finalize(stmt);
1851 free(sqlstr);
1852 return 1;
1853 }
1854
1855 /* Optimize the index for faster search */
1856 static void
1857 optimize(sqlite3 *db)
1858 {
1859 const char *sqlstr;
1860 char *errmsg = NULL;
1861
1862 if (mflags.verbosity == 2)
1863 printf("Optimizing the database index\n");
1864 sqlstr = "INSERT INTO mandb(mandb) VALUES (\'optimize\');"
1865 "VACUUM";
1866 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
1867 if (errmsg != NULL) {
1868 if (mflags.verbosity)
1869 warnx("%s", errmsg);
1870 free(errmsg);
1871 return;
1872 }
1873 }
1874
1875 /*
1876 * cleanup --
1877 * cleans up the global buffers
1878 */
1879 static void
1880 cleanup(mandb_rec *rec)
1881 {
1882 rec->desc.offset = 0;
1883 rec->lib.offset = 0;
1884 rec->return_vals.offset = 0;
1885 rec->env.offset = 0;
1886 rec->exit_status.offset = 0;
1887 rec->diagnostics.offset = 0;
1888 rec->errors.offset = 0;
1889 rec->files.offset = 0;
1890
1891 free(rec->machine);
1892 rec->machine = NULL;
1893
1894 free(rec->links);
1895 rec->links = NULL;
1896
1897 free(rec->file_path);
1898 rec->file_path = NULL;
1899
1900 free(rec->name);
1901 rec->name = NULL;
1902
1903 free(rec->name_desc);
1904 rec->name_desc = NULL;
1905
1906 free(rec->md5_hash);
1907 rec->md5_hash = NULL;
1908
1909 free(rec->section);
1910 rec->section = NULL;
1911 }
1912
1913 /*
1914 * init_secbuffs--
1915 * Sets the value of buflen for all the sec_buff field of rec. And then
1916 * allocate memory to each sec_buff member of rec.
1917 */
1918 static void
1919 init_secbuffs(mandb_rec *rec)
1920 {
1921 /*
1922 * Some sec_buff might need more memory, for example desc,
1923 * which stores the data of the DESCRIPTION section,
1924 * while some might need very small amount of memory.
1925 * Therefore explicitly setting the value of buflen field for
1926 * each sec_buff.
1927 */
1928 rec->desc.buflen = 10 * BUFLEN;
1929 rec->desc.data = emalloc(rec->desc.buflen);
1930 rec->desc.offset = 0;
1931
1932 rec->lib.buflen = BUFLEN / 2;
1933 rec->lib.data = emalloc(rec->lib.buflen);
1934 rec->lib.offset = 0;
1935
1936 rec->return_vals.buflen = BUFLEN;
1937 rec->return_vals.data = emalloc(rec->return_vals.buflen);
1938 rec->return_vals.offset = 0;
1939
1940 rec->exit_status.buflen = BUFLEN;
1941 rec->exit_status.data = emalloc(rec->exit_status.buflen);
1942 rec->exit_status.offset = 0;
1943
1944 rec->env.buflen = BUFLEN;
1945 rec->env.data = emalloc(rec->env.buflen);
1946 rec->env.offset = 0;
1947
1948 rec->files.buflen = BUFLEN;
1949 rec->files.data = emalloc(rec->files.buflen);
1950 rec->files.offset = 0;
1951
1952 rec->diagnostics.buflen = BUFLEN;
1953 rec->diagnostics.data = emalloc(rec->diagnostics.buflen);
1954 rec->diagnostics.offset = 0;
1955
1956 rec->errors.buflen = BUFLEN;
1957 rec->errors.data = emalloc(rec->errors.buflen);
1958 rec->errors.offset = 0;
1959 }
1960
1961 /*
1962 * free_secbuffs--
1963 * This function should be called at the end, when all the pages have been
1964 * parsed.
1965 * It frees the memory allocated to sec_buffs by init_secbuffs in the starting.
1966 */
1967 static void
1968 free_secbuffs(mandb_rec *rec)
1969 {
1970 free(rec->desc.data);
1971 free(rec->lib.data);
1972 free(rec->return_vals.data);
1973 free(rec->exit_status.data);
1974 free(rec->env.data);
1975 free(rec->files.data);
1976 free(rec->diagnostics.data);
1977 free(rec->errors.data);
1978 }
1979
1980 static void
1981 replace_hyph(char *str)
1982 {
1983 char *iter = str;
1984 while ((iter = strchr(iter, ASCII_HYPH)) != NULL)
1985 *iter = '-';
1986
1987 iter = str;
1988 while ((iter = strchr(iter, ASCII_NBRSP)) != NULL)
1989 *iter = '-';
1990 }
1991
1992 static char *
1993 parse_escape(const char *str)
1994 {
1995 const char *backslash, *last_backslash;
1996 char *result, *iter;
1997 size_t len;
1998
1999 assert(str);
2000
2001 last_backslash = str;
2002 backslash = strchr(str, '\\');
2003 if (backslash == NULL) {
2004 result = estrdup(str);
2005 replace_hyph(result);
2006 return result;
2007 }
2008
2009 result = emalloc(strlen(str) + 1);
2010 iter = result;
2011
2012 do {
2013 len = backslash - last_backslash;
2014 memcpy(iter, last_backslash, len);
2015 iter += len;
2016 if (backslash[1] == '-' || backslash[1] == ' ') {
2017 *iter++ = backslash[1];
2018 last_backslash = backslash + 2;
2019 backslash = strchr(last_backslash, '\\');
2020 } else {
2021 ++backslash;
2022 mandoc_escape(&backslash, NULL, NULL);
2023 last_backslash = backslash;
2024 if (backslash == NULL)
2025 break;
2026 backslash = strchr(last_backslash, '\\');
2027 }
2028 } while (backslash != NULL);
2029 if (last_backslash != NULL)
2030 strcpy(iter, last_backslash);
2031
2032 replace_hyph(result);
2033 return result;
2034 }
2035
2036 /*
2037 * append--
2038 * Concatenates a space and src at the end of sbuff->data (much like concat in
2039 * apropos-utils.c).
2040 * Rather than reallocating space for writing data, it uses the value of the
2041 * offset field of sec_buff to write new data at the free space left in the
2042 * buffer.
2043 * In case the size of the data to be appended exceeds the number of bytes left
2044 * in the buffer, it reallocates buflen number of bytes and then continues.
2045 * Value of offset field should be adjusted as new data is written.
2046 *
2047 * NOTE: This function does not write the null byte at the end of the buffers,
2048 * write a null byte at the position pointed to by offset before inserting data
2049 * in the db.
2050 */
2051 static void
2052 append(secbuff *sbuff, const char *src)
2053 {
2054 short flag = 0;
2055 size_t srclen, newlen;
2056 char *temp;
2057
2058 assert(src != NULL);
2059 temp = parse_escape(src);
2060 srclen = strlen(temp);
2061
2062 if (sbuff->data == NULL) {
2063 sbuff->data = emalloc(sbuff->buflen);
2064 sbuff->offset = 0;
2065 }
2066
2067 newlen = sbuff->offset + srclen + 2;
2068 if (newlen >= sbuff->buflen) {
2069 while (sbuff->buflen < newlen)
2070 sbuff->buflen += sbuff->buflen;
2071 sbuff->data = erealloc(sbuff->data, sbuff->buflen);
2072 flag = 1;
2073 }
2074
2075 /* Append a space at the end of the buffer. */
2076 if (sbuff->offset || flag)
2077 sbuff->data[sbuff->offset++] = ' ';
2078 /* Now, copy src at the end of the buffer. */
2079 memcpy(sbuff->data + sbuff->offset, temp, srclen);
2080 sbuff->offset += srclen;
2081 free(temp);
2082 }
2083
2084 static void
2085 usage(void)
2086 {
2087 fprintf(stderr, "Usage: %s [-floQqv] [-C path]\n", getprogname());
2088 exit(1);
2089 }
2090