makemandb.c revision 1.47.2.2 1 /* $NetBSD: makemandb.c,v 1.47.2.2 2017/05/11 02:58:43 pgoyette Exp $ */
2 /*
3 * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay (at) gmail.com>
4 * Copyright (c) 2011 Kristaps Dzonsons <kristaps (at) bsd.lv>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19 #include <sys/cdefs.h>
20 __RCSID("$NetBSD: makemandb.c,v 1.47.2.2 2017/05/11 02:58:43 pgoyette Exp $");
21
22 #include <sys/stat.h>
23 #include <sys/types.h>
24
25 #include <assert.h>
26 #include <dirent.h>
27 #include <err.h>
28 #include <archive.h>
29 #include <libgen.h>
30 #include <md5.h>
31 #include <stdio.h>
32 #include <stdlib.h>
33 #include <string.h>
34 #include <unistd.h>
35 #include <util.h>
36
37 #include "apropos-utils.h"
38 #include "dist/man.h"
39 #include "dist/mandoc.h"
40 #include "dist/mdoc.h"
41 #include "dist/roff.h"
42
43 #define BUFLEN 1024
44 #define MDOC 0 //If the page is of mdoc(7) type
45 #define MAN 1 //If the page is of man(7) type
46
47 /*
48 * A data structure for holding section specific data.
49 */
50 typedef struct secbuff {
51 char *data;
52 size_t buflen; //Total length of buffer allocated initially
53 size_t offset; // Current offset in the buffer.
54 } secbuff;
55
56 typedef struct makemandb_flags {
57 int optimize;
58 int limit; // limit the indexing to only NAME section
59 int recreate; // Database was created from scratch
60 int verbosity; // 0: quiet, 1: default, 2: verbose
61 } makemandb_flags;
62
63 typedef struct roff_mandb_rec {
64 /* Fields for mandb table */
65 char *name; // for storing the name of the man page
66 char *name_desc; // for storing the one line description (.Nd)
67 secbuff desc; // for storing the DESCRIPTION section
68 secbuff lib; // for the LIBRARY section
69 secbuff return_vals; // RETURN VALUES
70 secbuff env; // ENVIRONMENT
71 secbuff files; // FILES
72 secbuff exit_status; // EXIT STATUS
73 secbuff diagnostics; // DIAGNOSTICS
74 secbuff errors; // ERRORS
75 char *section;
76
77 int xr_found; // To track whether a .Xr was seen when parsing a section
78
79 /* Fields for mandb_meta table */
80 char *md5_hash;
81 dev_t device;
82 ino_t inode;
83 time_t mtime;
84
85 /* Fields for mandb_links table */
86 char *machine;
87 char *links; //all the links to a page in a space separated form
88 char *file_path;
89
90 /* Non-db fields */
91 int page_type; //Indicates the type of page: mdoc or man
92 } mandb_rec;
93
94 typedef void (*proff_nf)(const struct roff_node *n, mandb_rec *);
95
96 static void append(secbuff *sbuff, const char *src);
97 static void init_secbuffs(mandb_rec *);
98 static void free_secbuffs(mandb_rec *);
99 static int check_md5(const char *, sqlite3 *, char **, void *, size_t);
100 static void cleanup(mandb_rec *);
101 static void set_section(const struct roff_man *, mandb_rec *);
102 static void set_machine(const struct roff_man *, mandb_rec *);
103 static int insert_into_db(sqlite3 *, mandb_rec *);
104 static void begin_parse(const char *, struct mparse *, mandb_rec *,
105 const void *, size_t len);
106 static void proff_node(const struct roff_node *, mandb_rec *, const proff_nf *);
107 static void pmdoc_Nm(const struct roff_node *, mandb_rec *);
108 static void pmdoc_Nd(const struct roff_node *, mandb_rec *);
109 static void pmdoc_Sh(const struct roff_node *, mandb_rec *);
110 static void mdoc_parse_Sh(const struct roff_node *, mandb_rec *);
111 static void pmdoc_Xr(const struct roff_node *, mandb_rec *);
112 static void pmdoc_Pp(const struct roff_node *, mandb_rec *);
113 static void pmdoc_macro_handler(const struct roff_node *, mandb_rec *, int);
114 static void pman_parse_node(const struct roff_node *, secbuff *);
115 static void pman_parse_name(const struct roff_node *, mandb_rec *);
116 static void pman_sh(const struct roff_node *, mandb_rec *);
117 static void pman_block(const struct roff_node *, mandb_rec *);
118 static void traversedir(const char *, const char *, sqlite3 *, struct mparse *);
119 static void mdoc_parse_section(enum roff_sec, const char *, mandb_rec *);
120 static void man_parse_section(enum man_sec, const struct roff_node *, mandb_rec *);
121 static void build_file_cache(sqlite3 *, const char *, const char *,
122 struct stat *);
123 static void update_db(sqlite3 *, struct mparse *, mandb_rec *);
124 __dead static void usage(void);
125 static void optimize(sqlite3 *);
126 static char *parse_escape(const char *);
127 static void replace_hyph(char *);
128 static makemandb_flags mflags = { .verbosity = 1 };
129
130 static const proff_nf mdocs[MDOC_MAX + 1] = {
131 NULL, /* Ap */
132 NULL, /* Dd */
133 NULL, /* Dt */
134 NULL, /* Os */
135
136 pmdoc_Sh, /* Sh */
137 NULL, /* Ss */
138 pmdoc_Pp, /* Pp */
139 NULL, /* D1 */
140
141 NULL, /* Dl */
142 NULL, /* Bd */
143 NULL, /* Ed */
144 NULL, /* Bl */
145
146 NULL, /* El */
147 NULL, /* It */
148 NULL, /* Ad */
149 NULL, /* An */
150
151 NULL, /* Ar */
152 NULL, /* Cd */
153 NULL, /* Cm */
154 NULL, /* Dv */
155
156 NULL, /* Er */
157 NULL, /* Ev */
158 NULL, /* Ex */
159 NULL, /* Fa */
160
161 NULL, /* Fd */
162 NULL, /* Fl */
163 NULL, /* Fn */
164 NULL, /* Ft */
165
166 NULL, /* Ic */
167 NULL, /* In */
168 NULL, /* Li */
169 pmdoc_Nd, /* Nd */
170
171 pmdoc_Nm, /* Nm */
172 NULL, /* Op */
173 NULL, /* Ot */
174 NULL, /* Pa */
175
176 NULL, /* Rv */
177 NULL, /* St */
178 NULL, /* Va */
179 NULL, /* Vt */
180
181 pmdoc_Xr, /* Xr */
182 NULL, /* %A */
183 NULL, /* %B */
184 NULL, /* %D */
185
186 NULL, /* %I */
187 NULL, /* %J */
188 NULL, /* %N */
189 NULL, /* %O */
190
191 NULL, /* %P */
192 NULL, /* %R */
193 NULL, /* %T */
194 NULL, /* %V */
195
196 NULL, /* Ac */
197 NULL, /* Ao */
198 NULL, /* Aq */
199 NULL, /* At */
200
201 NULL, /* Bc */
202 NULL, /* Bf */
203 NULL, /* Bo */
204 NULL, /* Bq */
205
206 NULL, /* Bsx */
207 NULL, /* Bx */
208 NULL, /* Db */
209 NULL, /* Dc */
210
211 NULL, /* Do */
212 NULL, /* Dq */
213 NULL, /* Ec */
214 NULL, /* Ef */
215
216 NULL, /* Em */
217 NULL, /* Eo */
218 NULL, /* Fx */
219 NULL, /* Ms */
220
221 NULL, /* No */
222 NULL, /* Ns */
223 NULL, /* Nx */
224 NULL, /* Ox */
225
226 NULL, /* Pc */
227 NULL, /* Pf */
228 NULL, /* Po */
229 NULL, /* Pq */
230
231 NULL, /* Qc */
232 NULL, /* Ql */
233 NULL, /* Qo */
234 NULL, /* Qq */
235
236 NULL, /* Re */
237 NULL, /* Rs */
238 NULL, /* Sc */
239 NULL, /* So */
240
241 NULL, /* Sq */
242 NULL, /* Sm */
243 NULL, /* Sx */
244 NULL, /* Sy */
245
246 NULL, /* Tn */
247 NULL, /* Ux */
248 NULL, /* Xc */
249 NULL, /* Xo */
250
251 NULL, /* Fo */
252 NULL, /* Fc */
253 NULL, /* Oo */
254 NULL, /* Oc */
255
256 NULL, /* Bk */
257 NULL, /* Ek */
258 NULL, /* Bt */
259 NULL, /* Hf */
260
261 NULL, /* Fr */
262 NULL, /* Ud */
263 NULL, /* Lb */
264 NULL, /* Lp */
265
266 NULL, /* Lk */
267 NULL, /* Mt */
268 NULL, /* Brq */
269 NULL, /* Bro */
270
271 NULL, /* Brc */
272 NULL, /* %C */
273 NULL, /* Es */
274 NULL, /* En */
275
276 NULL, /* Dx */
277 NULL, /* %Q */
278 NULL, /* br */
279 NULL, /* sp */
280
281 NULL, /* %U */
282 NULL, /* Ta */
283 NULL, /* ll */
284 NULL, /* text */
285 };
286
287 static const proff_nf mans[MAN_MAX] = {
288 NULL, //br
289 NULL, //TH
290 pman_sh, //SH
291 NULL, //SS
292 NULL, //TP
293 NULL, //LP
294 NULL, //PP
295 NULL, //P
296 NULL, //IP
297 NULL, //HP
298 NULL, //SM
299 NULL, //SB
300 NULL, //BI
301 NULL, //IB
302 NULL, //BR
303 NULL, //RB
304 NULL, //R
305 pman_block, //B
306 NULL, //I
307 NULL, //IR
308 NULL, //RI
309 NULL, //sp
310 NULL, //nf
311 NULL, //fi
312 NULL, //RE
313 NULL, //RS
314 NULL, //DT
315 NULL, //UC
316 NULL, //PD
317 NULL, //AT
318 NULL, //in
319 NULL, //ft
320 NULL, //OP
321 NULL, //EX
322 NULL, //EE
323 NULL, //UR
324 NULL, //UE
325 NULL, //ll
326 };
327
328
329 int
330 main(int argc, char *argv[])
331 {
332 FILE *file;
333 const char *sqlstr, *manconf = NULL;
334 char *line, *command;
335 char *errmsg;
336 int ch;
337 struct mparse *mp;
338 sqlite3 *db;
339 ssize_t len;
340 size_t linesize;
341 struct roff_mandb_rec rec;
342
343 while ((ch = getopt(argc, argv, "C:floQqv")) != -1) {
344 switch (ch) {
345 case 'C':
346 manconf = optarg;
347 break;
348 case 'f':
349 mflags.recreate = 1;
350 break;
351 case 'l':
352 mflags.limit = 1;
353 break;
354 case 'o':
355 mflags.optimize = 1;
356 break;
357 case 'Q':
358 mflags.verbosity = 0;
359 break;
360 case 'q':
361 mflags.verbosity = 1;
362 break;
363 case 'v':
364 mflags.verbosity = 2;
365 break;
366 default:
367 usage();
368 }
369 }
370
371 memset(&rec, 0, sizeof(rec));
372
373 init_secbuffs(&rec);
374 mchars_alloc();
375 mp = mparse_alloc(0, MANDOCLEVEL_BADARG, NULL, NULL);
376
377 if (manconf) {
378 char *arg;
379 size_t command_len = shquote(manconf, NULL, 0) + 1;
380 arg = emalloc(command_len);
381 shquote(manconf, arg, command_len);
382 easprintf(&command, "man -p -C %s", arg);
383 free(arg);
384 } else {
385 command = estrdup("man -p");
386 manconf = MANCONF;
387 }
388
389 if (mflags.recreate) {
390 char *dbp = get_dbpath(manconf);
391 /* No error here, it will fail in init_db in the same call */
392 if (dbp != NULL)
393 remove(dbp);
394 }
395
396 if ((db = init_db(MANDB_CREATE, manconf)) == NULL)
397 exit(EXIT_FAILURE);
398
399 sqlite3_exec(db, "PRAGMA synchronous = 0", NULL, NULL, &errmsg);
400 if (errmsg != NULL) {
401 warnx("%s", errmsg);
402 free(errmsg);
403 close_db(db);
404 exit(EXIT_FAILURE);
405 }
406
407 sqlite3_exec(db, "ATTACH DATABASE \':memory:\' AS metadb", NULL, NULL,
408 &errmsg);
409 if (errmsg != NULL) {
410 warnx("%s", errmsg);
411 free(errmsg);
412 close_db(db);
413 exit(EXIT_FAILURE);
414 }
415
416
417 /* Call man -p to get the list of man page dirs */
418 if ((file = popen(command, "r")) == NULL) {
419 close_db(db);
420 err(EXIT_FAILURE, "fopen failed");
421 }
422 free(command);
423
424 /* Begin the transaction for indexing the pages */
425 sqlite3_exec(db, "BEGIN", NULL, NULL, &errmsg);
426 if (errmsg != NULL) {
427 warnx("%s", errmsg);
428 free(errmsg);
429 close_db(db);
430 exit(EXIT_FAILURE);
431 }
432
433 sqlstr = "CREATE TABLE metadb.file_cache(device, inode, mtime, parent,"
434 " file PRIMARY KEY);"
435 "CREATE UNIQUE INDEX metadb.index_file_cache_dev"
436 " ON file_cache (device, inode)";
437
438 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
439 if (errmsg != NULL) {
440 warnx("%s", errmsg);
441 free(errmsg);
442 close_db(db);
443 exit(EXIT_FAILURE);
444 }
445
446 if (mflags.verbosity)
447 printf("Building temporary file cache\n");
448 line = NULL;
449 linesize = 0;
450 while ((len = getline(&line, &linesize, file)) != -1) {
451 /* Replace the new line character at the end of string with '\0' */
452 line[len - 1] = '\0';
453 char *pdir = estrdup(dirname(line));
454 /* Traverse the man page directories and parse the pages */
455 traversedir(pdir, line, db, mp);
456 free(pdir);
457 }
458 free(line);
459
460 if (pclose(file) == -1) {
461 close_db(db);
462 cleanup(&rec);
463 free_secbuffs(&rec);
464 err(EXIT_FAILURE, "pclose error");
465 }
466
467 if (mflags.verbosity)
468 printf("Performing index update\n");
469 update_db(db, mp, &rec);
470 mparse_free(mp);
471 mchars_free();
472 free_secbuffs(&rec);
473
474 /* Commit the transaction */
475 sqlite3_exec(db, "COMMIT", NULL, NULL, &errmsg);
476 if (errmsg != NULL) {
477 warnx("%s", errmsg);
478 free(errmsg);
479 close_db(db);
480 exit(EXIT_FAILURE);
481 }
482
483 if (mflags.optimize)
484 optimize(db);
485
486 close_db(db);
487 return 0;
488 }
489
490 /*
491 * traversedir --
492 * Traverses the given directory recursively and passes all the man page files
493 * in the way to build_file_cache()
494 */
495 static void
496 traversedir(const char *parent, const char *file, sqlite3 *db,
497 struct mparse *mp)
498 {
499 struct stat sb;
500 struct dirent *dirp;
501 DIR *dp;
502 char *buf;
503
504 if (stat(file, &sb) < 0) {
505 if (mflags.verbosity)
506 warn("stat failed: %s", file);
507 return;
508 }
509
510 /* If it is a directory, traverse it recursively */
511 if (S_ISDIR(sb.st_mode)) {
512 if ((dp = opendir(file)) == NULL) {
513 if (mflags.verbosity)
514 warn("opendir error: %s", file);
515 return;
516 }
517
518 while ((dirp = readdir(dp)) != NULL) {
519 /* Avoid . and .. entries in a directory */
520 if (dirp->d_name[0] != '.') {
521 easprintf(&buf, "%s/%s", file, dirp->d_name);
522 traversedir(parent, buf, db, mp);
523 free(buf);
524 }
525 }
526 closedir(dp);
527 return;
528 }
529
530 if (!S_ISREG(sb.st_mode))
531 return;
532
533 if (sb.st_size == 0) {
534 if (mflags.verbosity)
535 warnx("Empty file: %s", file);
536 return;
537 }
538 build_file_cache(db, parent, file, &sb);
539 }
540
541 /* build_file_cache --
542 * This function generates an md5 hash of the file passed as its 2nd parameter
543 * and stores it in a temporary table file_cache along with the full file path.
544 * This is done to support incremental updation of the database.
545 * The temporary table file_cache is dropped thereafter in the function
546 * update_db(), once the database has been updated.
547 */
548 static void
549 build_file_cache(sqlite3 *db, const char *parent, const char *file,
550 struct stat *sb)
551 {
552 const char *sqlstr;
553 sqlite3_stmt *stmt = NULL;
554 int rc, idx;
555 assert(file != NULL);
556 dev_t device_cache = sb->st_dev;
557 ino_t inode_cache = sb->st_ino;
558 time_t mtime_cache = sb->st_mtime;
559
560 sqlstr = "INSERT INTO metadb.file_cache VALUES (:device, :inode,"
561 " :mtime, :parent, :file)";
562 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
563 if (rc != SQLITE_OK) {
564 if (mflags.verbosity)
565 warnx("%s", sqlite3_errmsg(db));
566 return;
567 }
568
569 idx = sqlite3_bind_parameter_index(stmt, ":device");
570 rc = sqlite3_bind_int64(stmt, idx, device_cache);
571 if (rc != SQLITE_OK) {
572 if (mflags.verbosity)
573 warnx("%s", sqlite3_errmsg(db));
574 sqlite3_finalize(stmt);
575 return;
576 }
577
578 idx = sqlite3_bind_parameter_index(stmt, ":inode");
579 rc = sqlite3_bind_int64(stmt, idx, inode_cache);
580 if (rc != SQLITE_OK) {
581 if (mflags.verbosity)
582 warnx("%s", sqlite3_errmsg(db));
583 sqlite3_finalize(stmt);
584 return;
585 }
586
587 idx = sqlite3_bind_parameter_index(stmt, ":mtime");
588 rc = sqlite3_bind_int64(stmt, idx, mtime_cache);
589 if (rc != SQLITE_OK) {
590 if (mflags.verbosity)
591 warnx("%s", sqlite3_errmsg(db));
592 sqlite3_finalize(stmt);
593 return;
594 }
595
596 idx = sqlite3_bind_parameter_index(stmt, ":parent");
597 rc = sqlite3_bind_text(stmt, idx, parent, -1, NULL);
598 if (rc != SQLITE_OK) {
599 if (mflags.verbosity)
600 warnx("%s", sqlite3_errmsg(db));
601 sqlite3_finalize(stmt);
602 return;
603 }
604
605 idx = sqlite3_bind_parameter_index(stmt, ":file");
606 rc = sqlite3_bind_text(stmt, idx, file, -1, NULL);
607 if (rc != SQLITE_OK) {
608 if (mflags.verbosity)
609 warnx("%s", sqlite3_errmsg(db));
610 sqlite3_finalize(stmt);
611 return;
612 }
613
614 sqlite3_step(stmt);
615 sqlite3_finalize(stmt);
616 }
617
618 static void
619 update_existing_entry(sqlite3 *db, const char *file, const char *hash,
620 mandb_rec *rec, int *new_count, int *link_count, int *err_count)
621 {
622 int update_count, rc, idx;
623 const char *inner_sqlstr;
624 sqlite3_stmt *inner_stmt;
625
626 update_count = sqlite3_total_changes(db);
627 inner_sqlstr = "UPDATE mandb_meta SET device = :device,"
628 " inode = :inode, mtime = :mtime WHERE"
629 " md5_hash = :md5 AND file = :file AND"
630 " (device <> :device2 OR inode <> "
631 " :inode2 OR mtime <> :mtime2)";
632 rc = sqlite3_prepare_v2(db, inner_sqlstr, -1, &inner_stmt, NULL);
633 if (rc != SQLITE_OK) {
634 if (mflags.verbosity)
635 warnx("%s", sqlite3_errmsg(db));
636 return;
637 }
638 idx = sqlite3_bind_parameter_index(inner_stmt, ":device");
639 sqlite3_bind_int64(inner_stmt, idx, rec->device);
640 idx = sqlite3_bind_parameter_index(inner_stmt, ":inode");
641 sqlite3_bind_int64(inner_stmt, idx, rec->inode);
642 idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime");
643 sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
644 idx = sqlite3_bind_parameter_index(inner_stmt, ":md5");
645 sqlite3_bind_text(inner_stmt, idx, hash, -1, NULL);
646 idx = sqlite3_bind_parameter_index(inner_stmt, ":file");
647 sqlite3_bind_text(inner_stmt, idx, file, -1, NULL);
648 idx = sqlite3_bind_parameter_index(inner_stmt, ":device2");
649 sqlite3_bind_int64(inner_stmt, idx, rec->device);
650 idx = sqlite3_bind_parameter_index(inner_stmt, ":inode2");
651 sqlite3_bind_int64(inner_stmt, idx, rec->inode);
652 idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime2");
653 sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
654
655 rc = sqlite3_step(inner_stmt);
656 if (rc == SQLITE_DONE) {
657 /* Check if an update has been performed. */
658 if (update_count != sqlite3_total_changes(db)) {
659 if (mflags.verbosity == 2)
660 printf("Updated %s\n", file);
661 (*new_count)++;
662 } else {
663 /* Otherwise it was a hardlink. */
664 (*link_count)++;
665 }
666 } else {
667 if (mflags.verbosity == 2)
668 warnx("Could not update the meta data for %s", file);
669 (*err_count)++;
670 }
671 sqlite3_finalize(inner_stmt);
672 }
673
674 /* read_and_decompress --
675 * Reads the given file into memory. If it is compressed, decompress
676 * it before returning to the caller.
677 */
678 static int
679 read_and_decompress(const char *file, void **bufp, size_t *len)
680 {
681 size_t off;
682 ssize_t r;
683 struct archive *a;
684 struct archive_entry *ae;
685 char *buf;
686
687 if ((a = archive_read_new()) == NULL)
688 errx(EXIT_FAILURE, "memory allocation failed");
689
690 *bufp = NULL;
691 if (archive_read_support_filter_all(a) != ARCHIVE_OK ||
692 archive_read_support_format_raw(a) != ARCHIVE_OK ||
693 archive_read_open_filename(a, file, 65536) != ARCHIVE_OK ||
694 archive_read_next_header(a, &ae) != ARCHIVE_OK)
695 goto archive_error;
696 *len = 65536;
697 buf = emalloc(*len);
698 off = 0;
699 for (;;) {
700 r = archive_read_data(a, buf + off, *len - off);
701 if (r == ARCHIVE_OK) {
702 archive_read_free(a);
703 *bufp = buf;
704 *len = off;
705 return 0;
706 }
707 if (r <= 0) {
708 free(buf);
709 break;
710 }
711 off += r;
712 if (off == *len) {
713 *len *= 2;
714 if (*len < off) {
715 if (mflags.verbosity)
716 warnx("File too large: %s", file);
717 free(buf);
718 archive_read_free(a);
719 return -1;
720 }
721 buf = erealloc(buf, *len);
722 }
723 }
724
725 archive_error:
726 warnx("Error while reading `%s': %s", file, archive_error_string(a));
727 archive_read_free(a);
728 return -1;
729 }
730
731 /* update_db --
732 * Does an incremental updation of the database by checking the file_cache.
733 * It parses and adds the pages which are present in file_cache,
734 * but not in the database.
735 * It also removes the pages which are present in the databse,
736 * but not in the file_cache.
737 */
738 static void
739 update_db(sqlite3 *db, struct mparse *mp, mandb_rec *rec)
740 {
741 const char *sqlstr;
742 sqlite3_stmt *stmt = NULL;
743 char *file;
744 char *parent;
745 char *errmsg = NULL;
746 char *md5sum;
747 void *buf;
748 size_t buflen;
749 struct sql_row {
750 struct sql_row *next;
751 dev_t device;
752 ino_t inode;
753 time_t mtime;
754 char *parent;
755 char *file;
756 } *rows, *row;
757 int new_count = 0; /* Counter for newly indexed/updated pages */
758 int total_count = 0; /* Counter for total number of pages */
759 int err_count = 0; /* Counter for number of failed pages */
760 int link_count = 0; /* Counter for number of hard/sym links */
761 int md5_status;
762 int rc;
763
764 sqlstr = "SELECT device, inode, mtime, parent, file"
765 " FROM metadb.file_cache fc"
766 " WHERE NOT EXISTS(SELECT 1 FROM mandb_meta WHERE"
767 " device = fc.device AND inode = fc.inode AND "
768 " mtime = fc.mtime AND file = fc.file)";
769
770 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
771 if (rc != SQLITE_OK) {
772 if (mflags.verbosity)
773 warnx("%s", sqlite3_errmsg(db));
774 close_db(db);
775 errx(EXIT_FAILURE, "Could not query file cache");
776 }
777
778 buf = NULL;
779 rows = NULL;
780 while (sqlite3_step(stmt) == SQLITE_ROW) {
781 row = emalloc(sizeof(struct sql_row));
782 row->device = sqlite3_column_int64(stmt, 0);
783 row->inode = sqlite3_column_int64(stmt, 1);
784 row->mtime = sqlite3_column_int64(stmt, 2);
785 row->parent = estrdup((const char *) sqlite3_column_text(stmt, 3));
786 row->file = estrdup((const char *) sqlite3_column_text(stmt, 4));
787 row->next = rows;
788 rows = row;
789 total_count++;
790 }
791 sqlite3_finalize(stmt);
792
793 for ( ; rows != NULL; free(parent), free(file), free(buf)) {
794 row = rows;
795 rows = rows->next;
796
797 rec->device = row->device;
798 rec->inode = row->inode;
799 rec->mtime = row->mtime;
800 parent = row->parent;
801 file = row->file;
802 free(row);
803
804 if (read_and_decompress(file, &buf, &buflen)) {
805 err_count++;
806 continue;
807 }
808 md5_status = check_md5(file, db, &md5sum, buf, buflen);
809 assert(md5sum != NULL);
810 if (md5_status == -1) {
811 if (mflags.verbosity)
812 warnx("An error occurred in checking md5 value"
813 " for file %s", file);
814 err_count++;
815 continue;
816 }
817
818 if (md5_status == 0) {
819 /*
820 * The MD5 hash is already present in the database,
821 * so simply update the metadata.
822 */
823 update_existing_entry(db, file, md5sum, rec,
824 &new_count, &link_count, &err_count);
825 free(md5sum);
826 continue;
827 }
828
829 if (md5_status == 1) {
830 /*
831 * The MD5 hash was not present in the database.
832 * This means is either a new file or an updated file.
833 * We should go ahead with parsing.
834 */
835 if (chdir(parent) == -1) {
836 if (mflags.verbosity)
837 warn("chdir failed for `%s', could "
838 "not index `%s'", parent, file);
839 err_count++;
840 free(md5sum);
841 continue;
842 }
843
844 if (mflags.verbosity == 2)
845 printf("Parsing: %s\n", file);
846 rec->md5_hash = md5sum;
847 rec->file_path = estrdup(file);
848 // file_path is freed by insert_into_db itself.
849 begin_parse(file, mp, rec, buf, buflen);
850 if (insert_into_db(db, rec) < 0) {
851 if (mflags.verbosity)
852 warnx("Error in indexing `%s'", file);
853 err_count++;
854 } else {
855 new_count++;
856 }
857 }
858 }
859
860 if (mflags.verbosity == 2) {
861 printf("Number of new or updated pages encountered: %d\n"
862 "Number of hard links found: %d\n"
863 "Number of pages that were successfully"
864 " indexed or updated: %d\n"
865 "Number of pages that could not be indexed"
866 " due to errors: %d\n",
867 total_count - link_count, link_count, new_count, err_count);
868 }
869
870 if (mflags.recreate)
871 return;
872
873 if (mflags.verbosity == 2)
874 printf("Deleting stale index entries\n");
875
876 sqlstr = "DELETE FROM mandb_meta WHERE file NOT IN"
877 " (SELECT file FROM metadb.file_cache);"
878 "DELETE FROM mandb_links WHERE md5_hash NOT IN"
879 " (SELECT md5_hash from mandb_meta);"
880 "DROP TABLE metadb.file_cache;"
881 "DELETE FROM mandb WHERE rowid NOT IN"
882 " (SELECT id FROM mandb_meta);";
883
884 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
885 if (errmsg != NULL) {
886 warnx("Removing old entries failed: %s", errmsg);
887 warnx("Please rebuild database from scratch with -f.");
888 free(errmsg);
889 return;
890 }
891 }
892
893 /*
894 * begin_parse --
895 * parses the man page using libmandoc
896 */
897 static void
898 begin_parse(const char *file, struct mparse *mp, mandb_rec *rec,
899 const void *buf, size_t len)
900 {
901 struct roff_man *roff;
902 mparse_reset(mp);
903
904 rec->xr_found = 0;
905
906 if (mparse_readmem(mp, buf, len, file) >= MANDOCLEVEL_BADARG) {
907 /* Printing this warning at verbosity level 2
908 * because some packages from pkgsrc might trigger several
909 * of such warnings.
910 */
911 if (mflags.verbosity == 2)
912 warnx("%s: Parse failure", file);
913 return;
914 }
915
916 mparse_result(mp, &roff, NULL);
917 if (roff == NULL) {
918 if (mflags.verbosity == 2)
919 warnx("Not a roff(7) page");
920 return;
921 }
922
923 if (roff->macroset == MACROSET_MDOC) {
924 mdoc_validate(roff);
925 rec->page_type = MDOC;
926 proff_node(roff->first->child, rec, mdocs);
927 } else if (roff->macroset == MACROSET_MAN) {
928 man_validate(roff);
929 rec->page_type = MAN;
930 proff_node(roff->first->child, rec, mans);
931 } else
932 warnx("Unknown macroset %d", roff->macroset);
933 set_machine(roff, rec);
934 set_section(roff, rec);
935 }
936
937 /*
938 * set_section --
939 * Extracts the section number and normalizes it to only the numeric part
940 * (Which should be the first character of the string).
941 */
942 static void
943 set_section(const struct roff_man *rm, mandb_rec *rec)
944 {
945 if (!rm)
946 return;
947 const char *s = rm->meta.msec == NULL ? "?" : rm->meta.msec;
948 easprintf(&rec->section, "%s", s);
949 if (rec->section[0] == '?' && mflags.verbosity == 2)
950 warnx("%s: Missing section number", rec->file_path);
951 }
952
953 /*
954 * get_machine --
955 * Extracts the machine architecture information if available.
956 */
957 static void
958 set_machine(const struct roff_man *rm, mandb_rec *rec)
959 {
960 if (rm == NULL)
961 return;
962 if (rm->meta.arch)
963 rec->machine = estrdup(rm->meta.arch);
964 }
965
966 /*
967 * pmdoc_Nm --
968 * Extracts the Name of the manual page from the .Nm macro
969 */
970 static void
971 pmdoc_Nm(const struct roff_node *n, mandb_rec *rec)
972 {
973 if (n->sec != SEC_NAME)
974 return;
975
976 for (n = n->child; n; n = n->next) {
977 if (n->type == ROFFT_TEXT) {
978 char *escaped_name = parse_escape(n->string);
979 concat(&rec->name, escaped_name);
980 free(escaped_name);
981 }
982 }
983 }
984
985 /*
986 * pmdoc_Nd --
987 * Extracts the one line description of the man page from the .Nd macro
988 */
989 static void
990 pmdoc_Nd(const struct roff_node *n, mandb_rec *rec)
991 {
992 if (n->type == ROFFT_BODY)
993 deroff(&rec->name_desc, n);
994 if (rec->name_desc)
995 replace_hyph(rec->name_desc);
996
997 }
998
999 /*
1000 * pmdoc_macro_handler--
1001 * This function is a single point of handling all the special macros that we
1002 * want to handle especially. For example the .Xr macro for properly parsing
1003 * the referenced page name along with the section number, or the .Pp macro
1004 * for adding a new line whenever we encounter it.
1005 */
1006 static void
1007 pmdoc_macro_handler(const struct roff_node *n, mandb_rec *rec, int doct)
1008 {
1009 const struct roff_node *sn;
1010 assert(n);
1011
1012 switch (doct) {
1013 /* Parse the man page references.
1014 * Basically the .Xr macros are used like:
1015 * .Xr ls 1
1016 * and formatted like this:
1017 * ls(1)
1018 * Prepare a buffer to format the data like the above example and call
1019 * pmdoc_parse_section to append it.
1020 */
1021 case MDOC_Xr:
1022 n = n->child;
1023 while (n->type != ROFFT_TEXT && n->next)
1024 n = n->next;
1025
1026 if (n && n->type != ROFFT_TEXT)
1027 return;
1028 sn = n;
1029 if (n->next)
1030 n = n->next;
1031
1032 while (n->type != ROFFT_TEXT && n->next)
1033 n = n->next;
1034
1035 if (n && n->type == ROFFT_TEXT) {
1036 char *buf;
1037 easprintf(&buf, "%s(%s)", sn->string, n->string);
1038 mdoc_parse_section(n->sec, buf, rec);
1039 free(buf);
1040 }
1041
1042 break;
1043
1044 /* Parse the .Pp macro to add a new line */
1045 case MDOC_Pp:
1046 if (n->type == ROFFT_TEXT)
1047 mdoc_parse_section(n->sec, "\n", rec);
1048 break;
1049 default:
1050 break;
1051 }
1052
1053 }
1054
1055 /*
1056 * pmdoc_Xr, pmdoc_Pp--
1057 * Empty stubs.
1058 * The parser calls these functions each time it encounters
1059 * a .Xr or .Pp macro. We are parsing all the data from
1060 * the pmdoc_Sh function, so don't do anything here.
1061 * (See if else blocks in pmdoc_Sh.)
1062 */
1063 static void
1064 pmdoc_Xr(const struct roff_node *n, mandb_rec *rec)
1065 {
1066 }
1067
1068 static void
1069 pmdoc_Pp(const struct roff_node *n, mandb_rec *rec)
1070 {
1071 }
1072
1073 /*
1074 * pmdoc_Sh --
1075 * Called when a .Sh macro is encountered and tries to parse its body
1076 */
1077 static void
1078 pmdoc_Sh(const struct roff_node *n, mandb_rec *rec)
1079 {
1080 if (n == NULL)
1081 return;
1082
1083 switch (n->sec) {
1084 case SEC_NAME:
1085 case SEC_SYNOPSIS:
1086 case SEC_EXAMPLES:
1087 case SEC_STANDARDS:
1088 case SEC_HISTORY:
1089 case SEC_AUTHORS:
1090 case SEC_BUGS:
1091 /*
1092 * We don't care about text from these sections
1093 */
1094 return;
1095 default:
1096 break;
1097 }
1098
1099 if (n->type == ROFFT_BLOCK)
1100 mdoc_parse_Sh(n->body, rec);
1101 }
1102
1103 /*
1104 * Called from pmdoc_Sh to parse body of a .Sh macro. It calls
1105 * mdoc_parse_section to append the data to the section specific buffer.
1106 * The .Xr macro needs special handling, thus the separate if branch for it.
1107 */
1108 static void
1109 mdoc_parse_Sh(const struct roff_node *n, mandb_rec *rec)
1110 {
1111 if (n == NULL || (n->type != ROFFT_TEXT && n->tok == MDOC_MAX))
1112 return;
1113 int xr_found = 0;
1114
1115 if (n->type == ROFFT_TEXT) {
1116 mdoc_parse_section(n->sec, n->string, rec);
1117 } else if (mdocs[n->tok] == pmdoc_Xr) {
1118 /*
1119 * When encountering other inline macros,
1120 * call pmdoc_macro_handler.
1121 */
1122 pmdoc_macro_handler(n, rec, MDOC_Xr);
1123 xr_found = 1;
1124 } else if (mdocs[n->tok] == pmdoc_Pp) {
1125 pmdoc_macro_handler(n, rec, MDOC_Pp);
1126 }
1127
1128 /*
1129 * If an Xr macro was encountered then the child node has
1130 * already been explored by pmdoc_macro_handler.
1131 */
1132 if (xr_found == 0)
1133 mdoc_parse_Sh(n->child, rec);
1134 mdoc_parse_Sh(n->next, rec);
1135 }
1136
1137 /*
1138 * mdoc_parse_section--
1139 * Utility function for parsing sections of the mdoc type pages.
1140 * Takes two params:
1141 * 1. sec is an enum which indicates the section in which we are present
1142 * 2. string is the string which we need to append to the secbuff for this
1143 * particular section.
1144 * The function appends string to the global section buffer and returns.
1145 */
1146 static void
1147 mdoc_parse_section(enum roff_sec sec, const char *string, mandb_rec *rec)
1148 {
1149 /*
1150 * If the user specified the 'l' flag, then parse and store only the
1151 * NAME section. Ignore the rest.
1152 */
1153 if (mflags.limit)
1154 return;
1155
1156 switch (sec) {
1157 case SEC_LIBRARY:
1158 append(&rec->lib, string);
1159 break;
1160 case SEC_RETURN_VALUES:
1161 append(&rec->return_vals, string);
1162 break;
1163 case SEC_ENVIRONMENT:
1164 append(&rec->env, string);
1165 break;
1166 case SEC_FILES:
1167 append(&rec->files, string);
1168 break;
1169 case SEC_EXIT_STATUS:
1170 append(&rec->exit_status, string);
1171 break;
1172 case SEC_DIAGNOSTICS:
1173 append(&rec->diagnostics, string);
1174 break;
1175 case SEC_ERRORS:
1176 append(&rec->errors, string);
1177 break;
1178 default:
1179 append(&rec->desc, string);
1180 break;
1181 }
1182 }
1183
1184 static void
1185 proff_node(const struct roff_node *n, mandb_rec *rec, const proff_nf *func)
1186 {
1187 if (n == NULL)
1188 return;
1189
1190 switch (n->type) {
1191 case (ROFFT_BODY):
1192 /* FALLTHROUGH */
1193 case (ROFFT_BLOCK):
1194 /* FALLTHROUGH */
1195 case (ROFFT_ELEM):
1196 if (func[n->tok] != NULL)
1197 (*func[n->tok])(n, rec);
1198 break;
1199 default:
1200 break;
1201 }
1202
1203 proff_node(n->child, rec, func);
1204 proff_node(n->next, rec, func);
1205 }
1206
1207 /*
1208 * pman_parse_name --
1209 * Parses the NAME section and puts the complete content in the name_desc
1210 * variable.
1211 */
1212 static void
1213 pman_parse_name(const struct roff_node *n, mandb_rec *rec)
1214 {
1215 if (n == NULL)
1216 return;
1217
1218 if (n->type == ROFFT_TEXT) {
1219 char *tmp = parse_escape(n->string);
1220 concat(&rec->name_desc, tmp);
1221 free(tmp);
1222 }
1223
1224 if (n->child)
1225 pman_parse_name(n->child, rec);
1226
1227 if(n->next)
1228 pman_parse_name(n->next, rec);
1229 }
1230
1231 /*
1232 * A stub function to be able to parse the macros like .B embedded inside
1233 * a section.
1234 */
1235 static void
1236 pman_block(const struct roff_node *n, mandb_rec *rec)
1237 {
1238 }
1239
1240 /*
1241 * pman_sh --
1242 * This function does one of the two things:
1243 * 1. If the present section is NAME, then it will:
1244 * (a) Extract the name of the page (in case of multiple comma separated
1245 * names, it will pick up the first one).
1246 * (b) Build a space spearated list of all the symlinks/hardlinks to
1247 * this page and store in the buffer 'links'. These are extracted from
1248 * the comma separated list of names in the NAME section as well.
1249 * (c) Move on to the one line description section, which is after the list
1250 * of names in the NAME section.
1251 * 2. Otherwise, it will check the section name and call the man_parse_section
1252 * function, passing the enum corresponding to that section.
1253 */
1254 static void
1255 pman_sh(const struct roff_node *n, mandb_rec *rec)
1256 {
1257 static const struct {
1258 enum man_sec section;
1259 const char *header;
1260 } mapping[] = {
1261 { MANSEC_DESCRIPTION, "DESCRIPTION" },
1262 { MANSEC_SYNOPSIS, "SYNOPSIS" },
1263 { MANSEC_LIBRARY, "LIBRARY" },
1264 { MANSEC_ERRORS, "ERRORS" },
1265 { MANSEC_FILES, "FILES" },
1266 { MANSEC_RETURN_VALUES, "RETURN VALUE" },
1267 { MANSEC_RETURN_VALUES, "RETURN VALUES" },
1268 { MANSEC_EXIT_STATUS, "EXIT STATUS" },
1269 { MANSEC_EXAMPLES, "EXAMPLES" },
1270 { MANSEC_EXAMPLES, "EXAMPLE" },
1271 { MANSEC_STANDARDS, "STANDARDS" },
1272 { MANSEC_HISTORY, "HISTORY" },
1273 { MANSEC_BUGS, "BUGS" },
1274 { MANSEC_AUTHORS, "AUTHORS" },
1275 { MANSEC_COPYRIGHT, "COPYRIGHT" },
1276 };
1277 const struct roff_node *head;
1278 char *name_desc;
1279 size_t sz;
1280 size_t i;
1281
1282 if ((head = n->parent->head) == NULL || (head = head->child) == NULL ||
1283 head->type != ROFFT_TEXT)
1284 return;
1285
1286 /*
1287 * Check if this section should be extracted and
1288 * where it should be stored. Handled the trival cases first.
1289 */
1290 for (i = 0; i < sizeof(mapping) / sizeof(mapping[0]); ++i) {
1291 if (strcmp(head->string, mapping[i].header) == 0) {
1292 man_parse_section(mapping[i].section, n, rec);
1293 return;
1294 }
1295 }
1296
1297 if (strcmp(head->string, "NAME") == 0) {
1298 /*
1299 * We are in the NAME section.
1300 * pman_parse_name will put the complete content in name_desc.
1301 */
1302 pman_parse_name(n, rec);
1303
1304 name_desc = rec->name_desc;
1305 if (name_desc == NULL)
1306 return;
1307
1308 /* Remove any leading spaces. */
1309 while (name_desc[0] == ' ')
1310 name_desc++;
1311
1312 /* If the line begins with a "\&", avoid those */
1313 if (name_desc[0] == '\\' && name_desc[1] == '&')
1314 name_desc += 2;
1315
1316 /* Now name_desc should be left with a comma-space
1317 * separated list of names and the one line description
1318 * of the page:
1319 * "a, b, c \- sample description"
1320 * Take out the first name, before the first comma
1321 * (or space) and store it in rec->name.
1322 * If the page has aliases then they should be
1323 * in the form of a comma separated list.
1324 * Keep looping while there is a comma in name_desc,
1325 * extract the alias name and store in rec->links.
1326 * When there are no more commas left, break out.
1327 */
1328 int has_alias = 0; // Any more aliases left?
1329 while (*name_desc) {
1330 /* Remove any leading spaces or hyphens. */
1331 if (name_desc[0] == ' ' || name_desc[0] == '-') {
1332 name_desc++;
1333 continue;
1334 }
1335 sz = strcspn(name_desc, ", ");
1336
1337 /* Extract the first term and store it in rec->name. */
1338 if (rec->name == NULL) {
1339 if (name_desc[sz] == ',')
1340 has_alias = 1;
1341 rec->name = estrndup(name_desc, sz);
1342 /* XXX This would only happen with a poorly
1343 * written man page, maybe warn? */
1344 if (name_desc[sz] == '\0')
1345 break;
1346 name_desc += sz + 1;
1347 continue;
1348 }
1349
1350 /*
1351 * Once rec->name is set, rest of the names
1352 * are to be treated as links or aliases.
1353 */
1354 if (rec->name && has_alias) {
1355 if (name_desc[sz] != ',') {
1356 /* No more commas left --> no more
1357 * aliases to take out */
1358 has_alias = 0;
1359 }
1360 concat2(&rec->links, name_desc, sz);
1361 /* XXX This would only happen with a poorly
1362 * written man page, maybe warn? */
1363 if (name_desc[sz] == '\0')
1364 break;
1365 name_desc += sz + 1;
1366 continue;
1367 }
1368 break;
1369 }
1370
1371 /* Parse any escape sequences that might be there */
1372 char *temp = parse_escape(name_desc);
1373 free(rec->name_desc);
1374 rec->name_desc = temp;
1375 temp = parse_escape(rec->name);
1376 free(rec->name);
1377 rec->name = temp;
1378 return;
1379 }
1380
1381 /* The RETURN VALUE section might be specified in multiple ways */
1382 if (strcmp(head->string, "RETURN") == 0 &&
1383 head->next != NULL && head->next->type == ROFFT_TEXT &&
1384 (strcmp(head->next->string, "VALUE") == 0 ||
1385 strcmp(head->next->string, "VALUES") == 0)) {
1386 man_parse_section(MANSEC_RETURN_VALUES, n, rec);
1387 return;
1388 }
1389
1390 /*
1391 * EXIT STATUS section can also be specified all on one line or on two
1392 * separate lines.
1393 */
1394 if (strcmp(head->string, "EXIT") == 0 &&
1395 head->next != NULL && head->next->type == ROFFT_TEXT &&
1396 strcmp(head->next->string, "STATUS") == 0) {
1397 man_parse_section(MANSEC_EXIT_STATUS, n, rec);
1398 return;
1399 }
1400
1401 /* Store the rest of the content in desc. */
1402 man_parse_section(MANSEC_NONE, n, rec);
1403 }
1404
1405 /*
1406 * pman_parse_node --
1407 * Generic function to iterate through a node. Usually called from
1408 * man_parse_section to parse a particular section of the man page.
1409 */
1410 static void
1411 pman_parse_node(const struct roff_node *n, secbuff *s)
1412 {
1413 if (n == NULL)
1414 return;
1415
1416 if (n->type == ROFFT_TEXT)
1417 append(s, n->string);
1418
1419 pman_parse_node(n->child, s);
1420 pman_parse_node(n->next, s);
1421 }
1422
1423 /*
1424 * man_parse_section --
1425 * Takes two parameters:
1426 * sec: Tells which section we are present in
1427 * n: Is the present node of the AST.
1428 * Depending on the section, we call pman_parse_node to parse that section and
1429 * concatenate the content from that section into the buffer for that section.
1430 */
1431 static void
1432 man_parse_section(enum man_sec sec, const struct roff_node *n, mandb_rec *rec)
1433 {
1434 /*
1435 * If the user sepecified the 'l' flag then just parse
1436 * the NAME section, ignore the rest.
1437 */
1438 if (mflags.limit)
1439 return;
1440
1441 switch (sec) {
1442 case MANSEC_LIBRARY:
1443 pman_parse_node(n, &rec->lib);
1444 break;
1445 case MANSEC_RETURN_VALUES:
1446 pman_parse_node(n, &rec->return_vals);
1447 break;
1448 case MANSEC_ENVIRONMENT:
1449 pman_parse_node(n, &rec->env);
1450 break;
1451 case MANSEC_FILES:
1452 pman_parse_node(n, &rec->files);
1453 break;
1454 case MANSEC_EXIT_STATUS:
1455 pman_parse_node(n, &rec->exit_status);
1456 break;
1457 case MANSEC_DIAGNOSTICS:
1458 pman_parse_node(n, &rec->diagnostics);
1459 break;
1460 case MANSEC_ERRORS:
1461 pman_parse_node(n, &rec->errors);
1462 break;
1463 case MANSEC_NAME:
1464 case MANSEC_SYNOPSIS:
1465 case MANSEC_EXAMPLES:
1466 case MANSEC_STANDARDS:
1467 case MANSEC_HISTORY:
1468 case MANSEC_BUGS:
1469 case MANSEC_AUTHORS:
1470 case MANSEC_COPYRIGHT:
1471 break;
1472 default:
1473 pman_parse_node(n, &rec->desc);
1474 break;
1475 }
1476
1477 }
1478
1479 /*
1480 * insert_into_db --
1481 * Inserts the parsed data of the man page in the Sqlite databse.
1482 * If any of the values is NULL, then we cleanup and return -1 indicating
1483 * an error.
1484 * Otherwise, store the data in the database and return 0.
1485 */
1486 static int
1487 insert_into_db(sqlite3 *db, mandb_rec *rec)
1488 {
1489 int rc = 0;
1490 int idx = -1;
1491 const char *sqlstr = NULL;
1492 sqlite3_stmt *stmt = NULL;
1493 char *ln = NULL;
1494 char *errmsg = NULL;
1495 long int mandb_rowid;
1496
1497 /*
1498 * At the very minimum we want to make sure that we store
1499 * the following data:
1500 * Name, one line description, and the MD5 hash
1501 */
1502 if (rec->name == NULL || rec->name_desc == NULL ||
1503 rec->md5_hash == NULL) {
1504 cleanup(rec);
1505 return -1;
1506 }
1507
1508 /* Write null byte at the end of all the sec_buffs */
1509 rec->desc.data[rec->desc.offset] = 0;
1510 rec->lib.data[rec->lib.offset] = 0;
1511 rec->env.data[rec->env.offset] = 0;
1512 rec->return_vals.data[rec->return_vals.offset] = 0;
1513 rec->exit_status.data[rec->exit_status.offset] = 0;
1514 rec->files.data[rec->files.offset] = 0;
1515 rec->diagnostics.data[rec->diagnostics.offset] = 0;
1516 rec->errors.data[rec->errors.offset] = 0;
1517
1518 /*
1519 * In case of a mdoc page: (sorry, no better place to put this code)
1520 * parse the comma separated list of names of man pages,
1521 * the first name will be stored in the mandb table, rest will be
1522 * treated as links and put in the mandb_links table.
1523 */
1524 if (rec->page_type == MDOC) {
1525 char *tmp;
1526 rec->links = estrdup(rec->name);
1527 free(rec->name);
1528 size_t sz = strcspn(rec->links, " \0");
1529 rec->name = emalloc(sz + 1);
1530 memcpy(rec->name, rec->links, sz);
1531 if(rec->name[sz - 1] == ',')
1532 rec->name[sz - 1] = 0;
1533 else
1534 rec->name[sz] = 0;
1535 while (rec->links[sz] == ' ')
1536 ++sz;
1537 tmp = estrdup(rec->links + sz);
1538 free(rec->links);
1539 rec->links = tmp;
1540 }
1541
1542 /*------------------------ Populate the mandb table---------------------------*/
1543 sqlstr = "INSERT INTO mandb VALUES (:section, :name, :name_desc, :desc,"
1544 " :lib, :return_vals, :env, :files, :exit_status,"
1545 " :diagnostics, :errors, :md5_hash, :machine)";
1546
1547 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1548 if (rc != SQLITE_OK)
1549 goto Out;
1550
1551 idx = sqlite3_bind_parameter_index(stmt, ":name");
1552 rc = sqlite3_bind_text(stmt, idx, rec->name, -1, NULL);
1553 if (rc != SQLITE_OK) {
1554 sqlite3_finalize(stmt);
1555 goto Out;
1556 }
1557
1558 idx = sqlite3_bind_parameter_index(stmt, ":section");
1559 rc = sqlite3_bind_text(stmt, idx, rec->section, -1, NULL);
1560 if (rc != SQLITE_OK) {
1561 sqlite3_finalize(stmt);
1562 goto Out;
1563 }
1564
1565 idx = sqlite3_bind_parameter_index(stmt, ":name_desc");
1566 rc = sqlite3_bind_text(stmt, idx, rec->name_desc, -1, NULL);
1567 if (rc != SQLITE_OK) {
1568 sqlite3_finalize(stmt);
1569 goto Out;
1570 }
1571
1572 idx = sqlite3_bind_parameter_index(stmt, ":desc");
1573 rc = sqlite3_bind_text(stmt, idx, rec->desc.data,
1574 rec->desc.offset + 1, NULL);
1575 if (rc != SQLITE_OK) {
1576 sqlite3_finalize(stmt);
1577 goto Out;
1578 }
1579
1580 idx = sqlite3_bind_parameter_index(stmt, ":lib");
1581 rc = sqlite3_bind_text(stmt, idx, rec->lib.data,
1582 rec->lib.offset + 1, NULL);
1583 if (rc != SQLITE_OK) {
1584 sqlite3_finalize(stmt);
1585 goto Out;
1586 }
1587
1588 idx = sqlite3_bind_parameter_index(stmt, ":return_vals");
1589 rc = sqlite3_bind_text(stmt, idx, rec->return_vals.data,
1590 rec->return_vals.offset + 1, NULL);
1591 if (rc != SQLITE_OK) {
1592 sqlite3_finalize(stmt);
1593 goto Out;
1594 }
1595
1596 idx = sqlite3_bind_parameter_index(stmt, ":env");
1597 rc = sqlite3_bind_text(stmt, idx, rec->env.data,
1598 rec->env.offset + 1, NULL);
1599 if (rc != SQLITE_OK) {
1600 sqlite3_finalize(stmt);
1601 goto Out;
1602 }
1603
1604 idx = sqlite3_bind_parameter_index(stmt, ":files");
1605 rc = sqlite3_bind_text(stmt, idx, rec->files.data,
1606 rec->files.offset + 1, NULL);
1607 if (rc != SQLITE_OK) {
1608 sqlite3_finalize(stmt);
1609 goto Out;
1610 }
1611
1612 idx = sqlite3_bind_parameter_index(stmt, ":exit_status");
1613 rc = sqlite3_bind_text(stmt, idx, rec->exit_status.data,
1614 rec->exit_status.offset + 1, NULL);
1615 if (rc != SQLITE_OK) {
1616 sqlite3_finalize(stmt);
1617 goto Out;
1618 }
1619
1620 idx = sqlite3_bind_parameter_index(stmt, ":diagnostics");
1621 rc = sqlite3_bind_text(stmt, idx, rec->diagnostics.data,
1622 rec->diagnostics.offset + 1, NULL);
1623 if (rc != SQLITE_OK) {
1624 sqlite3_finalize(stmt);
1625 goto Out;
1626 }
1627
1628 idx = sqlite3_bind_parameter_index(stmt, ":errors");
1629 rc = sqlite3_bind_text(stmt, idx, rec->errors.data,
1630 rec->errors.offset + 1, NULL);
1631 if (rc != SQLITE_OK) {
1632 sqlite3_finalize(stmt);
1633 goto Out;
1634 }
1635
1636 idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1637 rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1638 if (rc != SQLITE_OK) {
1639 sqlite3_finalize(stmt);
1640 goto Out;
1641 }
1642
1643 idx = sqlite3_bind_parameter_index(stmt, ":machine");
1644 if (rec->machine)
1645 rc = sqlite3_bind_text(stmt, idx, rec->machine, -1, NULL);
1646 else
1647 rc = sqlite3_bind_null(stmt, idx);
1648 if (rc != SQLITE_OK) {
1649 sqlite3_finalize(stmt);
1650 goto Out;
1651 }
1652
1653 rc = sqlite3_step(stmt);
1654 if (rc != SQLITE_DONE) {
1655 sqlite3_finalize(stmt);
1656 goto Out;
1657 }
1658
1659 sqlite3_finalize(stmt);
1660
1661 /* Get the row id of the last inserted row */
1662 mandb_rowid = sqlite3_last_insert_rowid(db);
1663
1664 /*------------------------Populate the mandb_meta table-----------------------*/
1665 sqlstr = "INSERT INTO mandb_meta VALUES (:device, :inode, :mtime,"
1666 " :file, :md5_hash, :id)";
1667 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1668 if (rc != SQLITE_OK)
1669 goto Out;
1670
1671 idx = sqlite3_bind_parameter_index(stmt, ":device");
1672 rc = sqlite3_bind_int64(stmt, idx, rec->device);
1673 if (rc != SQLITE_OK) {
1674 sqlite3_finalize(stmt);
1675 goto Out;
1676 }
1677
1678 idx = sqlite3_bind_parameter_index(stmt, ":inode");
1679 rc = sqlite3_bind_int64(stmt, idx, rec->inode);
1680 if (rc != SQLITE_OK) {
1681 sqlite3_finalize(stmt);
1682 goto Out;
1683 }
1684
1685 idx = sqlite3_bind_parameter_index(stmt, ":mtime");
1686 rc = sqlite3_bind_int64(stmt, idx, rec->mtime);
1687 if (rc != SQLITE_OK) {
1688 sqlite3_finalize(stmt);
1689 goto Out;
1690 }
1691
1692 idx = sqlite3_bind_parameter_index(stmt, ":file");
1693 rc = sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
1694 if (rc != SQLITE_OK) {
1695 sqlite3_finalize(stmt);
1696 goto Out;
1697 }
1698
1699 idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1700 rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1701 if (rc != SQLITE_OK) {
1702 sqlite3_finalize(stmt);
1703 goto Out;
1704 }
1705
1706 idx = sqlite3_bind_parameter_index(stmt, ":id");
1707 rc = sqlite3_bind_int64(stmt, idx, mandb_rowid);
1708 if (rc != SQLITE_OK) {
1709 sqlite3_finalize(stmt);
1710 goto Out;
1711 }
1712
1713 rc = sqlite3_step(stmt);
1714 sqlite3_finalize(stmt);
1715 if (rc == SQLITE_CONSTRAINT_UNIQUE) {
1716 /* The *most* probable reason for reaching here is that
1717 * the UNIQUE contraint on the file column of the mandb_meta
1718 * table was violated.
1719 * This can happen when a file was updated/modified.
1720 * To fix this we need to do two things:
1721 * 1. Delete the row for the older version of this file
1722 * from mandb table.
1723 * 2. Run an UPDATE query to update the row for this file
1724 * in the mandb_meta table.
1725 */
1726 warnx("Trying to update index for %s", rec->file_path);
1727 char *sql = sqlite3_mprintf("DELETE FROM mandb "
1728 "WHERE rowid = (SELECT id"
1729 " FROM mandb_meta"
1730 " WHERE file = %Q)",
1731 rec->file_path);
1732 sqlite3_exec(db, sql, NULL, NULL, &errmsg);
1733 sqlite3_free(sql);
1734 if (errmsg != NULL) {
1735 if (mflags.verbosity)
1736 warnx("%s", errmsg);
1737 free(errmsg);
1738 }
1739 sqlstr = "UPDATE mandb_meta SET device = :device,"
1740 " inode = :inode, mtime = :mtime, id = :id,"
1741 " md5_hash = :md5 WHERE file = :file";
1742 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1743 if (rc != SQLITE_OK) {
1744 if (mflags.verbosity)
1745 warnx("Update failed with error: %s",
1746 sqlite3_errmsg(db));
1747 close_db(db);
1748 cleanup(rec);
1749 errx(EXIT_FAILURE,
1750 "Consider running makemandb with -f option");
1751 }
1752
1753 idx = sqlite3_bind_parameter_index(stmt, ":device");
1754 sqlite3_bind_int64(stmt, idx, rec->device);
1755 idx = sqlite3_bind_parameter_index(stmt, ":inode");
1756 sqlite3_bind_int64(stmt, idx, rec->inode);
1757 idx = sqlite3_bind_parameter_index(stmt, ":mtime");
1758 sqlite3_bind_int64(stmt, idx, rec->mtime);
1759 idx = sqlite3_bind_parameter_index(stmt, ":id");
1760 sqlite3_bind_int64(stmt, idx, mandb_rowid);
1761 idx = sqlite3_bind_parameter_index(stmt, ":md5");
1762 sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
1763 idx = sqlite3_bind_parameter_index(stmt, ":file");
1764 sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
1765 rc = sqlite3_step(stmt);
1766 sqlite3_finalize(stmt);
1767
1768 if (rc != SQLITE_DONE) {
1769 if (mflags.verbosity)
1770 warnx("%s", sqlite3_errmsg(db));
1771 close_db(db);
1772 cleanup(rec);
1773 errx(EXIT_FAILURE,
1774 "Consider running makemandb with -f option");
1775 }
1776 } else if (rc != SQLITE_DONE) {
1777 /* Otherwise make this error fatal */
1778 warnx("Failed at %s\n%s", rec->file_path, sqlite3_errmsg(db));
1779 cleanup(rec);
1780 close_db(db);
1781 exit(EXIT_FAILURE);
1782 }
1783
1784 /*------------------------ Populate the mandb_links table---------------------*/
1785 char *str = NULL;
1786 char *links;
1787 if (rec->links && strlen(rec->links)) {
1788 links = rec->links;
1789 for(ln = strtok(links, " "); ln; ln = strtok(NULL, " ")) {
1790 if (ln[0] == ',')
1791 ln++;
1792 if(ln[strlen(ln) - 1] == ',')
1793 ln[strlen(ln) - 1] = 0;
1794
1795 str = sqlite3_mprintf("INSERT INTO mandb_links"
1796 " VALUES (%Q, %Q, %Q, %Q, %Q)",
1797 ln, rec->name, rec->section,
1798 rec->machine, rec->md5_hash);
1799 sqlite3_exec(db, str, NULL, NULL, &errmsg);
1800 sqlite3_free(str);
1801 if (errmsg != NULL) {
1802 warnx("%s", errmsg);
1803 cleanup(rec);
1804 free(errmsg);
1805 return -1;
1806 }
1807 }
1808 }
1809
1810 cleanup(rec);
1811 return 0;
1812
1813 Out:
1814 if (mflags.verbosity)
1815 warnx("%s", sqlite3_errmsg(db));
1816 cleanup(rec);
1817 return -1;
1818 }
1819
1820 /*
1821 * check_md5--
1822 * Generates the md5 hash of the file and checks if it already doesn't exist
1823 * in the table.
1824 * This function is being used to avoid hardlinks.
1825 * On successful completion it will also set the value of the fourth parameter
1826 * to the md5 hash of the file (computed previously). It is the responsibility
1827 * of the caller to free this buffer.
1828 * Return values:
1829 * -1: If an error occurs somewhere and sets the md5 return buffer to NULL.
1830 * 0: If the md5 hash does not exist in the table.
1831 * 1: If the hash exists in the database.
1832 */
1833 static int
1834 check_md5(const char *file, sqlite3 *db, char **md5, void *buf, size_t buflen)
1835 {
1836 int rc = 0;
1837 int idx = -1;
1838 char *sqlstr = NULL;
1839 char *mymd5;
1840 sqlite3_stmt *stmt = NULL;
1841 *md5 = NULL;
1842
1843 assert(file != NULL);
1844 if ((mymd5 = MD5Data(buf, buflen, NULL)) == NULL) {
1845 if (mflags.verbosity)
1846 warn("md5 failed: %s", file);
1847 return -1;
1848 }
1849
1850 easprintf(&sqlstr, "SELECT * FROM mandb_meta WHERE md5_hash = :md5_hash");
1851 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
1852 if (rc != SQLITE_OK) {
1853 free(sqlstr);
1854 free(mymd5);
1855 return -1;
1856 }
1857
1858 idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
1859 rc = sqlite3_bind_text(stmt, idx, mymd5, -1, NULL);
1860 if (rc != SQLITE_OK) {
1861 if (mflags.verbosity)
1862 warnx("%s", sqlite3_errmsg(db));
1863 sqlite3_finalize(stmt);
1864 free(sqlstr);
1865 free(mymd5);
1866 return -1;
1867 }
1868
1869 *md5 = mymd5;
1870 if (sqlite3_step(stmt) == SQLITE_ROW) {
1871 sqlite3_finalize(stmt);
1872 free(sqlstr);
1873 return 0;
1874 }
1875
1876 sqlite3_finalize(stmt);
1877 free(sqlstr);
1878 return 1;
1879 }
1880
1881 /* Optimize the index for faster search */
1882 static void
1883 optimize(sqlite3 *db)
1884 {
1885 const char *sqlstr;
1886 char *errmsg = NULL;
1887
1888 if (mflags.verbosity == 2)
1889 printf("Optimizing the database index\n");
1890 sqlstr = "INSERT INTO mandb(mandb) VALUES (\'optimize\');"
1891 "VACUUM";
1892 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
1893 if (errmsg != NULL) {
1894 if (mflags.verbosity)
1895 warnx("%s", errmsg);
1896 free(errmsg);
1897 return;
1898 }
1899 }
1900
1901 /*
1902 * cleanup --
1903 * cleans up the global buffers
1904 */
1905 static void
1906 cleanup(mandb_rec *rec)
1907 {
1908 rec->desc.offset = 0;
1909 rec->lib.offset = 0;
1910 rec->return_vals.offset = 0;
1911 rec->env.offset = 0;
1912 rec->exit_status.offset = 0;
1913 rec->diagnostics.offset = 0;
1914 rec->errors.offset = 0;
1915 rec->files.offset = 0;
1916
1917 free(rec->machine);
1918 rec->machine = NULL;
1919
1920 free(rec->links);
1921 rec->links = NULL;
1922
1923 free(rec->file_path);
1924 rec->file_path = NULL;
1925
1926 free(rec->name);
1927 rec->name = NULL;
1928
1929 free(rec->name_desc);
1930 rec->name_desc = NULL;
1931
1932 free(rec->md5_hash);
1933 rec->md5_hash = NULL;
1934
1935 free(rec->section);
1936 rec->section = NULL;
1937 }
1938
1939 /*
1940 * init_secbuffs--
1941 * Sets the value of buflen for all the sec_buff field of rec. And then
1942 * allocate memory to each sec_buff member of rec.
1943 */
1944 static void
1945 init_secbuffs(mandb_rec *rec)
1946 {
1947 /*
1948 * Some sec_buff might need more memory, for example desc,
1949 * which stores the data of the DESCRIPTION section,
1950 * while some might need very small amount of memory.
1951 * Therefore explicitly setting the value of buflen field for
1952 * each sec_buff.
1953 */
1954 rec->desc.buflen = 10 * BUFLEN;
1955 rec->desc.data = emalloc(rec->desc.buflen);
1956 rec->desc.offset = 0;
1957
1958 rec->lib.buflen = BUFLEN / 2;
1959 rec->lib.data = emalloc(rec->lib.buflen);
1960 rec->lib.offset = 0;
1961
1962 rec->return_vals.buflen = BUFLEN;
1963 rec->return_vals.data = emalloc(rec->return_vals.buflen);
1964 rec->return_vals.offset = 0;
1965
1966 rec->exit_status.buflen = BUFLEN;
1967 rec->exit_status.data = emalloc(rec->exit_status.buflen);
1968 rec->exit_status.offset = 0;
1969
1970 rec->env.buflen = BUFLEN;
1971 rec->env.data = emalloc(rec->env.buflen);
1972 rec->env.offset = 0;
1973
1974 rec->files.buflen = BUFLEN;
1975 rec->files.data = emalloc(rec->files.buflen);
1976 rec->files.offset = 0;
1977
1978 rec->diagnostics.buflen = BUFLEN;
1979 rec->diagnostics.data = emalloc(rec->diagnostics.buflen);
1980 rec->diagnostics.offset = 0;
1981
1982 rec->errors.buflen = BUFLEN;
1983 rec->errors.data = emalloc(rec->errors.buflen);
1984 rec->errors.offset = 0;
1985 }
1986
1987 /*
1988 * free_secbuffs--
1989 * This function should be called at the end, when all the pages have been
1990 * parsed.
1991 * It frees the memory allocated to sec_buffs by init_secbuffs in the starting.
1992 */
1993 static void
1994 free_secbuffs(mandb_rec *rec)
1995 {
1996 free(rec->desc.data);
1997 free(rec->lib.data);
1998 free(rec->return_vals.data);
1999 free(rec->exit_status.data);
2000 free(rec->env.data);
2001 free(rec->files.data);
2002 free(rec->diagnostics.data);
2003 free(rec->errors.data);
2004 }
2005
2006 static void
2007 replace_hyph(char *str)
2008 {
2009 char *iter = str;
2010 while ((iter = strchr(iter, ASCII_HYPH)) != NULL)
2011 *iter = '-';
2012
2013 iter = str;
2014 while ((iter = strchr(iter, ASCII_NBRSP)) != NULL)
2015 *iter = '-';
2016 }
2017
2018 static char *
2019 parse_escape(const char *str)
2020 {
2021 const char *backslash, *last_backslash;
2022 char *result, *iter;
2023 size_t len;
2024
2025 assert(str);
2026
2027 last_backslash = str;
2028 backslash = strchr(str, '\\');
2029 if (backslash == NULL) {
2030 result = estrdup(str);
2031 replace_hyph(result);
2032 return result;
2033 }
2034
2035 result = emalloc(strlen(str) + 1);
2036 iter = result;
2037
2038 do {
2039 len = backslash - last_backslash;
2040 memcpy(iter, last_backslash, len);
2041 iter += len;
2042 if (backslash[1] == '-' || backslash[1] == ' ') {
2043 *iter++ = backslash[1];
2044 last_backslash = backslash + 2;
2045 backslash = strchr(last_backslash, '\\');
2046 } else {
2047 ++backslash;
2048 mandoc_escape(&backslash, NULL, NULL);
2049 last_backslash = backslash;
2050 if (backslash == NULL)
2051 break;
2052 backslash = strchr(last_backslash, '\\');
2053 }
2054 } while (backslash != NULL);
2055 if (last_backslash != NULL)
2056 strcpy(iter, last_backslash);
2057
2058 replace_hyph(result);
2059 return result;
2060 }
2061
2062 /*
2063 * append--
2064 * Concatenates a space and src at the end of sbuff->data (much like concat in
2065 * apropos-utils.c).
2066 * Rather than reallocating space for writing data, it uses the value of the
2067 * offset field of sec_buff to write new data at the free space left in the
2068 * buffer.
2069 * In case the size of the data to be appended exceeds the number of bytes left
2070 * in the buffer, it reallocates buflen number of bytes and then continues.
2071 * Value of offset field should be adjusted as new data is written.
2072 *
2073 * NOTE: This function does not write the null byte at the end of the buffers,
2074 * write a null byte at the position pointed to by offset before inserting data
2075 * in the db.
2076 */
2077 static void
2078 append(secbuff *sbuff, const char *src)
2079 {
2080 short flag = 0;
2081 size_t srclen, newlen;
2082 char *temp;
2083
2084 assert(src != NULL);
2085 temp = parse_escape(src);
2086 srclen = strlen(temp);
2087
2088 if (sbuff->data == NULL) {
2089 sbuff->data = emalloc(sbuff->buflen);
2090 sbuff->offset = 0;
2091 }
2092
2093 newlen = sbuff->offset + srclen + 2;
2094 if (newlen >= sbuff->buflen) {
2095 while (sbuff->buflen < newlen)
2096 sbuff->buflen += sbuff->buflen;
2097 sbuff->data = erealloc(sbuff->data, sbuff->buflen);
2098 flag = 1;
2099 }
2100
2101 /* Append a space at the end of the buffer. */
2102 if (sbuff->offset || flag)
2103 sbuff->data[sbuff->offset++] = ' ';
2104 /* Now, copy src at the end of the buffer. */
2105 memcpy(sbuff->data + sbuff->offset, temp, srclen);
2106 sbuff->offset += srclen;
2107 free(temp);
2108 }
2109
2110 static void
2111 usage(void)
2112 {
2113 fprintf(stderr, "Usage: %s [-floQqv] [-C path]\n", getprogname());
2114 exit(1);
2115 }
2116