Home | History | Annotate | Line # | Download | only in makemandb
makemandb.c revision 1.60
      1 /*	$NetBSD: makemandb.c,v 1.60 2019/05/18 07:56:43 abhinav Exp $	*/
      2 /*
      3  * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay (at) gmail.com>
      4  * Copyright (c) 2011 Kristaps Dzonsons <kristaps (at) bsd.lv>
      5  *
      6  * Permission to use, copy, modify, and distribute this software for any
      7  * purpose with or without fee is hereby granted, provided that the above
      8  * copyright notice and this permission notice appear in all copies.
      9  *
     10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
     11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
     12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
     13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
     14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
     15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
     16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
     17  */
     18 
     19 #include <sys/cdefs.h>
     20 __RCSID("$NetBSD: makemandb.c,v 1.60 2019/05/18 07:56:43 abhinav Exp $");
     21 
     22 #include <sys/stat.h>
     23 #include <sys/types.h>
     24 
     25 #include <assert.h>
     26 #include <dirent.h>
     27 #include <err.h>
     28 #include <archive.h>
     29 #include <libgen.h>
     30 #include <md5.h>
     31 #include <stdio.h>
     32 #include <stdlib.h>
     33 #include <string.h>
     34 #include <unistd.h>
     35 #include <util.h>
     36 
     37 #include "apropos-utils.h"
     38 #include "dist/man.h"
     39 #include "dist/mandoc.h"
     40 #include "dist/mdoc.h"
     41 #include "dist/roff.h"
     42 #include "dist/mandoc_parse.h"
     43 
     44 #define BUFLEN 1024
     45 #define MDOC 0	//If the page is of mdoc(7) type
     46 #define MAN 1	//If the page  is of man(7) type
     47 
     48 /*
     49  * A data structure for holding section specific data.
     50  */
     51 typedef struct secbuff {
     52 	char *data;
     53 	size_t buflen;	//Total length of buffer allocated initially
     54 	size_t offset;	// Current offset in the buffer.
     55 } secbuff;
     56 
     57 typedef struct makemandb_flags {
     58 	int optimize;
     59 	int limit;	// limit the indexing to only NAME section
     60 	int recreate;	// Database was created from scratch
     61 	int verbosity;	// 0: quiet, 1: default, 2: verbose
     62 } makemandb_flags;
     63 
     64 typedef struct roff_mandb_rec {
     65 	/* Fields for mandb table */
     66 	char *name;	// for storing the name of the man page
     67 	char *name_desc; // for storing the one line description (.Nd)
     68 	secbuff desc; // for storing the DESCRIPTION section
     69 	secbuff lib; // for the LIBRARY section
     70 	secbuff return_vals; // RETURN VALUES
     71 	secbuff env; // ENVIRONMENT
     72 	secbuff files; // FILES
     73 	secbuff exit_status; // EXIT STATUS
     74 	secbuff diagnostics; // DIAGNOSTICS
     75 	secbuff errors; // ERRORS
     76 	char *section;
     77 
     78 	int xr_found; // To track whether a .Xr was seen when parsing a section
     79 
     80 	/* Fields for mandb_meta table */
     81 	char *md5_hash;
     82 	dev_t device;
     83 	ino_t inode;
     84 	time_t mtime;
     85 
     86 	/* Fields for mandb_links table */
     87 	char *machine;
     88 	char *links; //all the links to a page in a space separated form
     89 	char *file_path;
     90 
     91 	/* Non-db fields */
     92 	int page_type; //Indicates the type of page: mdoc or man
     93 } mandb_rec;
     94 
     95 typedef	void (*proff_nf)(const struct roff_node *n, mandb_rec *);
     96 
     97 static void append(secbuff *sbuff, const char *src);
     98 static void init_secbuffs(mandb_rec *);
     99 static void free_secbuffs(mandb_rec *);
    100 static int check_md5(const char *, sqlite3 *, char **, void *, size_t);
    101 static void cleanup(mandb_rec *);
    102 static void set_section(const struct roff_meta *, mandb_rec *);
    103 static void set_machine(const struct roff_meta *, mandb_rec *);
    104 static int insert_into_db(sqlite3 *, mandb_rec *);
    105 static	void begin_parse(const char *, struct mparse *, mandb_rec *, int);
    106 static void proff_node(const struct roff_node *, mandb_rec *,
    107 		       struct roff_meta *, const proff_nf *);
    108 static void pmdoc_Nm(const struct roff_node *, mandb_rec *);
    109 static void pmdoc_Nd(const struct roff_node *, mandb_rec *);
    110 static void pmdoc_Sh(const struct roff_node *, mandb_rec *);
    111 static void mdoc_parse_Sh(const struct roff_node *, mandb_rec *);
    112 static void pmdoc_Xr(const struct roff_node *, mandb_rec *);
    113 static void pmdoc_Pp(const struct roff_node *, mandb_rec *);
    114 static void pmdoc_macro_handler(const struct roff_node *, mandb_rec *, int);
    115 static void pman_parse_node(const struct roff_node *, secbuff *);
    116 static void pman_parse_name(const struct roff_node *, mandb_rec *);
    117 static void pman_sh(const struct roff_node *, mandb_rec *);
    118 static void pman_block(const struct roff_node *, mandb_rec *);
    119 static void traversedir(const char *, const char *, sqlite3 *, struct mparse *);
    120 static void mdoc_parse_section(enum roff_sec, const char *, mandb_rec *);
    121 static void man_parse_section(enum man_sec, const struct roff_node *, mandb_rec *);
    122 static void build_file_cache(sqlite3 *, const char *, const char *,
    123 			     struct stat *);
    124 static void update_db(sqlite3 *, struct mparse *, mandb_rec *);
    125 __dead static void usage(void);
    126 static void optimize(sqlite3 *);
    127 static char *parse_escape(const char *);
    128 static void replace_hyph(char *);
    129 static makemandb_flags mflags = { .verbosity = 1 };
    130 
    131 static	const proff_nf mdocs[MDOC_MAX - MDOC_Dd] = {
    132 	NULL, /* Dd */
    133 	NULL, /* Dt */
    134 	NULL, /* Os */
    135 	pmdoc_Sh, /* Sh */
    136 	NULL, /* Ss */
    137 	pmdoc_Pp, /* Pp */
    138 	NULL, /* D1 */
    139 	NULL, /* Dl */
    140 	NULL, /* Bd */
    141 	NULL, /* Ed */
    142 	NULL, /* Bl */
    143 	NULL, /* El */
    144 	NULL, /* It */
    145 	NULL, /* Ad */
    146 	NULL, /* An */
    147 	NULL, /* Ap */
    148 	NULL, /* Ar */
    149 	NULL, /* Cd */
    150 	NULL, /* Cm */
    151 	NULL, /* Dv */
    152 	NULL, /* Er */
    153 	NULL, /* Ev */
    154 	NULL, /* Ex */
    155 	NULL, /* Fa */
    156 	NULL, /* Fd */
    157 	NULL, /* Fl */
    158 	NULL, /* Fn */
    159 	NULL, /* Ft */
    160 	NULL, /* Ic */
    161 	NULL, /* In */
    162 	NULL, /* Li */
    163 	pmdoc_Nd, /* Nd */
    164 	pmdoc_Nm, /* Nm */
    165 	NULL, /* Op */
    166 	NULL, /* Ot */
    167 	NULL, /* Pa */
    168 	NULL, /* Rv */
    169 	NULL, /* St */
    170 	NULL, /* Va */
    171 	NULL, /* Vt */
    172 	pmdoc_Xr, /* Xr */
    173 	NULL, /* %A */
    174 	NULL, /* %B */
    175 	NULL, /* %D */
    176 	NULL, /* %I */
    177 	NULL, /* %J */
    178 	NULL, /* %N */
    179 	NULL, /* %O */
    180 	NULL, /* %P */
    181 	NULL, /* %R */
    182 	NULL, /* %T */
    183 	NULL, /* %V */
    184 	NULL, /* Ac */
    185 	NULL, /* Ao */
    186 	NULL, /* Aq */
    187 	NULL, /* At */
    188 	NULL, /* Bc */
    189 	NULL, /* Bf */
    190 	NULL, /* Bo */
    191 	NULL, /* Bq */
    192 	NULL, /* Bsx */
    193 	NULL, /* Bx */
    194 	NULL, /* Db */
    195 	NULL, /* Dc */
    196 	NULL, /* Do */
    197 	NULL, /* Dq */
    198 	NULL, /* Ec */
    199 	NULL, /* Ef */
    200 	NULL, /* Em */
    201 	NULL, /* Eo */
    202 	NULL, /* Fx */
    203 	NULL, /* Ms */
    204 	NULL, /* No */
    205 	NULL, /* Ns */
    206 	NULL, /* Nx */
    207 	NULL, /* Ox */
    208 	NULL, /* Pc */
    209 	NULL, /* Pf */
    210 	NULL, /* Po */
    211 	NULL, /* Pq */
    212 	NULL, /* Qc */
    213 	NULL, /* Ql */
    214 	NULL, /* Qo */
    215 	NULL, /* Qq */
    216 	NULL, /* Re */
    217 	NULL, /* Rs */
    218 	NULL, /* Sc */
    219 	NULL, /* So */
    220 	NULL, /* Sq */
    221 	NULL, /* Sm */
    222 	NULL, /* Sx */
    223 	NULL, /* Sy */
    224 	NULL, /* Tn */
    225 	NULL, /* Ux */
    226 	NULL, /* Xc */
    227 	NULL, /* Xo */
    228 	NULL, /* Fo */
    229 	NULL, /* Fc */
    230 	NULL, /* Oo */
    231 	NULL, /* Oc */
    232 	NULL, /* Bk */
    233 	NULL, /* Ek */
    234 	NULL, /* Bt */
    235 	NULL, /* Hf */
    236 	NULL, /* Fr */
    237 	NULL, /* Ud */
    238 	NULL, /* Lb */
    239 	NULL, /* Lp */
    240 	NULL, /* Lk */
    241 	NULL, /* Mt */
    242 	NULL, /* Brq */
    243 	NULL, /* Bro */
    244 	NULL, /* Brc */
    245 	NULL, /* %C */
    246 	NULL, /* Es */
    247 	NULL, /* En */
    248 	NULL, /* Dx */
    249 	NULL, /* %Q */
    250 	NULL, /* %U */
    251 	NULL /* Ta */
    252 };
    253 
    254 static	const proff_nf mans[MAN_MAX - MAN_TH] = {
    255 	NULL,	//TH
    256 	pman_sh, //SH
    257 	NULL,	//SS
    258 	NULL,	//TP
    259 	NULL,	//LP
    260 	NULL,	//PP
    261 	NULL,	//P
    262 	NULL,	//IP
    263 	NULL,	//HP
    264 	NULL,	//SM
    265 	NULL,	//SB
    266 	NULL,	//BI
    267 	NULL,	//IB
    268 	NULL,	//BR
    269 	NULL,	//RB
    270 	NULL,	//R
    271 	pman_block,	//B
    272 	NULL,	//I
    273 	NULL,	//IR
    274 	NULL,	//RI
    275 	NULL,	//nf
    276 	NULL,	//fi
    277 	NULL,	//RE
    278 	NULL,	//RS
    279 	NULL,	//DT
    280 	NULL,	//UC
    281 	NULL,	//PD
    282 	NULL,	//AT
    283 	NULL,	//in
    284 	NULL,	//OP
    285 	NULL,	//EX
    286 	NULL,	//EE
    287 	NULL,	//UR
    288 	NULL,	//UE
    289 	NULL,	//MT
    290 	NULL   //ME
    291 };
    292 
    293 int
    294 main(int argc, char *argv[])
    295 {
    296 	FILE *file;
    297 	const char *sqlstr, *manconf = NULL;
    298 	char *line, *command;
    299 	char *errmsg;
    300 	int ch;
    301 	struct mparse *mp;
    302 	sqlite3 *db;
    303 	ssize_t len;
    304 	size_t linesize;
    305 	struct roff_mandb_rec rec;
    306 
    307 	while ((ch = getopt(argc, argv, "C:floQqv")) != -1) {
    308 		switch (ch) {
    309 		case 'C':
    310 			manconf = optarg;
    311 			break;
    312 		case 'f':
    313 			mflags.recreate = 1;
    314 			break;
    315 		case 'l':
    316 			mflags.limit = 1;
    317 			break;
    318 		case 'o':
    319 			mflags.optimize = 1;
    320 			break;
    321 		case 'Q':
    322 			mflags.verbosity = 0;
    323 			break;
    324 		case 'q':
    325 			mflags.verbosity = 1;
    326 			break;
    327 		case 'v':
    328 			mflags.verbosity = 2;
    329 			break;
    330 		default:
    331 			usage();
    332 		}
    333 	}
    334 
    335 	memset(&rec, 0, sizeof(rec));
    336 
    337 	init_secbuffs(&rec);
    338 	mchars_alloc();
    339 	mp = mparse_alloc(MPARSE_SO | MPARSE_UTF8 | MPARSE_LATIN1 |
    340 	    MPARSE_VALIDATE, MANDOC_OS_OTHER, NULL);
    341 
    342 	if (manconf) {
    343 		char *arg;
    344 		size_t command_len = shquote(manconf, NULL, 0) + 1;
    345 		arg = emalloc(command_len);
    346 		shquote(manconf, arg, command_len);
    347 		easprintf(&command, "man -p -C %s", arg);
    348 		free(arg);
    349 	} else {
    350 		command = estrdup("man -p");
    351 		manconf = MANCONF;
    352 	}
    353 
    354 	if (mflags.recreate) {
    355 		char *dbp = get_dbpath(manconf);
    356 		/* No error here, it will fail in init_db in the same call */
    357 		if (dbp != NULL)
    358 			remove(dbp);
    359 	}
    360 
    361 	if ((db = init_db(MANDB_CREATE, manconf)) == NULL)
    362 		exit(EXIT_FAILURE);
    363 
    364 	sqlite3_exec(db, "PRAGMA synchronous = 0", NULL, NULL, 	&errmsg);
    365 	if (errmsg != NULL) {
    366 		warnx("%s", errmsg);
    367 		free(errmsg);
    368 		close_db(db);
    369 		exit(EXIT_FAILURE);
    370 	}
    371 
    372 	sqlite3_exec(db, "ATTACH DATABASE \':memory:\' AS metadb", NULL, NULL,
    373 	    &errmsg);
    374 	if (errmsg != NULL) {
    375 		warnx("%s", errmsg);
    376 		free(errmsg);
    377 		close_db(db);
    378 		exit(EXIT_FAILURE);
    379 	}
    380 
    381 
    382 	/* Call man -p to get the list of man page dirs */
    383 	if ((file = popen(command, "r")) == NULL) {
    384 		close_db(db);
    385 		err(EXIT_FAILURE, "fopen failed");
    386 	}
    387 	free(command);
    388 
    389 	/* Begin the transaction for indexing the pages	*/
    390 	sqlite3_exec(db, "BEGIN", NULL, NULL, &errmsg);
    391 	if (errmsg != NULL) {
    392 		warnx("%s", errmsg);
    393 		free(errmsg);
    394 		close_db(db);
    395 		exit(EXIT_FAILURE);
    396 	}
    397 
    398 	sqlstr = "CREATE TABLE metadb.file_cache(device, inode, mtime, parent,"
    399 		 " file PRIMARY KEY);"
    400 		 "CREATE UNIQUE INDEX metadb.index_file_cache_dev"
    401 		 " ON file_cache (device, inode)";
    402 
    403 	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
    404 	if (errmsg != NULL) {
    405 		warnx("%s", errmsg);
    406 		free(errmsg);
    407 		close_db(db);
    408 		exit(EXIT_FAILURE);
    409 	}
    410 
    411 	if (mflags.verbosity)
    412 		printf("Building temporary file cache\n");
    413 	line = NULL;
    414 	linesize = 0;
    415 	while ((len = getline(&line, &linesize, file)) != -1) {
    416 		/* Replace the new line character at the end of string with '\0' */
    417 		line[len - 1] = '\0';
    418 		char *pdir = estrdup(dirname(line));
    419 		/* Traverse the man page directories and parse the pages */
    420 		traversedir(pdir, line, db, mp);
    421 		free(pdir);
    422 	}
    423 	free(line);
    424 
    425 	if (pclose(file) == -1) {
    426 		close_db(db);
    427 		cleanup(&rec);
    428 		free_secbuffs(&rec);
    429 		err(EXIT_FAILURE, "pclose error");
    430 	}
    431 
    432 	if (mflags.verbosity)
    433 		printf("Performing index update\n");
    434 	update_db(db, mp, &rec);
    435 	mparse_free(mp);
    436 	mchars_free();
    437 	free_secbuffs(&rec);
    438 
    439 	/* Commit the transaction */
    440 	sqlite3_exec(db, "COMMIT", NULL, NULL, &errmsg);
    441 	if (errmsg != NULL) {
    442 		warnx("%s", errmsg);
    443 		free(errmsg);
    444 		close_db(db);
    445 		exit(EXIT_FAILURE);
    446 	}
    447 
    448 	if (mflags.optimize)
    449 		optimize(db);
    450 
    451 	close_db(db);
    452 	return 0;
    453 }
    454 
    455 /*
    456  * traversedir --
    457  *  Traverses the given directory recursively and passes all the man page files
    458  *  in the way to build_file_cache()
    459  */
    460 static void
    461 traversedir(const char *parent, const char *file, sqlite3 *db,
    462             struct mparse *mp)
    463 {
    464 	struct stat sb;
    465 	struct dirent *dirp;
    466 	DIR *dp;
    467 	char *buf;
    468 
    469 	if (stat(file, &sb) < 0) {
    470 		if (mflags.verbosity)
    471 			warn("stat failed: %s", file);
    472 		return;
    473 	}
    474 
    475 	/* If it is a directory, traverse it recursively */
    476 	if (S_ISDIR(sb.st_mode)) {
    477 		if ((dp = opendir(file)) == NULL) {
    478 			if (mflags.verbosity)
    479 				warn("opendir error: %s", file);
    480 			return;
    481 		}
    482 
    483 		while ((dirp = readdir(dp)) != NULL) {
    484 			/* Avoid . and .. entries in a directory */
    485 			if (dirp->d_name[0] != '.') {
    486 				easprintf(&buf, "%s/%s", file, dirp->d_name);
    487 				traversedir(parent, buf, db, mp);
    488 				free(buf);
    489 			}
    490 		}
    491 		closedir(dp);
    492 		return;
    493 	}
    494 
    495 	if (!S_ISREG(sb.st_mode))
    496 		return;
    497 
    498 	if (sb.st_size == 0) {
    499 		if (mflags.verbosity)
    500 			warnx("Empty file: %s", file);
    501 		return;
    502 	}
    503 	build_file_cache(db, parent, file, &sb);
    504 }
    505 
    506 /* build_file_cache --
    507  *   This function generates an md5 hash of the file passed as its 2nd parameter
    508  *   and stores it in a temporary table file_cache along with the full file path.
    509  *   This is done to support incremental updation of the database.
    510  *   The temporary table file_cache is dropped thereafter in the function
    511  *   update_db(), once the database has been updated.
    512  */
    513 static void
    514 build_file_cache(sqlite3 *db, const char *parent, const char *file,
    515 		 struct stat *sb)
    516 {
    517 	const char *sqlstr;
    518 	sqlite3_stmt *stmt = NULL;
    519 	int rc, idx;
    520 	assert(file != NULL);
    521 	dev_t device_cache = sb->st_dev;
    522 	ino_t inode_cache = sb->st_ino;
    523 	time_t mtime_cache = sb->st_mtime;
    524 
    525 	sqlstr = "INSERT INTO metadb.file_cache VALUES (:device, :inode,"
    526 		 " :mtime, :parent, :file)";
    527 	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
    528 	if (rc != SQLITE_OK) {
    529 		if (mflags.verbosity)
    530 			warnx("%s", sqlite3_errmsg(db));
    531 		return;
    532 	}
    533 
    534 	idx = sqlite3_bind_parameter_index(stmt, ":device");
    535 	rc = sqlite3_bind_int64(stmt, idx, device_cache);
    536 	if (rc != SQLITE_OK) {
    537 		if (mflags.verbosity)
    538 			warnx("%s", sqlite3_errmsg(db));
    539 		sqlite3_finalize(stmt);
    540 		return;
    541 	}
    542 
    543 	idx = sqlite3_bind_parameter_index(stmt, ":inode");
    544 	rc = sqlite3_bind_int64(stmt, idx, inode_cache);
    545 	if (rc != SQLITE_OK) {
    546 		if (mflags.verbosity)
    547 			warnx("%s", sqlite3_errmsg(db));
    548 		sqlite3_finalize(stmt);
    549 		return;
    550 	}
    551 
    552 	idx = sqlite3_bind_parameter_index(stmt, ":mtime");
    553 	rc = sqlite3_bind_int64(stmt, idx, mtime_cache);
    554 	if (rc != SQLITE_OK) {
    555 		if (mflags.verbosity)
    556 			warnx("%s", sqlite3_errmsg(db));
    557 		sqlite3_finalize(stmt);
    558 		return;
    559 	}
    560 
    561 	idx = sqlite3_bind_parameter_index(stmt, ":parent");
    562 	rc = sqlite3_bind_text(stmt, idx, parent, -1, NULL);
    563 	if (rc != SQLITE_OK) {
    564 		if (mflags.verbosity)
    565 			warnx("%s", sqlite3_errmsg(db));
    566 		sqlite3_finalize(stmt);
    567 		return;
    568 	}
    569 
    570 	idx = sqlite3_bind_parameter_index(stmt, ":file");
    571 	rc = sqlite3_bind_text(stmt, idx, file, -1, NULL);
    572 	if (rc != SQLITE_OK) {
    573 		if (mflags.verbosity)
    574 			warnx("%s", sqlite3_errmsg(db));
    575 		sqlite3_finalize(stmt);
    576 		return;
    577 	}
    578 
    579 	sqlite3_step(stmt);
    580 	sqlite3_finalize(stmt);
    581 }
    582 
    583 /* read_and_decompress --
    584  *	Reads the given file into memory. If it is compressed, decompress
    585  *	it before returning to the caller.
    586  */
    587 static int
    588 read_and_decompress(const char *file, void **bufp, size_t *len)
    589 {
    590 	size_t off;
    591 	ssize_t r;
    592 	struct archive *a;
    593 	struct archive_entry *ae;
    594 	char *buf;
    595 
    596 	if ((a = archive_read_new()) == NULL)
    597 		errx(EXIT_FAILURE, "memory allocation failed");
    598 
    599 	*bufp = NULL;
    600 	if (archive_read_support_filter_all(a) != ARCHIVE_OK ||
    601 	    archive_read_support_format_raw(a) != ARCHIVE_OK ||
    602 	    archive_read_open_filename(a, file, 65536) != ARCHIVE_OK ||
    603 	    archive_read_next_header(a, &ae) != ARCHIVE_OK)
    604 		goto archive_error;
    605 	*len = 65536;
    606 	buf = emalloc(*len);
    607 	off = 0;
    608 	for (;;) {
    609 		r = archive_read_data(a, buf + off, *len - off);
    610 		if (r == ARCHIVE_OK) {
    611 			archive_read_free(a);
    612 			*bufp = buf;
    613 			*len = off;
    614 			return 0;
    615 		}
    616 		if (r <= 0) {
    617 			free(buf);
    618 			break;
    619 		}
    620 		off += r;
    621 		if (off == *len) {
    622 			*len *= 2;
    623 			if (*len < off) {
    624 				if (mflags.verbosity)
    625 					warnx("File too large: %s", file);
    626 				free(buf);
    627 				archive_read_free(a);
    628 				return -1;
    629 			}
    630 			buf = erealloc(buf, *len);
    631 		}
    632 	}
    633 
    634 archive_error:
    635 	warnx("Error while reading `%s': %s", file, archive_error_string(a));
    636 	archive_read_free(a);
    637 	return -1;
    638 }
    639 
    640 static void
    641 update_existing_entry(sqlite3 *db, const char *file, const char *hash,
    642     mandb_rec *rec, int *new_count, int *link_count, int *err_count)
    643 {
    644 	int update_count, rc, idx;
    645 	const char *inner_sqlstr;
    646 	sqlite3_stmt *inner_stmt;
    647 
    648 	update_count = sqlite3_total_changes(db);
    649 	inner_sqlstr = "UPDATE mandb_meta SET device = :device,"
    650 		       " inode = :inode, mtime = :mtime WHERE"
    651 		       " md5_hash = :md5 AND file = :file AND"
    652 		       " (device <> :device2 OR inode <> "
    653 		       "  :inode2 OR mtime <> :mtime2)";
    654 	rc = sqlite3_prepare_v2(db, inner_sqlstr, -1, &inner_stmt, NULL);
    655 	if (rc != SQLITE_OK) {
    656 		if (mflags.verbosity)
    657 			warnx("%s", sqlite3_errmsg(db));
    658 		return;
    659 	}
    660 	idx = sqlite3_bind_parameter_index(inner_stmt, ":device");
    661 	sqlite3_bind_int64(inner_stmt, idx, rec->device);
    662 	idx = sqlite3_bind_parameter_index(inner_stmt, ":inode");
    663 	sqlite3_bind_int64(inner_stmt, idx, rec->inode);
    664 	idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime");
    665 	sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
    666 	idx = sqlite3_bind_parameter_index(inner_stmt, ":md5");
    667 	sqlite3_bind_text(inner_stmt, idx, hash, -1, NULL);
    668 	idx = sqlite3_bind_parameter_index(inner_stmt, ":file");
    669 	sqlite3_bind_text(inner_stmt, idx, file, -1, NULL);
    670 	idx = sqlite3_bind_parameter_index(inner_stmt, ":device2");
    671 	sqlite3_bind_int64(inner_stmt, idx, rec->device);
    672 	idx = sqlite3_bind_parameter_index(inner_stmt, ":inode2");
    673 	sqlite3_bind_int64(inner_stmt, idx, rec->inode);
    674 	idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime2");
    675 	sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
    676 
    677 	rc = sqlite3_step(inner_stmt);
    678 	if (rc == SQLITE_DONE) {
    679 		/* Check if an update has been performed. */
    680 		if (update_count != sqlite3_total_changes(db)) {
    681 			if (mflags.verbosity == 2)
    682 				printf("Updated %s\n", file);
    683 			(*new_count)++;
    684 		} else {
    685 			/* Otherwise it was a hardlink. */
    686 			(*link_count)++;
    687 		}
    688 	} else {
    689 		if (mflags.verbosity == 2)
    690 			warnx("Could not update the meta data for %s", file);
    691 		(*err_count)++;
    692 	}
    693 	sqlite3_finalize(inner_stmt);
    694 }
    695 
    696 /* update_db --
    697  *	Does an incremental updation of the database by checking the file_cache.
    698  *	It parses and adds the pages which are present in file_cache,
    699  *	but not in the database.
    700  *	It also removes the pages which are present in the databse,
    701  *	but not in the file_cache.
    702  */
    703 static void
    704 update_db(sqlite3 *db, struct mparse *mp, mandb_rec *rec)
    705 {
    706 	const char *sqlstr;
    707 	sqlite3_stmt *stmt = NULL;
    708 	char *file;
    709 	char *parent;
    710 	char *errmsg = NULL;
    711 	char *md5sum;
    712 	void *buf;
    713 	size_t buflen;
    714 	struct sql_row {
    715 		struct sql_row *next;
    716 		dev_t device;
    717 		ino_t inode;
    718 		time_t mtime;
    719 		char *parent;
    720 		char *file;
    721 	} *rows, *row;
    722 	int new_count = 0;	/* Counter for newly indexed/updated pages */
    723 	int total_count = 0;	/* Counter for total number of pages */
    724 	int err_count = 0;	/* Counter for number of failed pages */
    725 	int link_count = 0;	/* Counter for number of hard/sym links */
    726 	int md5_status;
    727 	int rc;
    728 
    729 	sqlstr = "SELECT device, inode, mtime, parent, file"
    730 	         " FROM metadb.file_cache fc"
    731 	         " WHERE NOT EXISTS(SELECT 1 FROM mandb_meta WHERE"
    732 	         "  device = fc.device AND inode = fc.inode AND "
    733 	         "  mtime = fc.mtime AND file = fc.file)";
    734 
    735 	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
    736 	if (rc != SQLITE_OK) {
    737 		if (mflags.verbosity)
    738 			warnx("%s", sqlite3_errmsg(db));
    739 		close_db(db);
    740 		errx(EXIT_FAILURE, "Could not query file cache");
    741 	}
    742 
    743 	buf = NULL;
    744 	rows = NULL;
    745 	while (sqlite3_step(stmt) == SQLITE_ROW) {
    746 		row = emalloc(sizeof(struct sql_row));
    747 		row->device = sqlite3_column_int64(stmt, 0);
    748 		row->inode = sqlite3_column_int64(stmt, 1);
    749 		row->mtime = sqlite3_column_int64(stmt, 2);
    750 		row->parent = estrdup((const char *) sqlite3_column_text(stmt, 3));
    751 		row->file = estrdup((const char *) sqlite3_column_text(stmt, 4));
    752 		row->next = rows;
    753 		rows = row;
    754 		total_count++;
    755 	}
    756 	sqlite3_finalize(stmt);
    757 
    758 	for ( ; rows != NULL; free(parent), free(file), free(buf)) {
    759 		int fd;
    760 
    761 		row = rows;
    762 		rows = rows->next;
    763 
    764 		rec->device = row->device;
    765 		rec->inode = row->inode;
    766 		rec->mtime = row->mtime;
    767 		parent = row->parent;
    768 		file = row->file;
    769 		free(row);
    770 
    771 		// XXX: reading twice!
    772 		if (read_and_decompress(file, &buf, &buflen)) {
    773 			err_count++;
    774 			continue;
    775 		}
    776 		if ((fd = mparse_open(mp, file)) == -1) {
    777 			err_count++;
    778 			continue;
    779 		}
    780 
    781 		md5_status = check_md5(file, db, &md5sum, buf, buflen);
    782 		assert(md5sum != NULL);
    783 		if (md5_status == -1) {
    784 			if (mflags.verbosity)
    785 				warnx("An error occurred in checking md5 value"
    786 			      " for file %s", file);
    787 			err_count++;
    788 			close(fd);
    789 			continue;
    790 		}
    791 
    792 		if (md5_status == 0) {
    793 			/*
    794 			 * The MD5 hash is already present in the database,
    795 			 * so simply update the metadata.
    796 			 */
    797 			update_existing_entry(db, file, md5sum, rec,
    798 			    &new_count, &link_count, &err_count);
    799 			free(md5sum);
    800 			close(fd);
    801 			continue;
    802 		}
    803 
    804 		if (md5_status == 1) {
    805 			/*
    806 			 * The MD5 hash was not present in the database.
    807 			 * This means is either a new file or an updated file.
    808 			 * We should go ahead with parsing.
    809 			 */
    810 			if (chdir(parent) == -1) {
    811 				if (mflags.verbosity)
    812 					warn("chdir failed for `%s', could "
    813 					    "not index `%s'", parent, file);
    814 				err_count++;
    815 				free(md5sum);
    816 				close(fd);
    817 				continue;
    818 			}
    819 
    820 			if (mflags.verbosity == 2)
    821 				printf("Parsing: %s\n", file);
    822 			rec->md5_hash = md5sum;
    823 			rec->file_path = estrdup(file);
    824 			// file_path is freed by insert_into_db itself.
    825 			begin_parse(file, mp, rec, fd);
    826 			if (insert_into_db(db, rec) < 0) {
    827 				if (mflags.verbosity)
    828 					warnx("Error in indexing `%s'", file);
    829 				err_count++;
    830 			} else {
    831 				new_count++;
    832 			}
    833 		}
    834 		close(fd);
    835 	}
    836 
    837 	if (mflags.verbosity == 2) {
    838 		printf("Number of new or updated pages encountered: %d\n"
    839 		    "Number of hard links found: %d\n"
    840 		    "Number of pages that were successfully"
    841 		    " indexed or updated: %d\n"
    842 		    "Number of pages that could not be indexed"
    843 		    " due to errors: %d\n",
    844 		    total_count - link_count, link_count, new_count, err_count);
    845 	}
    846 
    847 	if (mflags.recreate)
    848 		return;
    849 
    850 	if (mflags.verbosity == 2)
    851 		printf("Deleting stale index entries\n");
    852 
    853 	sqlstr = "DELETE FROM mandb_meta WHERE file NOT IN"
    854 		 " (SELECT file FROM metadb.file_cache);"
    855 		 "DELETE FROM mandb_links WHERE md5_hash NOT IN"
    856 		 " (SELECT md5_hash from mandb_meta);"
    857 		 "DROP TABLE metadb.file_cache;"
    858 		 "DELETE FROM mandb WHERE rowid NOT IN"
    859 		 " (SELECT id FROM mandb_meta);";
    860 
    861 	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
    862 	if (errmsg != NULL) {
    863 		warnx("Removing old entries failed: %s", errmsg);
    864 		warnx("Please rebuild database from scratch with -f.");
    865 		free(errmsg);
    866 		return;
    867 	}
    868 }
    869 
    870 /*
    871  * begin_parse --
    872  *  parses the man page using libmandoc
    873  */
    874 static void
    875 begin_parse(const char *file, struct mparse *mp, mandb_rec *rec, int fd)
    876 {
    877 	struct roff_meta *roff;
    878 	mparse_reset(mp);
    879 
    880 	rec->xr_found = 0;
    881 
    882 	mparse_readfd(mp, fd, file);
    883 	roff = mparse_result(mp);
    884 	if (roff == NULL) {
    885 		if (mflags.verbosity == 2)
    886 			warnx("Not a roff(7) page");
    887 		return;
    888 	}
    889 
    890 	if (roff->macroset == MACROSET_MDOC) {
    891 		rec->page_type = MDOC;
    892 		proff_node(roff->first->child, rec, roff, mdocs);
    893 	} else if (roff->macroset == MACROSET_MAN) {
    894 		rec->page_type = MAN;
    895 		proff_node(roff->first->child, rec, roff, mans);
    896 	} else
    897 		warnx("Unknown macroset %d", roff->macroset);
    898 	set_machine(roff, rec);
    899 	set_section(roff, rec);
    900 }
    901 
    902 /*
    903  * set_section --
    904  *  Extracts the section number and normalizes it to only the numeric part
    905  *  (Which should be the first character of the string).
    906  */
    907 static void
    908 set_section(const struct roff_meta *rm, mandb_rec *rec)
    909 {
    910 	if (!rm)
    911 		return;
    912 	const char *s = rm->msec == NULL ? "?" : rm->msec;
    913 	easprintf(&rec->section, "%s", s);
    914 	if (rec->section[0] == '?' && mflags.verbosity == 2)
    915 		warnx("%s: Missing section number", rec->file_path);
    916 }
    917 
    918 /*
    919  * get_machine --
    920  *  Extracts the machine architecture information if available.
    921  */
    922 static void
    923 set_machine(const struct roff_meta *rm, mandb_rec *rec)
    924 {
    925 	if (rm == NULL)
    926 		return;
    927 	if (rm->arch)
    928 		rec->machine = estrdup(rm->arch);
    929 }
    930 
    931 /*
    932  * pmdoc_Nm --
    933  *  Extracts the Name of the manual page from the .Nm macro
    934  */
    935 static void
    936 pmdoc_Nm(const struct roff_node *n, mandb_rec *rec)
    937 {
    938 	if (n->sec != SEC_NAME)
    939 		return;
    940 
    941 	for (n = n->child; n; n = n->next) {
    942 		if (n->type == ROFFT_TEXT) {
    943 			char *escaped_name = parse_escape(n->string);
    944 			concat(&rec->name, escaped_name);
    945 			free(escaped_name);
    946 		}
    947 	}
    948 }
    949 
    950 /*
    951  * pmdoc_Nd --
    952  *  Extracts the one line description of the man page from the .Nd macro
    953  */
    954 static void
    955 pmdoc_Nd(const struct roff_node *n, mandb_rec *rec)
    956 {
    957 	if (n->type == ROFFT_BODY)
    958 		deroff(&rec->name_desc, n);
    959 	if (rec->name_desc)
    960 		replace_hyph(rec->name_desc);
    961 
    962 }
    963 
    964 /*
    965  * pmdoc_macro_handler--
    966  *  This function is a single point of handling all the special macros that we
    967  *  want to handle especially. For example the .Xr macro for properly parsing
    968  *  the referenced page name along with the section number, or the .Pp macro
    969  *  for adding a new line whenever we encounter it.
    970  */
    971 static void
    972 pmdoc_macro_handler(const struct roff_node *n, mandb_rec *rec, int doct)
    973 {
    974 	const struct roff_node *sn;
    975 	assert(n);
    976 
    977 	switch (doct) {
    978 	/*  Parse the man page references.
    979 	 * Basically the .Xr macros are used like:
    980 	 *  .Xr ls 1
    981  	 *  and formatted like this:
    982 	 *  ls(1)
    983 	 *  Prepare a buffer to format the data like the above example and call
    984 	 *  pmdoc_parse_section to append it.
    985 	 */
    986 	case MDOC_Xr:
    987 		n = n->child;
    988 		while (n->type != ROFFT_TEXT && n->next)
    989 			n = n->next;
    990 
    991 		if (n && n->type != ROFFT_TEXT)
    992 			return;
    993 		sn = n;
    994 		if (n->next)
    995 			n = n->next;
    996 
    997 		while (n->type != ROFFT_TEXT && n->next)
    998 			n = n->next;
    999 
   1000 		if (n && n->type == ROFFT_TEXT) {
   1001 			char *buf;
   1002 			easprintf(&buf, "%s(%s)", sn->string, n->string);
   1003 			mdoc_parse_section(n->sec, buf, rec);
   1004 			free(buf);
   1005 		}
   1006 
   1007 		break;
   1008 
   1009 	/* Parse the .Pp macro to add a new line */
   1010 	case MDOC_Pp:
   1011 		if (n->type == ROFFT_TEXT)
   1012 			mdoc_parse_section(n->sec, "\n", rec);
   1013 		break;
   1014 	default:
   1015 		break;
   1016 	}
   1017 
   1018 }
   1019 
   1020 /*
   1021  * pmdoc_Xr, pmdoc_Pp--
   1022  *  Empty stubs.
   1023  *  The parser calls these functions each time it encounters
   1024  *  a .Xr or .Pp macro. We are parsing all the data from
   1025  *  the pmdoc_Sh function, so don't do anything here.
   1026  *  (See if else blocks in pmdoc_Sh.)
   1027  */
   1028 static void
   1029 pmdoc_Xr(const struct roff_node *n, mandb_rec *rec)
   1030 {
   1031 }
   1032 
   1033 static void
   1034 pmdoc_Pp(const struct roff_node *n, mandb_rec *rec)
   1035 {
   1036 }
   1037 
   1038 /*
   1039  * pmdoc_Sh --
   1040  *  Called when a .Sh macro is encountered and tries to parse its body
   1041  */
   1042 static void
   1043 pmdoc_Sh(const struct roff_node *n, mandb_rec *rec)
   1044 {
   1045 	if (n == NULL)
   1046 		return;
   1047 
   1048 	switch (n->sec) {
   1049 	case SEC_NAME:
   1050 	case SEC_SYNOPSIS:
   1051 	case SEC_EXAMPLES:
   1052 	case SEC_STANDARDS:
   1053 	case SEC_HISTORY:
   1054 	case SEC_AUTHORS:
   1055 	case SEC_BUGS:
   1056 		/*
   1057 		 * We don't care about text from these sections
   1058 		 */
   1059 		return;
   1060 	default:
   1061 		break;
   1062 	}
   1063 
   1064 	if (n->type == ROFFT_BLOCK)
   1065 		mdoc_parse_Sh(n->body, rec);
   1066 }
   1067 
   1068 /*
   1069  *  Called from pmdoc_Sh to parse body of a .Sh macro. It calls
   1070  *  mdoc_parse_section to append the data to the section specific buffer.
   1071  *  The .Xr macro needs special handling, thus the separate if branch for it.
   1072  */
   1073 static void
   1074 mdoc_parse_Sh(const struct roff_node *n, mandb_rec *rec)
   1075 {
   1076 	if (n == NULL || (n->type != ROFFT_TEXT && n->tok == MDOC_MAX))
   1077 		return;
   1078 	int xr_found = 0;
   1079 
   1080 	if (n->type == ROFFT_TEXT) {
   1081 		mdoc_parse_section(n->sec, n->string, rec);
   1082 	} else if (mdocs[n->tok] == pmdoc_Xr) {
   1083 		/*
   1084 		 * When encountering other inline macros,
   1085 		 * call pmdoc_macro_handler.
   1086 		 */
   1087 		pmdoc_macro_handler(n, rec, MDOC_Xr);
   1088 		xr_found = 1;
   1089 	} else if (mdocs[n->tok] == pmdoc_Pp) {
   1090 		pmdoc_macro_handler(n, rec, MDOC_Pp);
   1091 	}
   1092 
   1093 	/*
   1094 	 * If an Xr macro was encountered then the child node has
   1095 	 * already been explored by pmdoc_macro_handler.
   1096 	 */
   1097 	if (xr_found == 0)
   1098 		mdoc_parse_Sh(n->child, rec);
   1099 	mdoc_parse_Sh(n->next, rec);
   1100 }
   1101 
   1102 /*
   1103  * mdoc_parse_section--
   1104  *  Utility function for parsing sections of the mdoc type pages.
   1105  *  Takes two params:
   1106  *   1. sec is an enum which indicates the section in which we are present
   1107  *   2. string is the string which we need to append to the secbuff for this
   1108  *      particular section.
   1109  *  The function appends string to the global section buffer and returns.
   1110  */
   1111 static void
   1112 mdoc_parse_section(enum roff_sec sec, const char *string, mandb_rec *rec)
   1113 {
   1114 	/*
   1115 	 * If the user specified the 'l' flag, then parse and store only the
   1116 	 * NAME section. Ignore the rest.
   1117 	 */
   1118 	if (mflags.limit)
   1119 		return;
   1120 
   1121 	switch (sec) {
   1122 	case SEC_LIBRARY:
   1123 		append(&rec->lib, string);
   1124 		break;
   1125 	case SEC_RETURN_VALUES:
   1126 		append(&rec->return_vals, string);
   1127 		break;
   1128 	case SEC_ENVIRONMENT:
   1129 		append(&rec->env, string);
   1130 		break;
   1131 	case SEC_FILES:
   1132 		append(&rec->files, string);
   1133 		break;
   1134 	case SEC_EXIT_STATUS:
   1135 		append(&rec->exit_status, string);
   1136 		break;
   1137 	case SEC_DIAGNOSTICS:
   1138 		append(&rec->diagnostics, string);
   1139 		break;
   1140 	case SEC_ERRORS:
   1141 		append(&rec->errors, string);
   1142 		break;
   1143 	default:
   1144 		append(&rec->desc, string);
   1145 		break;
   1146 	}
   1147 }
   1148 
   1149 static void
   1150 proff_node(const struct roff_node * n, mandb_rec * rec,
   1151     struct roff_meta * roff, const proff_nf * func)
   1152 {
   1153 	if (n == NULL)
   1154 		return;
   1155 
   1156 	int tok_idx;
   1157 
   1158 	switch (n->type) {
   1159 	case (ROFFT_BODY):
   1160 		/* FALLTHROUGH */
   1161 	case (ROFFT_BLOCK):
   1162 		/* FALLTHROUGH */
   1163 	case (ROFFT_ELEM):
   1164 		if (roff->macroset == MACROSET_MAN)
   1165 			tok_idx = n->tok - MAN_TH;
   1166 		else if (roff->macroset == MACROSET_MDOC)
   1167 			tok_idx = n->tok - MDOC_Dd;
   1168 		else
   1169 			tok_idx = -1;
   1170 		if (tok_idx >= 0 && func[tok_idx] != NULL)
   1171 			(*func[tok_idx]) (n, rec);
   1172 		break;
   1173 	default:
   1174 		break;
   1175 	}
   1176 
   1177 	proff_node(n->child, rec, roff, func);
   1178 	proff_node(n->next, rec, roff, func);
   1179 }
   1180 
   1181 /*
   1182  * pman_parse_name --
   1183  *  Parses the NAME section and puts the complete content in the name_desc
   1184  *  variable.
   1185  */
   1186 static void
   1187 pman_parse_name(const struct roff_node *n, mandb_rec *rec)
   1188 {
   1189 	if (n == NULL)
   1190 		return;
   1191 
   1192 	if (n->type == ROFFT_TEXT) {
   1193 		char *tmp = parse_escape(n->string);
   1194 		concat(&rec->name_desc, tmp);
   1195 		free(tmp);
   1196 	}
   1197 
   1198 	if (n->child)
   1199 		pman_parse_name(n->child, rec);
   1200 
   1201 	if(n->next)
   1202 		pman_parse_name(n->next, rec);
   1203 }
   1204 
   1205 /*
   1206  * A stub function to be able to parse the macros like .B embedded inside
   1207  * a section.
   1208  */
   1209 static void
   1210 pman_block(const struct roff_node *n, mandb_rec *rec)
   1211 {
   1212 }
   1213 
   1214 /*
   1215  * pman_sh --
   1216  * This function does one of the two things:
   1217  *  1. If the present section is NAME, then it will:
   1218  *    (a) Extract the name of the page (in case of multiple comma separated
   1219  *        names, it will pick up the first one).
   1220  *    (b) Build a space spearated list of all the symlinks/hardlinks to
   1221  *        this page and store in the buffer 'links'. These are extracted from
   1222  *        the comma separated list of names in the NAME section as well.
   1223  *    (c) Move on to the one line description section, which is after the list
   1224  *        of names in the NAME section.
   1225  *  2. Otherwise, it will check the section name and call the man_parse_section
   1226  *     function, passing the enum corresponding to that section.
   1227  */
   1228 static void
   1229 pman_sh(const struct roff_node *n, mandb_rec *rec)
   1230 {
   1231 	static const struct {
   1232 		enum man_sec section;
   1233 		const char *header;
   1234 	} mapping[] = {
   1235 	    { MANSEC_DESCRIPTION, "DESCRIPTION" },
   1236 	    { MANSEC_SYNOPSIS, "SYNOPSIS" },
   1237 	    { MANSEC_LIBRARY, "LIBRARY" },
   1238 	    { MANSEC_ERRORS, "ERRORS" },
   1239 	    { MANSEC_FILES, "FILES" },
   1240 	    { MANSEC_RETURN_VALUES, "RETURN VALUE" },
   1241 	    { MANSEC_RETURN_VALUES, "RETURN VALUES" },
   1242 	    { MANSEC_EXIT_STATUS, "EXIT STATUS" },
   1243 	    { MANSEC_EXAMPLES, "EXAMPLES" },
   1244 	    { MANSEC_EXAMPLES, "EXAMPLE" },
   1245 	    { MANSEC_STANDARDS, "STANDARDS" },
   1246 	    { MANSEC_HISTORY, "HISTORY" },
   1247 	    { MANSEC_BUGS, "BUGS" },
   1248 	    { MANSEC_AUTHORS, "AUTHORS" },
   1249 	    { MANSEC_COPYRIGHT, "COPYRIGHT" },
   1250 	};
   1251 	const struct roff_node *head;
   1252 	char *name_desc;
   1253 	size_t sz;
   1254 	size_t i;
   1255 
   1256 	if ((head = n->parent->head) == NULL || (head = head->child) == NULL ||
   1257 	    head->type != ROFFT_TEXT)
   1258 		return;
   1259 
   1260 	/*
   1261 	 * Check if this section should be extracted and
   1262 	 * where it should be stored. Handled the trival cases first.
   1263 	 */
   1264 	for (i = 0; i < sizeof(mapping) / sizeof(mapping[0]); ++i) {
   1265 		if (strcmp(head->string, mapping[i].header) == 0) {
   1266 			man_parse_section(mapping[i].section, n, rec);
   1267 			return;
   1268 		}
   1269 	}
   1270 
   1271 	if (strcmp(head->string, "NAME") == 0) {
   1272 		/*
   1273 		 * We are in the NAME section.
   1274 		 * pman_parse_name will put the complete content in name_desc.
   1275 		 */
   1276 		pman_parse_name(n, rec);
   1277 
   1278 		name_desc = rec->name_desc;
   1279 		if (name_desc == NULL)
   1280 			return;
   1281 
   1282 		/* Remove any leading spaces. */
   1283 		while (name_desc[0] == ' ')
   1284 			name_desc++;
   1285 
   1286 		/* If the line begins with a "\&", avoid those */
   1287 		if (name_desc[0] == '\\' && name_desc[1] == '&')
   1288 			name_desc += 2;
   1289 
   1290 		/* Now name_desc should be left with a comma-space
   1291 		 * separated list of names and the one line description
   1292 		 * of the page:
   1293 		 *     "a, b, c \- sample description"
   1294 		 * Take out the first name, before the first comma
   1295 		 * (or space) and store it in rec->name.
   1296 		 * If the page has aliases then they should be
   1297 		 * in the form of a comma separated list.
   1298 		 * Keep looping while there is a comma in name_desc,
   1299 		 * extract the alias name and store in rec->links.
   1300 		 * When there are no more commas left, break out.
   1301 		 */
   1302 		int has_alias = 0;	// Any more aliases left?
   1303 		while (*name_desc) {
   1304 			/* Remove any leading spaces or hyphens. */
   1305 			if (name_desc[0] == ' ' || name_desc[0] == '-') {
   1306 				name_desc++;
   1307 				continue;
   1308 			}
   1309 			sz = strcspn(name_desc, ", ");
   1310 
   1311 			/* Extract the first term and store it in rec->name. */
   1312 			if (rec->name == NULL) {
   1313 				if (name_desc[sz] == ',')
   1314 					has_alias = 1;
   1315 				rec->name = estrndup(name_desc, sz);
   1316 				/* XXX This would only happen with a poorly
   1317 				 * written man page, maybe warn? */
   1318 				if (name_desc[sz] == '\0')
   1319 					break;
   1320 				name_desc += sz + 1;
   1321 				continue;
   1322 			}
   1323 
   1324 			/*
   1325 			 * Once rec->name is set, rest of the names
   1326 			 * are to be treated as links or aliases.
   1327 			 */
   1328 			if (rec->name && has_alias) {
   1329 				if (name_desc[sz] != ',') {
   1330 					/* No more commas left --> no more
   1331 					 * aliases to take out */
   1332 					has_alias = 0;
   1333 				}
   1334 				concat2(&rec->links, name_desc, sz);
   1335 				/* XXX This would only happen with a poorly
   1336 				 * written man page, maybe warn? */
   1337 				if (name_desc[sz] == '\0')
   1338 					break;
   1339 				name_desc += sz + 1;
   1340 				continue;
   1341 			}
   1342 			break;
   1343 		}
   1344 
   1345 		/* Parse any escape sequences that might be there */
   1346 		char *temp = parse_escape(name_desc);
   1347 		free(rec->name_desc);
   1348 		rec->name_desc = temp;
   1349 		temp = parse_escape(rec->name);
   1350 		free(rec->name);
   1351 		rec->name = temp;
   1352 		return;
   1353 	}
   1354 
   1355 	/* The RETURN VALUE section might be specified in multiple ways */
   1356 	if (strcmp(head->string, "RETURN") == 0 &&
   1357 	    head->next != NULL && head->next->type == ROFFT_TEXT &&
   1358 	    (strcmp(head->next->string, "VALUE") == 0 ||
   1359 	    strcmp(head->next->string, "VALUES") == 0)) {
   1360 		man_parse_section(MANSEC_RETURN_VALUES, n, rec);
   1361 		return;
   1362 	}
   1363 
   1364 	/*
   1365 	 * EXIT STATUS section can also be specified all on one line or on two
   1366 	 * separate lines.
   1367 	 */
   1368 	if (strcmp(head->string, "EXIT") == 0 &&
   1369 	    head->next != NULL && head->next->type == ROFFT_TEXT &&
   1370 	    strcmp(head->next->string, "STATUS") == 0) {
   1371 		man_parse_section(MANSEC_EXIT_STATUS, n, rec);
   1372 		return;
   1373 	}
   1374 
   1375 	/* Store the rest of the content in desc. */
   1376 	man_parse_section(MANSEC_NONE, n, rec);
   1377 }
   1378 
   1379 /*
   1380  * pman_parse_node --
   1381  *  Generic function to iterate through a node. Usually called from
   1382  *  man_parse_section to parse a particular section of the man page.
   1383  */
   1384 static void
   1385 pman_parse_node(const struct roff_node *n, secbuff *s)
   1386 {
   1387 	if (n == NULL)
   1388 		return;
   1389 
   1390 	if (n->type == ROFFT_TEXT)
   1391 		append(s, n->string);
   1392 
   1393 	pman_parse_node(n->child, s);
   1394 	pman_parse_node(n->next, s);
   1395 }
   1396 
   1397 /*
   1398  * man_parse_section --
   1399  *  Takes two parameters:
   1400  *   sec: Tells which section we are present in
   1401  *   n: Is the present node of the AST.
   1402  * Depending on the section, we call pman_parse_node to parse that section and
   1403  * concatenate the content from that section into the buffer for that section.
   1404  */
   1405 static void
   1406 man_parse_section(enum man_sec sec, const struct roff_node *n, mandb_rec *rec)
   1407 {
   1408 	/*
   1409 	 * If the user sepecified the 'l' flag then just parse
   1410 	 * the NAME section, ignore the rest.
   1411 	 */
   1412 	if (mflags.limit)
   1413 		return;
   1414 
   1415 	switch (sec) {
   1416 	case MANSEC_LIBRARY:
   1417 		pman_parse_node(n, &rec->lib);
   1418 		break;
   1419 	case MANSEC_RETURN_VALUES:
   1420 		pman_parse_node(n, &rec->return_vals);
   1421 		break;
   1422 	case MANSEC_ENVIRONMENT:
   1423 		pman_parse_node(n, &rec->env);
   1424 		break;
   1425 	case MANSEC_FILES:
   1426 		pman_parse_node(n, &rec->files);
   1427 		break;
   1428 	case MANSEC_EXIT_STATUS:
   1429 		pman_parse_node(n, &rec->exit_status);
   1430 		break;
   1431 	case MANSEC_DIAGNOSTICS:
   1432 		pman_parse_node(n, &rec->diagnostics);
   1433 		break;
   1434 	case MANSEC_ERRORS:
   1435 		pman_parse_node(n, &rec->errors);
   1436 		break;
   1437 	case MANSEC_NAME:
   1438 	case MANSEC_SYNOPSIS:
   1439 	case MANSEC_EXAMPLES:
   1440 	case MANSEC_STANDARDS:
   1441 	case MANSEC_HISTORY:
   1442 	case MANSEC_BUGS:
   1443 	case MANSEC_AUTHORS:
   1444 	case MANSEC_COPYRIGHT:
   1445 		break;
   1446 	default:
   1447 		pman_parse_node(n, &rec->desc);
   1448 		break;
   1449 	}
   1450 
   1451 }
   1452 
   1453 /*
   1454  * insert_into_db --
   1455  *  Inserts the parsed data of the man page in the Sqlite databse.
   1456  *  If any of the values is NULL, then we cleanup and return -1 indicating
   1457  *  an error.
   1458  *  Otherwise, store the data in the database and return 0.
   1459  */
   1460 static int
   1461 insert_into_db(sqlite3 *db, mandb_rec *rec)
   1462 {
   1463 	int rc = 0;
   1464 	int idx = -1;
   1465 	const char *sqlstr = NULL;
   1466 	sqlite3_stmt *stmt = NULL;
   1467 	char *ln = NULL;
   1468 	char *errmsg = NULL;
   1469 	long int mandb_rowid;
   1470 
   1471 	/*
   1472 	 * At the very minimum we want to make sure that we store
   1473 	 * the following data:
   1474 	 *   Name, one line description, and the MD5 hash
   1475 	 */
   1476 	if (rec->name == NULL || rec->name_desc == NULL ||
   1477 	    rec->md5_hash == NULL) {
   1478 		cleanup(rec);
   1479 		return -1;
   1480 	}
   1481 
   1482 	/* Write null byte at the end of all the sec_buffs */
   1483 	rec->desc.data[rec->desc.offset] = 0;
   1484 	rec->lib.data[rec->lib.offset] = 0;
   1485 	rec->env.data[rec->env.offset] = 0;
   1486 	rec->return_vals.data[rec->return_vals.offset] = 0;
   1487 	rec->exit_status.data[rec->exit_status.offset] = 0;
   1488 	rec->files.data[rec->files.offset] = 0;
   1489 	rec->diagnostics.data[rec->diagnostics.offset] = 0;
   1490 	rec->errors.data[rec->errors.offset] = 0;
   1491 
   1492 	/*
   1493 	 * In case of a mdoc page: (sorry, no better place to put this code)
   1494 	 * parse the comma separated list of names of man pages,
   1495 	 * the first name will be stored in the mandb table, rest will be
   1496 	 * treated as links and put in the mandb_links table.
   1497 	 */
   1498 	if (rec->page_type == MDOC) {
   1499 		char *tmp;
   1500 		rec->links = estrdup(rec->name);
   1501 		free(rec->name);
   1502 		size_t sz = strcspn(rec->links, " \0");
   1503 		rec->name = emalloc(sz + 1);
   1504 		memcpy(rec->name, rec->links, sz);
   1505 		if(rec->name[sz - 1] == ',')
   1506 			rec->name[sz - 1] = 0;
   1507 		else
   1508 			rec->name[sz] = 0;
   1509 		while (rec->links[sz] == ' ')
   1510 			++sz;
   1511 		tmp = estrdup(rec->links + sz);
   1512 		free(rec->links);
   1513 		rec->links = tmp;
   1514 	}
   1515 
   1516 /*------------------------ Populate the mandb table---------------------------*/
   1517 	sqlstr = "INSERT INTO mandb VALUES (:section, :name, :name_desc, :desc,"
   1518 		 " :lib, :return_vals, :env, :files, :exit_status,"
   1519 		 " :diagnostics, :errors, :md5_hash, :machine)";
   1520 
   1521 	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
   1522 	if (rc != SQLITE_OK)
   1523 		goto Out;
   1524 
   1525 	idx = sqlite3_bind_parameter_index(stmt, ":name");
   1526 	rc = sqlite3_bind_text(stmt, idx, rec->name, -1, NULL);
   1527 	if (rc != SQLITE_OK) {
   1528 		sqlite3_finalize(stmt);
   1529 		goto Out;
   1530 	}
   1531 
   1532 	idx = sqlite3_bind_parameter_index(stmt, ":section");
   1533 	rc = sqlite3_bind_text(stmt, idx, rec->section, -1, NULL);
   1534 	if (rc != SQLITE_OK) {
   1535 		sqlite3_finalize(stmt);
   1536 		goto Out;
   1537 	}
   1538 
   1539 	idx = sqlite3_bind_parameter_index(stmt, ":name_desc");
   1540 	rc = sqlite3_bind_text(stmt, idx, rec->name_desc, -1, NULL);
   1541 	if (rc != SQLITE_OK) {
   1542 		sqlite3_finalize(stmt);
   1543 		goto Out;
   1544 	}
   1545 
   1546 	idx = sqlite3_bind_parameter_index(stmt, ":desc");
   1547 	rc = sqlite3_bind_text(stmt, idx, rec->desc.data,
   1548 	                       rec->desc.offset + 1, NULL);
   1549 	if (rc != SQLITE_OK) {
   1550 		sqlite3_finalize(stmt);
   1551 		goto Out;
   1552 	}
   1553 
   1554 	idx = sqlite3_bind_parameter_index(stmt, ":lib");
   1555 	rc = sqlite3_bind_text(stmt, idx, rec->lib.data,
   1556 	    rec->lib.offset + 1, NULL);
   1557 	if (rc != SQLITE_OK) {
   1558 		sqlite3_finalize(stmt);
   1559 		goto Out;
   1560 	}
   1561 
   1562 	idx = sqlite3_bind_parameter_index(stmt, ":return_vals");
   1563 	rc = sqlite3_bind_text(stmt, idx, rec->return_vals.data,
   1564 	                      rec->return_vals.offset + 1, NULL);
   1565 	if (rc != SQLITE_OK) {
   1566 		sqlite3_finalize(stmt);
   1567 		goto Out;
   1568 	}
   1569 
   1570 	idx = sqlite3_bind_parameter_index(stmt, ":env");
   1571 	rc = sqlite3_bind_text(stmt, idx, rec->env.data,
   1572 	    rec->env.offset + 1, NULL);
   1573 	if (rc != SQLITE_OK) {
   1574 		sqlite3_finalize(stmt);
   1575 		goto Out;
   1576 	}
   1577 
   1578 	idx = sqlite3_bind_parameter_index(stmt, ":files");
   1579 	rc = sqlite3_bind_text(stmt, idx, rec->files.data,
   1580 	                       rec->files.offset + 1, NULL);
   1581 	if (rc != SQLITE_OK) {
   1582 		sqlite3_finalize(stmt);
   1583 		goto Out;
   1584 	}
   1585 
   1586 	idx = sqlite3_bind_parameter_index(stmt, ":exit_status");
   1587 	rc = sqlite3_bind_text(stmt, idx, rec->exit_status.data,
   1588 	                       rec->exit_status.offset + 1, NULL);
   1589 	if (rc != SQLITE_OK) {
   1590 		sqlite3_finalize(stmt);
   1591 		goto Out;
   1592 	}
   1593 
   1594 	idx = sqlite3_bind_parameter_index(stmt, ":diagnostics");
   1595 	rc = sqlite3_bind_text(stmt, idx, rec->diagnostics.data,
   1596 	                       rec->diagnostics.offset + 1, NULL);
   1597 	if (rc != SQLITE_OK) {
   1598 		sqlite3_finalize(stmt);
   1599 		goto Out;
   1600 	}
   1601 
   1602 	idx = sqlite3_bind_parameter_index(stmt, ":errors");
   1603 	rc = sqlite3_bind_text(stmt, idx, rec->errors.data,
   1604 	                       rec->errors.offset + 1, NULL);
   1605 	if (rc != SQLITE_OK) {
   1606 		sqlite3_finalize(stmt);
   1607 		goto Out;
   1608 	}
   1609 
   1610 	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
   1611 	rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
   1612 	if (rc != SQLITE_OK) {
   1613 		sqlite3_finalize(stmt);
   1614 		goto Out;
   1615 	}
   1616 
   1617 	idx = sqlite3_bind_parameter_index(stmt, ":machine");
   1618 	if (rec->machine)
   1619 		rc = sqlite3_bind_text(stmt, idx, rec->machine, -1, NULL);
   1620 	else
   1621 		rc = sqlite3_bind_null(stmt, idx);
   1622 	if (rc != SQLITE_OK) {
   1623 		sqlite3_finalize(stmt);
   1624 		goto Out;
   1625 	}
   1626 
   1627 	rc = sqlite3_step(stmt);
   1628 	if (rc != SQLITE_DONE) {
   1629 		sqlite3_finalize(stmt);
   1630 		goto Out;
   1631 	}
   1632 
   1633 	sqlite3_finalize(stmt);
   1634 
   1635 	/* Get the row id of the last inserted row */
   1636 	mandb_rowid = sqlite3_last_insert_rowid(db);
   1637 
   1638 /*------------------------Populate the mandb_meta table-----------------------*/
   1639 	sqlstr = "INSERT INTO mandb_meta VALUES (:device, :inode, :mtime,"
   1640 		 " :file, :md5_hash, :id)";
   1641 	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
   1642 	if (rc != SQLITE_OK)
   1643 		goto Out;
   1644 
   1645 	idx = sqlite3_bind_parameter_index(stmt, ":device");
   1646 	rc = sqlite3_bind_int64(stmt, idx, rec->device);
   1647 	if (rc != SQLITE_OK) {
   1648 		sqlite3_finalize(stmt);
   1649 		goto Out;
   1650 	}
   1651 
   1652 	idx = sqlite3_bind_parameter_index(stmt, ":inode");
   1653 	rc = sqlite3_bind_int64(stmt, idx, rec->inode);
   1654 	if (rc != SQLITE_OK) {
   1655 		sqlite3_finalize(stmt);
   1656 		goto Out;
   1657 	}
   1658 
   1659 	idx = sqlite3_bind_parameter_index(stmt, ":mtime");
   1660 	rc = sqlite3_bind_int64(stmt, idx, rec->mtime);
   1661 	if (rc != SQLITE_OK) {
   1662 		sqlite3_finalize(stmt);
   1663 		goto Out;
   1664 	}
   1665 
   1666 	idx = sqlite3_bind_parameter_index(stmt, ":file");
   1667 	rc = sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
   1668 	if (rc != SQLITE_OK) {
   1669 		sqlite3_finalize(stmt);
   1670 		goto Out;
   1671 	}
   1672 
   1673 	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
   1674 	rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
   1675 	if (rc != SQLITE_OK) {
   1676 		sqlite3_finalize(stmt);
   1677 		goto Out;
   1678 	}
   1679 
   1680 	idx = sqlite3_bind_parameter_index(stmt, ":id");
   1681 	rc = sqlite3_bind_int64(stmt, idx, mandb_rowid);
   1682 	if (rc != SQLITE_OK) {
   1683 		sqlite3_finalize(stmt);
   1684 		goto Out;
   1685 	}
   1686 
   1687 	rc = sqlite3_step(stmt);
   1688 	sqlite3_finalize(stmt);
   1689 	if (rc == SQLITE_CONSTRAINT_UNIQUE) {
   1690 		/* The *most* probable reason for reaching here is that
   1691 		 * the UNIQUE contraint on the file column of the mandb_meta
   1692 		 * table was violated.
   1693 		 * This can happen when a file was updated/modified.
   1694 		 * To fix this we need to do two things:
   1695 		 * 1. Delete the row for the older version of this file
   1696 		 *    from mandb table.
   1697 		 * 2. Run an UPDATE query to update the row for this file
   1698 		 *    in the mandb_meta table.
   1699 		 */
   1700 		warnx("Trying to update index for %s", rec->file_path);
   1701 		char *sql = sqlite3_mprintf("DELETE FROM mandb "
   1702 					    "WHERE rowid = (SELECT id"
   1703 					    "  FROM mandb_meta"
   1704 					    "  WHERE file = %Q)",
   1705 					    rec->file_path);
   1706 		sqlite3_exec(db, sql, NULL, NULL, &errmsg);
   1707 		sqlite3_free(sql);
   1708 		if (errmsg != NULL) {
   1709 			if (mflags.verbosity)
   1710 				warnx("%s", errmsg);
   1711 			free(errmsg);
   1712 		}
   1713 		sqlstr = "UPDATE mandb_meta SET device = :device,"
   1714 			 " inode = :inode, mtime = :mtime, id = :id,"
   1715 			 " md5_hash = :md5 WHERE file = :file";
   1716 		rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
   1717 		if (rc != SQLITE_OK) {
   1718 			if (mflags.verbosity)
   1719 				warnx("Update failed with error: %s",
   1720 			    sqlite3_errmsg(db));
   1721 			close_db(db);
   1722 			cleanup(rec);
   1723 			errx(EXIT_FAILURE,
   1724 			    "Consider running makemandb with -f option");
   1725 		}
   1726 
   1727 		idx = sqlite3_bind_parameter_index(stmt, ":device");
   1728 		sqlite3_bind_int64(stmt, idx, rec->device);
   1729 		idx = sqlite3_bind_parameter_index(stmt, ":inode");
   1730 		sqlite3_bind_int64(stmt, idx, rec->inode);
   1731 		idx = sqlite3_bind_parameter_index(stmt, ":mtime");
   1732 		sqlite3_bind_int64(stmt, idx, rec->mtime);
   1733 		idx = sqlite3_bind_parameter_index(stmt, ":id");
   1734 		sqlite3_bind_int64(stmt, idx, mandb_rowid);
   1735 		idx = sqlite3_bind_parameter_index(stmt, ":md5");
   1736 		sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
   1737 		idx = sqlite3_bind_parameter_index(stmt, ":file");
   1738 		sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
   1739 		rc = sqlite3_step(stmt);
   1740 		sqlite3_finalize(stmt);
   1741 
   1742 		if (rc != SQLITE_DONE) {
   1743 			if (mflags.verbosity)
   1744 				warnx("%s", sqlite3_errmsg(db));
   1745 			close_db(db);
   1746 			cleanup(rec);
   1747 			errx(EXIT_FAILURE,
   1748 			    "Consider running makemandb with -f option");
   1749 		}
   1750 	} else if (rc != SQLITE_DONE) {
   1751 		/* Otherwise make this error fatal */
   1752 		warnx("Failed at %s\n%s", rec->file_path, sqlite3_errmsg(db));
   1753 		cleanup(rec);
   1754 		close_db(db);
   1755 		exit(EXIT_FAILURE);
   1756 	}
   1757 
   1758 /*------------------------ Populate the mandb_links table---------------------*/
   1759 	char *str = NULL;
   1760 	char *links;
   1761 	if (rec->links && strlen(rec->links)) {
   1762 		links = rec->links;
   1763 		for(ln = strtok(links, " "); ln; ln = strtok(NULL, " ")) {
   1764 			if (ln[0] == ',')
   1765 				ln++;
   1766 			if(ln[strlen(ln) - 1] == ',')
   1767 				ln[strlen(ln) - 1] = 0;
   1768 
   1769 			str = sqlite3_mprintf("INSERT INTO mandb_links"
   1770 					      " VALUES (%Q, %Q, %Q, %Q, %Q, %Q)",
   1771 					      ln, rec->name, rec->section,
   1772 					      rec->machine, rec->md5_hash, rec->name_desc);
   1773 			sqlite3_exec(db, str, NULL, NULL, &errmsg);
   1774 			sqlite3_free(str);
   1775 			if (errmsg != NULL) {
   1776 				warnx("%s", errmsg);
   1777 				cleanup(rec);
   1778 				free(errmsg);
   1779 				return -1;
   1780 			}
   1781 		}
   1782 	}
   1783 
   1784 	cleanup(rec);
   1785 	return 0;
   1786 
   1787   Out:
   1788 	if (mflags.verbosity)
   1789 		warnx("%s", sqlite3_errmsg(db));
   1790 	cleanup(rec);
   1791 	return -1;
   1792 }
   1793 
   1794 /*
   1795  * check_md5--
   1796  *  Generates the md5 hash of the file and checks if it already doesn't exist
   1797  *  in the table.
   1798  *  This function is being used to avoid hardlinks.
   1799  *  On successful completion it will also set the value of the fourth parameter
   1800  *  to the md5 hash of the file (computed previously). It is the responsibility
   1801  *  of the caller to free this buffer.
   1802  *  Return values:
   1803  *  -1: If an error occurs somewhere and sets the md5 return buffer to NULL.
   1804  *  0: If the md5 hash does not exist in the table.
   1805  *  1: If the hash exists in the database.
   1806  */
   1807 static int
   1808 check_md5(const char *file, sqlite3 *db, char **md5, void *buf, size_t buflen)
   1809 {
   1810 	int rc = 0;
   1811 	int idx = -1;
   1812 	char *sqlstr = NULL;
   1813 	char *mymd5;
   1814 	sqlite3_stmt *stmt = NULL;
   1815 	*md5 = NULL;
   1816 
   1817 	assert(file != NULL);
   1818 	if ((mymd5 = MD5Data(buf, buflen, NULL)) == NULL) {
   1819 		if (mflags.verbosity)
   1820 			warn("md5 failed: %s", file);
   1821 		return -1;
   1822 	}
   1823 
   1824 	easprintf(&sqlstr, "SELECT * FROM mandb_meta WHERE md5_hash = :md5_hash");
   1825 	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
   1826 	if (rc != SQLITE_OK) {
   1827 		free(sqlstr);
   1828 		free(mymd5);
   1829 		return -1;
   1830 	}
   1831 
   1832 	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
   1833 	rc = sqlite3_bind_text(stmt, idx, mymd5, -1, NULL);
   1834 	if (rc != SQLITE_OK) {
   1835 		if (mflags.verbosity)
   1836 			warnx("%s", sqlite3_errmsg(db));
   1837 		sqlite3_finalize(stmt);
   1838 		free(sqlstr);
   1839 		free(mymd5);
   1840 		return -1;
   1841 	}
   1842 
   1843 	*md5 = mymd5;
   1844 	if (sqlite3_step(stmt) == SQLITE_ROW) {
   1845 		sqlite3_finalize(stmt);
   1846 		free(sqlstr);
   1847 		return 0;
   1848 	}
   1849 
   1850 	sqlite3_finalize(stmt);
   1851 	free(sqlstr);
   1852 	return 1;
   1853 }
   1854 
   1855 /* Optimize the index for faster search */
   1856 static void
   1857 optimize(sqlite3 *db)
   1858 {
   1859 	const char *sqlstr;
   1860 	char *errmsg = NULL;
   1861 
   1862 	if (mflags.verbosity == 2)
   1863 		printf("Optimizing the database index\n");
   1864 	sqlstr = "INSERT INTO mandb(mandb) VALUES (\'optimize\');"
   1865 		 "VACUUM";
   1866 	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
   1867 	if (errmsg != NULL) {
   1868 		if (mflags.verbosity)
   1869 			warnx("%s", errmsg);
   1870 		free(errmsg);
   1871 		return;
   1872 	}
   1873 }
   1874 
   1875 /*
   1876  * cleanup --
   1877  *  cleans up the global buffers
   1878  */
   1879 static void
   1880 cleanup(mandb_rec *rec)
   1881 {
   1882 	rec->desc.offset = 0;
   1883 	rec->lib.offset = 0;
   1884 	rec->return_vals.offset = 0;
   1885 	rec->env.offset = 0;
   1886 	rec->exit_status.offset = 0;
   1887 	rec->diagnostics.offset = 0;
   1888 	rec->errors.offset = 0;
   1889 	rec->files.offset = 0;
   1890 
   1891 	free(rec->machine);
   1892 	rec->machine = NULL;
   1893 
   1894 	free(rec->links);
   1895 	rec->links = NULL;
   1896 
   1897 	free(rec->file_path);
   1898 	rec->file_path = NULL;
   1899 
   1900 	free(rec->name);
   1901 	rec->name = NULL;
   1902 
   1903 	free(rec->name_desc);
   1904 	rec->name_desc = NULL;
   1905 
   1906 	free(rec->md5_hash);
   1907 	rec->md5_hash = NULL;
   1908 
   1909 	free(rec->section);
   1910 	rec->section = NULL;
   1911 }
   1912 
   1913 /*
   1914  * init_secbuffs--
   1915  *  Sets the value of buflen for all the sec_buff field of rec. And then
   1916  *  allocate memory to each sec_buff member of rec.
   1917  */
   1918 static void
   1919 init_secbuffs(mandb_rec *rec)
   1920 {
   1921 	/*
   1922 	 * Some sec_buff might need more memory, for example desc,
   1923 	 * which stores the data of the DESCRIPTION section,
   1924 	 * while some might need very small amount of memory.
   1925 	 * Therefore explicitly setting the value of buflen field for
   1926 	 * each sec_buff.
   1927 	 */
   1928 	rec->desc.buflen = 10 * BUFLEN;
   1929 	rec->desc.data = emalloc(rec->desc.buflen);
   1930 	rec->desc.offset = 0;
   1931 
   1932 	rec->lib.buflen = BUFLEN / 2;
   1933 	rec->lib.data = emalloc(rec->lib.buflen);
   1934 	rec->lib.offset = 0;
   1935 
   1936 	rec->return_vals.buflen = BUFLEN;
   1937 	rec->return_vals.data = emalloc(rec->return_vals.buflen);
   1938 	rec->return_vals.offset = 0;
   1939 
   1940 	rec->exit_status.buflen = BUFLEN;
   1941 	rec->exit_status.data = emalloc(rec->exit_status.buflen);
   1942 	rec->exit_status.offset = 0;
   1943 
   1944 	rec->env.buflen = BUFLEN;
   1945 	rec->env.data = emalloc(rec->env.buflen);
   1946 	rec->env.offset = 0;
   1947 
   1948 	rec->files.buflen = BUFLEN;
   1949 	rec->files.data = emalloc(rec->files.buflen);
   1950 	rec->files.offset = 0;
   1951 
   1952 	rec->diagnostics.buflen = BUFLEN;
   1953 	rec->diagnostics.data = emalloc(rec->diagnostics.buflen);
   1954 	rec->diagnostics.offset = 0;
   1955 
   1956 	rec->errors.buflen = BUFLEN;
   1957 	rec->errors.data = emalloc(rec->errors.buflen);
   1958 	rec->errors.offset = 0;
   1959 }
   1960 
   1961 /*
   1962  * free_secbuffs--
   1963  *  This function should be called at the end, when all the pages have been
   1964  *  parsed.
   1965  *  It frees the memory allocated to sec_buffs by init_secbuffs in the starting.
   1966  */
   1967 static void
   1968 free_secbuffs(mandb_rec *rec)
   1969 {
   1970 	free(rec->desc.data);
   1971 	free(rec->lib.data);
   1972 	free(rec->return_vals.data);
   1973 	free(rec->exit_status.data);
   1974 	free(rec->env.data);
   1975 	free(rec->files.data);
   1976 	free(rec->diagnostics.data);
   1977 	free(rec->errors.data);
   1978 }
   1979 
   1980 static void
   1981 replace_hyph(char *str)
   1982 {
   1983 	char *iter = str;
   1984 	while ((iter = strchr(iter, ASCII_HYPH)) != NULL)
   1985 		*iter = '-';
   1986 
   1987 	iter = str;
   1988 	while ((iter = strchr(iter, ASCII_NBRSP)) != NULL)
   1989 		*iter = '-';
   1990 }
   1991 
   1992 static char *
   1993 parse_escape(const char *str)
   1994 {
   1995 	const char *backslash, *last_backslash;
   1996 	char *result, *iter;
   1997 	size_t len;
   1998 
   1999 	assert(str);
   2000 
   2001 	last_backslash = str;
   2002 	backslash = strchr(str, '\\');
   2003 	if (backslash == NULL) {
   2004 		result = estrdup(str);
   2005 		replace_hyph(result);
   2006 		return result;
   2007 	}
   2008 
   2009 	result = emalloc(strlen(str) + 1);
   2010 	iter = result;
   2011 
   2012 	do {
   2013 		len = backslash - last_backslash;
   2014 		memcpy(iter, last_backslash, len);
   2015 		iter += len;
   2016 		if (backslash[1] == '-' || backslash[1] == ' ') {
   2017 			*iter++ = backslash[1];
   2018 			last_backslash = backslash + 2;
   2019 			backslash = strchr(last_backslash, '\\');
   2020 		} else {
   2021 			++backslash;
   2022 			mandoc_escape(&backslash, NULL, NULL);
   2023 			last_backslash = backslash;
   2024 			if (backslash == NULL)
   2025 				break;
   2026 			backslash = strchr(last_backslash, '\\');
   2027 		}
   2028 	} while (backslash != NULL);
   2029 	if (last_backslash != NULL)
   2030 		strcpy(iter, last_backslash);
   2031 
   2032 	replace_hyph(result);
   2033 	return result;
   2034 }
   2035 
   2036 /*
   2037  * append--
   2038  *  Concatenates a space and src at the end of sbuff->data (much like concat in
   2039  *  apropos-utils.c).
   2040  *  Rather than reallocating space for writing data, it uses the value of the
   2041  *  offset field of sec_buff to write new data at the free space left in the
   2042  *  buffer.
   2043  *  In case the size of the data to be appended exceeds the number of bytes left
   2044  *  in the buffer, it reallocates buflen number of bytes and then continues.
   2045  *  Value of offset field should be adjusted as new data is written.
   2046  *
   2047  *  NOTE: This function does not write the null byte at the end of the buffers,
   2048  *  write a null byte at the position pointed to by offset before inserting data
   2049  *  in the db.
   2050  */
   2051 static void
   2052 append(secbuff *sbuff, const char *src)
   2053 {
   2054 	short flag = 0;
   2055 	size_t srclen, newlen;
   2056 	char *temp;
   2057 
   2058 	assert(src != NULL);
   2059 	temp = parse_escape(src);
   2060 	srclen = strlen(temp);
   2061 
   2062 	if (sbuff->data == NULL) {
   2063 		sbuff->data = emalloc(sbuff->buflen);
   2064 		sbuff->offset = 0;
   2065 	}
   2066 
   2067 	newlen = sbuff->offset + srclen + 2;
   2068 	if (newlen >= sbuff->buflen) {
   2069 		while (sbuff->buflen < newlen)
   2070 			sbuff->buflen += sbuff->buflen;
   2071 		sbuff->data = erealloc(sbuff->data, sbuff->buflen);
   2072 		flag = 1;
   2073 	}
   2074 
   2075 	/* Append a space at the end of the buffer. */
   2076 	if (sbuff->offset || flag)
   2077 		sbuff->data[sbuff->offset++] = ' ';
   2078 	/* Now, copy src at the end of the buffer. */
   2079 	memcpy(sbuff->data + sbuff->offset, temp, srclen);
   2080 	sbuff->offset += srclen;
   2081 	free(temp);
   2082 }
   2083 
   2084 static void
   2085 usage(void)
   2086 {
   2087 	fprintf(stderr, "Usage: %s [-floQqv] [-C path]\n", getprogname());
   2088 	exit(1);
   2089 }
   2090