Home | History | Annotate | Line # | Download | only in makemandb
makemandb.c revision 1.58
      1 /*	$NetBSD: makemandb.c,v 1.58 2019/03/11 00:14:44 christos Exp $	*/
      2 /*
      3  * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay (at) gmail.com>
      4  * Copyright (c) 2011 Kristaps Dzonsons <kristaps (at) bsd.lv>
      5  *
      6  * Permission to use, copy, modify, and distribute this software for any
      7  * purpose with or without fee is hereby granted, provided that the above
      8  * copyright notice and this permission notice appear in all copies.
      9  *
     10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
     11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
     12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
     13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
     14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
     15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
     16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
     17  */
     18 
     19 #include <sys/cdefs.h>
     20 __RCSID("$NetBSD: makemandb.c,v 1.58 2019/03/11 00:14:44 christos Exp $");
     21 
     22 #include <sys/stat.h>
     23 #include <sys/types.h>
     24 
     25 #include <assert.h>
     26 #include <dirent.h>
     27 #include <err.h>
     28 #include <archive.h>
     29 #include <libgen.h>
     30 #include <md5.h>
     31 #include <stdio.h>
     32 #include <stdlib.h>
     33 #include <string.h>
     34 #include <unistd.h>
     35 #include <util.h>
     36 
     37 #include "apropos-utils.h"
     38 #include "dist/man.h"
     39 #include "dist/mandoc.h"
     40 #include "dist/mdoc.h"
     41 #include "dist/roff.h"
     42 #include "dist/roff_int.h"
     43 #include "dist/mandoc_parse.h"
     44 
     45 #define BUFLEN 1024
     46 #define MDOC 0	//If the page is of mdoc(7) type
     47 #define MAN 1	//If the page  is of man(7) type
     48 
     49 /*
     50  * A data structure for holding section specific data.
     51  */
     52 typedef struct secbuff {
     53 	char *data;
     54 	size_t buflen;	//Total length of buffer allocated initially
     55 	size_t offset;	// Current offset in the buffer.
     56 } secbuff;
     57 
     58 typedef struct makemandb_flags {
     59 	int optimize;
     60 	int limit;	// limit the indexing to only NAME section
     61 	int recreate;	// Database was created from scratch
     62 	int verbosity;	// 0: quiet, 1: default, 2: verbose
     63 } makemandb_flags;
     64 
     65 typedef struct roff_mandb_rec {
     66 	/* Fields for mandb table */
     67 	char *name;	// for storing the name of the man page
     68 	char *name_desc; // for storing the one line description (.Nd)
     69 	secbuff desc; // for storing the DESCRIPTION section
     70 	secbuff lib; // for the LIBRARY section
     71 	secbuff return_vals; // RETURN VALUES
     72 	secbuff env; // ENVIRONMENT
     73 	secbuff files; // FILES
     74 	secbuff exit_status; // EXIT STATUS
     75 	secbuff diagnostics; // DIAGNOSTICS
     76 	secbuff errors; // ERRORS
     77 	char *section;
     78 
     79 	int xr_found; // To track whether a .Xr was seen when parsing a section
     80 
     81 	/* Fields for mandb_meta table */
     82 	char *md5_hash;
     83 	dev_t device;
     84 	ino_t inode;
     85 	time_t mtime;
     86 
     87 	/* Fields for mandb_links table */
     88 	char *machine;
     89 	char *links; //all the links to a page in a space separated form
     90 	char *file_path;
     91 
     92 	/* Non-db fields */
     93 	int page_type; //Indicates the type of page: mdoc or man
     94 } mandb_rec;
     95 
     96 typedef	void (*proff_nf)(const struct roff_node *n, mandb_rec *);
     97 
     98 static void append(secbuff *sbuff, const char *src);
     99 static void init_secbuffs(mandb_rec *);
    100 static void free_secbuffs(mandb_rec *);
    101 static int check_md5(const char *, sqlite3 *, char **, void *, size_t);
    102 static void cleanup(mandb_rec *);
    103 static void set_section(const struct roff_meta *, mandb_rec *);
    104 static void set_machine(const struct roff_meta *, mandb_rec *);
    105 static int insert_into_db(sqlite3 *, mandb_rec *);
    106 static	void begin_parse(const char *, struct mparse *, mandb_rec *, int);
    107 static void proff_node(const struct roff_node *, mandb_rec *,
    108 		       struct roff_meta *, const proff_nf *);
    109 static void pmdoc_Nm(const struct roff_node *, mandb_rec *);
    110 static void pmdoc_Nd(const struct roff_node *, mandb_rec *);
    111 static void pmdoc_Sh(const struct roff_node *, mandb_rec *);
    112 static void mdoc_parse_Sh(const struct roff_node *, mandb_rec *);
    113 static void pmdoc_Xr(const struct roff_node *, mandb_rec *);
    114 static void pmdoc_Pp(const struct roff_node *, mandb_rec *);
    115 static void pmdoc_macro_handler(const struct roff_node *, mandb_rec *, int);
    116 static void pman_parse_node(const struct roff_node *, secbuff *);
    117 static void pman_parse_name(const struct roff_node *, mandb_rec *);
    118 static void pman_sh(const struct roff_node *, mandb_rec *);
    119 static void pman_block(const struct roff_node *, mandb_rec *);
    120 static void traversedir(const char *, const char *, sqlite3 *, struct mparse *);
    121 static void mdoc_parse_section(enum roff_sec, const char *, mandb_rec *);
    122 static void man_parse_section(enum man_sec, const struct roff_node *, mandb_rec *);
    123 static void build_file_cache(sqlite3 *, const char *, const char *,
    124 			     struct stat *);
    125 static void update_db(sqlite3 *, struct mparse *, mandb_rec *);
    126 __dead static void usage(void);
    127 static void optimize(sqlite3 *);
    128 static char *parse_escape(const char *);
    129 static void replace_hyph(char *);
    130 static makemandb_flags mflags = { .verbosity = 1 };
    131 
    132 static	const proff_nf mdocs[MDOC_MAX - MDOC_Dd] = {
    133 	NULL, /* Dd */
    134 	NULL, /* Dt */
    135 	NULL, /* Os */
    136 	pmdoc_Sh, /* Sh */
    137 	NULL, /* Ss */
    138 	pmdoc_Pp, /* Pp */
    139 	NULL, /* D1 */
    140 	NULL, /* Dl */
    141 	NULL, /* Bd */
    142 	NULL, /* Ed */
    143 	NULL, /* Bl */
    144 	NULL, /* El */
    145 	NULL, /* It */
    146 	NULL, /* Ad */
    147 	NULL, /* An */
    148 	NULL, /* Ap */
    149 	NULL, /* Ar */
    150 	NULL, /* Cd */
    151 	NULL, /* Cm */
    152 	NULL, /* Dv */
    153 	NULL, /* Er */
    154 	NULL, /* Ev */
    155 	NULL, /* Ex */
    156 	NULL, /* Fa */
    157 	NULL, /* Fd */
    158 	NULL, /* Fl */
    159 	NULL, /* Fn */
    160 	NULL, /* Ft */
    161 	NULL, /* Ic */
    162 	NULL, /* In */
    163 	NULL, /* Li */
    164 	pmdoc_Nd, /* Nd */
    165 	pmdoc_Nm, /* Nm */
    166 	NULL, /* Op */
    167 	NULL, /* Ot */
    168 	NULL, /* Pa */
    169 	NULL, /* Rv */
    170 	NULL, /* St */
    171 	NULL, /* Va */
    172 	NULL, /* Vt */
    173 	pmdoc_Xr, /* Xr */
    174 	NULL, /* %A */
    175 	NULL, /* %B */
    176 	NULL, /* %D */
    177 	NULL, /* %I */
    178 	NULL, /* %J */
    179 	NULL, /* %N */
    180 	NULL, /* %O */
    181 	NULL, /* %P */
    182 	NULL, /* %R */
    183 	NULL, /* %T */
    184 	NULL, /* %V */
    185 	NULL, /* Ac */
    186 	NULL, /* Ao */
    187 	NULL, /* Aq */
    188 	NULL, /* At */
    189 	NULL, /* Bc */
    190 	NULL, /* Bf */
    191 	NULL, /* Bo */
    192 	NULL, /* Bq */
    193 	NULL, /* Bsx */
    194 	NULL, /* Bx */
    195 	NULL, /* Db */
    196 	NULL, /* Dc */
    197 	NULL, /* Do */
    198 	NULL, /* Dq */
    199 	NULL, /* Ec */
    200 	NULL, /* Ef */
    201 	NULL, /* Em */
    202 	NULL, /* Eo */
    203 	NULL, /* Fx */
    204 	NULL, /* Ms */
    205 	NULL, /* No */
    206 	NULL, /* Ns */
    207 	NULL, /* Nx */
    208 	NULL, /* Ox */
    209 	NULL, /* Pc */
    210 	NULL, /* Pf */
    211 	NULL, /* Po */
    212 	NULL, /* Pq */
    213 	NULL, /* Qc */
    214 	NULL, /* Ql */
    215 	NULL, /* Qo */
    216 	NULL, /* Qq */
    217 	NULL, /* Re */
    218 	NULL, /* Rs */
    219 	NULL, /* Sc */
    220 	NULL, /* So */
    221 	NULL, /* Sq */
    222 	NULL, /* Sm */
    223 	NULL, /* Sx */
    224 	NULL, /* Sy */
    225 	NULL, /* Tn */
    226 	NULL, /* Ux */
    227 	NULL, /* Xc */
    228 	NULL, /* Xo */
    229 	NULL, /* Fo */
    230 	NULL, /* Fc */
    231 	NULL, /* Oo */
    232 	NULL, /* Oc */
    233 	NULL, /* Bk */
    234 	NULL, /* Ek */
    235 	NULL, /* Bt */
    236 	NULL, /* Hf */
    237 	NULL, /* Fr */
    238 	NULL, /* Ud */
    239 	NULL, /* Lb */
    240 	NULL, /* Lp */
    241 	NULL, /* Lk */
    242 	NULL, /* Mt */
    243 	NULL, /* Brq */
    244 	NULL, /* Bro */
    245 	NULL, /* Brc */
    246 	NULL, /* %C */
    247 	NULL, /* Es */
    248 	NULL, /* En */
    249 	NULL, /* Dx */
    250 	NULL, /* %Q */
    251 	NULL, /* %U */
    252 	NULL /* Ta */
    253 };
    254 
    255 static	const proff_nf mans[MAN_MAX - MAN_TH] = {
    256 	NULL,	//TH
    257 	pman_sh, //SH
    258 	NULL,	//SS
    259 	NULL,	//TP
    260 	NULL,	//LP
    261 	NULL,	//PP
    262 	NULL,	//P
    263 	NULL,	//IP
    264 	NULL,	//HP
    265 	NULL,	//SM
    266 	NULL,	//SB
    267 	NULL,	//BI
    268 	NULL,	//IB
    269 	NULL,	//BR
    270 	NULL,	//RB
    271 	NULL,	//R
    272 	pman_block,	//B
    273 	NULL,	//I
    274 	NULL,	//IR
    275 	NULL,	//RI
    276 	NULL,	//nf
    277 	NULL,	//fi
    278 	NULL,	//RE
    279 	NULL,	//RS
    280 	NULL,	//DT
    281 	NULL,	//UC
    282 	NULL,	//PD
    283 	NULL,	//AT
    284 	NULL,	//in
    285 	NULL,	//OP
    286 	NULL,	//EX
    287 	NULL,	//EE
    288 	NULL,	//UR
    289 	NULL,	//UE
    290 	NULL,	//MT
    291 	NULL   //ME
    292 };
    293 
    294 int
    295 main(int argc, char *argv[])
    296 {
    297 	FILE *file;
    298 	const char *sqlstr, *manconf = NULL;
    299 	char *line, *command;
    300 	char *errmsg;
    301 	int ch;
    302 	struct mparse *mp;
    303 	sqlite3 *db;
    304 	ssize_t len;
    305 	size_t linesize;
    306 	struct roff_mandb_rec rec;
    307 
    308 	while ((ch = getopt(argc, argv, "C:floQqv")) != -1) {
    309 		switch (ch) {
    310 		case 'C':
    311 			manconf = optarg;
    312 			break;
    313 		case 'f':
    314 			mflags.recreate = 1;
    315 			break;
    316 		case 'l':
    317 			mflags.limit = 1;
    318 			break;
    319 		case 'o':
    320 			mflags.optimize = 1;
    321 			break;
    322 		case 'Q':
    323 			mflags.verbosity = 0;
    324 			break;
    325 		case 'q':
    326 			mflags.verbosity = 1;
    327 			break;
    328 		case 'v':
    329 			mflags.verbosity = 2;
    330 			break;
    331 		default:
    332 			usage();
    333 		}
    334 	}
    335 
    336 	memset(&rec, 0, sizeof(rec));
    337 
    338 	init_secbuffs(&rec);
    339 	mchars_alloc();
    340 	mp = mparse_alloc(MPARSE_SO | MPARSE_UTF8 | MPARSE_LATIN1 |
    341 	    MPARSE_VALIDATE, MANDOC_OS_OTHER, NULL);
    342 
    343 	if (manconf) {
    344 		char *arg;
    345 		size_t command_len = shquote(manconf, NULL, 0) + 1;
    346 		arg = emalloc(command_len);
    347 		shquote(manconf, arg, command_len);
    348 		easprintf(&command, "man -p -C %s", arg);
    349 		free(arg);
    350 	} else {
    351 		command = estrdup("man -p");
    352 		manconf = MANCONF;
    353 	}
    354 
    355 	if (mflags.recreate) {
    356 		char *dbp = get_dbpath(manconf);
    357 		/* No error here, it will fail in init_db in the same call */
    358 		if (dbp != NULL)
    359 			remove(dbp);
    360 	}
    361 
    362 	if ((db = init_db(MANDB_CREATE, manconf)) == NULL)
    363 		exit(EXIT_FAILURE);
    364 
    365 	sqlite3_exec(db, "PRAGMA synchronous = 0", NULL, NULL, 	&errmsg);
    366 	if (errmsg != NULL) {
    367 		warnx("%s", errmsg);
    368 		free(errmsg);
    369 		close_db(db);
    370 		exit(EXIT_FAILURE);
    371 	}
    372 
    373 	sqlite3_exec(db, "ATTACH DATABASE \':memory:\' AS metadb", NULL, NULL,
    374 	    &errmsg);
    375 	if (errmsg != NULL) {
    376 		warnx("%s", errmsg);
    377 		free(errmsg);
    378 		close_db(db);
    379 		exit(EXIT_FAILURE);
    380 	}
    381 
    382 
    383 	/* Call man -p to get the list of man page dirs */
    384 	if ((file = popen(command, "r")) == NULL) {
    385 		close_db(db);
    386 		err(EXIT_FAILURE, "fopen failed");
    387 	}
    388 	free(command);
    389 
    390 	/* Begin the transaction for indexing the pages	*/
    391 	sqlite3_exec(db, "BEGIN", NULL, NULL, &errmsg);
    392 	if (errmsg != NULL) {
    393 		warnx("%s", errmsg);
    394 		free(errmsg);
    395 		close_db(db);
    396 		exit(EXIT_FAILURE);
    397 	}
    398 
    399 	sqlstr = "CREATE TABLE metadb.file_cache(device, inode, mtime, parent,"
    400 		 " file PRIMARY KEY);"
    401 		 "CREATE UNIQUE INDEX metadb.index_file_cache_dev"
    402 		 " ON file_cache (device, inode)";
    403 
    404 	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
    405 	if (errmsg != NULL) {
    406 		warnx("%s", errmsg);
    407 		free(errmsg);
    408 		close_db(db);
    409 		exit(EXIT_FAILURE);
    410 	}
    411 
    412 	if (mflags.verbosity)
    413 		printf("Building temporary file cache\n");
    414 	line = NULL;
    415 	linesize = 0;
    416 	while ((len = getline(&line, &linesize, file)) != -1) {
    417 		/* Replace the new line character at the end of string with '\0' */
    418 		line[len - 1] = '\0';
    419 		char *pdir = estrdup(dirname(line));
    420 		/* Traverse the man page directories and parse the pages */
    421 		traversedir(pdir, line, db, mp);
    422 		free(pdir);
    423 	}
    424 	free(line);
    425 
    426 	if (pclose(file) == -1) {
    427 		close_db(db);
    428 		cleanup(&rec);
    429 		free_secbuffs(&rec);
    430 		err(EXIT_FAILURE, "pclose error");
    431 	}
    432 
    433 	if (mflags.verbosity)
    434 		printf("Performing index update\n");
    435 	update_db(db, mp, &rec);
    436 	mparse_free(mp);
    437 	mchars_free();
    438 	free_secbuffs(&rec);
    439 
    440 	/* Commit the transaction */
    441 	sqlite3_exec(db, "COMMIT", NULL, NULL, &errmsg);
    442 	if (errmsg != NULL) {
    443 		warnx("%s", errmsg);
    444 		free(errmsg);
    445 		close_db(db);
    446 		exit(EXIT_FAILURE);
    447 	}
    448 
    449 	if (mflags.optimize)
    450 		optimize(db);
    451 
    452 	close_db(db);
    453 	return 0;
    454 }
    455 
    456 /*
    457  * traversedir --
    458  *  Traverses the given directory recursively and passes all the man page files
    459  *  in the way to build_file_cache()
    460  */
    461 static void
    462 traversedir(const char *parent, const char *file, sqlite3 *db,
    463             struct mparse *mp)
    464 {
    465 	struct stat sb;
    466 	struct dirent *dirp;
    467 	DIR *dp;
    468 	char *buf;
    469 
    470 	if (stat(file, &sb) < 0) {
    471 		if (mflags.verbosity)
    472 			warn("stat failed: %s", file);
    473 		return;
    474 	}
    475 
    476 	/* If it is a directory, traverse it recursively */
    477 	if (S_ISDIR(sb.st_mode)) {
    478 		if ((dp = opendir(file)) == NULL) {
    479 			if (mflags.verbosity)
    480 				warn("opendir error: %s", file);
    481 			return;
    482 		}
    483 
    484 		while ((dirp = readdir(dp)) != NULL) {
    485 			/* Avoid . and .. entries in a directory */
    486 			if (dirp->d_name[0] != '.') {
    487 				easprintf(&buf, "%s/%s", file, dirp->d_name);
    488 				traversedir(parent, buf, db, mp);
    489 				free(buf);
    490 			}
    491 		}
    492 		closedir(dp);
    493 		return;
    494 	}
    495 
    496 	if (!S_ISREG(sb.st_mode))
    497 		return;
    498 
    499 	if (sb.st_size == 0) {
    500 		if (mflags.verbosity)
    501 			warnx("Empty file: %s", file);
    502 		return;
    503 	}
    504 	build_file_cache(db, parent, file, &sb);
    505 }
    506 
    507 /* build_file_cache --
    508  *   This function generates an md5 hash of the file passed as its 2nd parameter
    509  *   and stores it in a temporary table file_cache along with the full file path.
    510  *   This is done to support incremental updation of the database.
    511  *   The temporary table file_cache is dropped thereafter in the function
    512  *   update_db(), once the database has been updated.
    513  */
    514 static void
    515 build_file_cache(sqlite3 *db, const char *parent, const char *file,
    516 		 struct stat *sb)
    517 {
    518 	const char *sqlstr;
    519 	sqlite3_stmt *stmt = NULL;
    520 	int rc, idx;
    521 	assert(file != NULL);
    522 	dev_t device_cache = sb->st_dev;
    523 	ino_t inode_cache = sb->st_ino;
    524 	time_t mtime_cache = sb->st_mtime;
    525 
    526 	sqlstr = "INSERT INTO metadb.file_cache VALUES (:device, :inode,"
    527 		 " :mtime, :parent, :file)";
    528 	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
    529 	if (rc != SQLITE_OK) {
    530 		if (mflags.verbosity)
    531 			warnx("%s", sqlite3_errmsg(db));
    532 		return;
    533 	}
    534 
    535 	idx = sqlite3_bind_parameter_index(stmt, ":device");
    536 	rc = sqlite3_bind_int64(stmt, idx, device_cache);
    537 	if (rc != SQLITE_OK) {
    538 		if (mflags.verbosity)
    539 			warnx("%s", sqlite3_errmsg(db));
    540 		sqlite3_finalize(stmt);
    541 		return;
    542 	}
    543 
    544 	idx = sqlite3_bind_parameter_index(stmt, ":inode");
    545 	rc = sqlite3_bind_int64(stmt, idx, inode_cache);
    546 	if (rc != SQLITE_OK) {
    547 		if (mflags.verbosity)
    548 			warnx("%s", sqlite3_errmsg(db));
    549 		sqlite3_finalize(stmt);
    550 		return;
    551 	}
    552 
    553 	idx = sqlite3_bind_parameter_index(stmt, ":mtime");
    554 	rc = sqlite3_bind_int64(stmt, idx, mtime_cache);
    555 	if (rc != SQLITE_OK) {
    556 		if (mflags.verbosity)
    557 			warnx("%s", sqlite3_errmsg(db));
    558 		sqlite3_finalize(stmt);
    559 		return;
    560 	}
    561 
    562 	idx = sqlite3_bind_parameter_index(stmt, ":parent");
    563 	rc = sqlite3_bind_text(stmt, idx, parent, -1, NULL);
    564 	if (rc != SQLITE_OK) {
    565 		if (mflags.verbosity)
    566 			warnx("%s", sqlite3_errmsg(db));
    567 		sqlite3_finalize(stmt);
    568 		return;
    569 	}
    570 
    571 	idx = sqlite3_bind_parameter_index(stmt, ":file");
    572 	rc = sqlite3_bind_text(stmt, idx, file, -1, NULL);
    573 	if (rc != SQLITE_OK) {
    574 		if (mflags.verbosity)
    575 			warnx("%s", sqlite3_errmsg(db));
    576 		sqlite3_finalize(stmt);
    577 		return;
    578 	}
    579 
    580 	sqlite3_step(stmt);
    581 	sqlite3_finalize(stmt);
    582 }
    583 
    584 /* read_and_decompress --
    585  *	Reads the given file into memory. If it is compressed, decompress
    586  *	it before returning to the caller.
    587  */
    588 static int
    589 read_and_decompress(const char *file, void **bufp, size_t *len)
    590 {
    591 	size_t off;
    592 	ssize_t r;
    593 	struct archive *a;
    594 	struct archive_entry *ae;
    595 	char *buf;
    596 
    597 	if ((a = archive_read_new()) == NULL)
    598 		errx(EXIT_FAILURE, "memory allocation failed");
    599 
    600 	*bufp = NULL;
    601 	if (archive_read_support_filter_all(a) != ARCHIVE_OK ||
    602 	    archive_read_support_format_raw(a) != ARCHIVE_OK ||
    603 	    archive_read_open_filename(a, file, 65536) != ARCHIVE_OK ||
    604 	    archive_read_next_header(a, &ae) != ARCHIVE_OK)
    605 		goto archive_error;
    606 	*len = 65536;
    607 	buf = emalloc(*len);
    608 	off = 0;
    609 	for (;;) {
    610 		r = archive_read_data(a, buf + off, *len - off);
    611 		if (r == ARCHIVE_OK) {
    612 			archive_read_free(a);
    613 			*bufp = buf;
    614 			*len = off;
    615 			return 0;
    616 		}
    617 		if (r <= 0) {
    618 			free(buf);
    619 			break;
    620 		}
    621 		off += r;
    622 		if (off == *len) {
    623 			*len *= 2;
    624 			if (*len < off) {
    625 				if (mflags.verbosity)
    626 					warnx("File too large: %s", file);
    627 				free(buf);
    628 				archive_read_free(a);
    629 				return -1;
    630 			}
    631 			buf = erealloc(buf, *len);
    632 		}
    633 	}
    634 
    635 archive_error:
    636 	warnx("Error while reading `%s': %s", file, archive_error_string(a));
    637 	archive_read_free(a);
    638 	return -1;
    639 }
    640 
    641 static void
    642 update_existing_entry(sqlite3 *db, const char *file, const char *hash,
    643     mandb_rec *rec, int *new_count, int *link_count, int *err_count)
    644 {
    645 	int update_count, rc, idx;
    646 	const char *inner_sqlstr;
    647 	sqlite3_stmt *inner_stmt;
    648 
    649 	update_count = sqlite3_total_changes(db);
    650 	inner_sqlstr = "UPDATE mandb_meta SET device = :device,"
    651 		       " inode = :inode, mtime = :mtime WHERE"
    652 		       " md5_hash = :md5 AND file = :file AND"
    653 		       " (device <> :device2 OR inode <> "
    654 		       "  :inode2 OR mtime <> :mtime2)";
    655 	rc = sqlite3_prepare_v2(db, inner_sqlstr, -1, &inner_stmt, NULL);
    656 	if (rc != SQLITE_OK) {
    657 		if (mflags.verbosity)
    658 			warnx("%s", sqlite3_errmsg(db));
    659 		return;
    660 	}
    661 	idx = sqlite3_bind_parameter_index(inner_stmt, ":device");
    662 	sqlite3_bind_int64(inner_stmt, idx, rec->device);
    663 	idx = sqlite3_bind_parameter_index(inner_stmt, ":inode");
    664 	sqlite3_bind_int64(inner_stmt, idx, rec->inode);
    665 	idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime");
    666 	sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
    667 	idx = sqlite3_bind_parameter_index(inner_stmt, ":md5");
    668 	sqlite3_bind_text(inner_stmt, idx, hash, -1, NULL);
    669 	idx = sqlite3_bind_parameter_index(inner_stmt, ":file");
    670 	sqlite3_bind_text(inner_stmt, idx, file, -1, NULL);
    671 	idx = sqlite3_bind_parameter_index(inner_stmt, ":device2");
    672 	sqlite3_bind_int64(inner_stmt, idx, rec->device);
    673 	idx = sqlite3_bind_parameter_index(inner_stmt, ":inode2");
    674 	sqlite3_bind_int64(inner_stmt, idx, rec->inode);
    675 	idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime2");
    676 	sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
    677 
    678 	rc = sqlite3_step(inner_stmt);
    679 	if (rc == SQLITE_DONE) {
    680 		/* Check if an update has been performed. */
    681 		if (update_count != sqlite3_total_changes(db)) {
    682 			if (mflags.verbosity == 2)
    683 				printf("Updated %s\n", file);
    684 			(*new_count)++;
    685 		} else {
    686 			/* Otherwise it was a hardlink. */
    687 			(*link_count)++;
    688 		}
    689 	} else {
    690 		if (mflags.verbosity == 2)
    691 			warnx("Could not update the meta data for %s", file);
    692 		(*err_count)++;
    693 	}
    694 	sqlite3_finalize(inner_stmt);
    695 }
    696 
    697 /* update_db --
    698  *	Does an incremental updation of the database by checking the file_cache.
    699  *	It parses and adds the pages which are present in file_cache,
    700  *	but not in the database.
    701  *	It also removes the pages which are present in the databse,
    702  *	but not in the file_cache.
    703  */
    704 static void
    705 update_db(sqlite3 *db, struct mparse *mp, mandb_rec *rec)
    706 {
    707 	const char *sqlstr;
    708 	sqlite3_stmt *stmt = NULL;
    709 	char *file;
    710 	char *parent;
    711 	char *errmsg = NULL;
    712 	char *md5sum;
    713 	void *buf;
    714 	size_t buflen;
    715 	struct sql_row {
    716 		struct sql_row *next;
    717 		dev_t device;
    718 		ino_t inode;
    719 		time_t mtime;
    720 		char *parent;
    721 		char *file;
    722 	} *rows, *row;
    723 	int new_count = 0;	/* Counter for newly indexed/updated pages */
    724 	int total_count = 0;	/* Counter for total number of pages */
    725 	int err_count = 0;	/* Counter for number of failed pages */
    726 	int link_count = 0;	/* Counter for number of hard/sym links */
    727 	int md5_status;
    728 	int rc;
    729 
    730 	sqlstr = "SELECT device, inode, mtime, parent, file"
    731 	         " FROM metadb.file_cache fc"
    732 	         " WHERE NOT EXISTS(SELECT 1 FROM mandb_meta WHERE"
    733 	         "  device = fc.device AND inode = fc.inode AND "
    734 	         "  mtime = fc.mtime AND file = fc.file)";
    735 
    736 	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
    737 	if (rc != SQLITE_OK) {
    738 		if (mflags.verbosity)
    739 			warnx("%s", sqlite3_errmsg(db));
    740 		close_db(db);
    741 		errx(EXIT_FAILURE, "Could not query file cache");
    742 	}
    743 
    744 	buf = NULL;
    745 	rows = NULL;
    746 	while (sqlite3_step(stmt) == SQLITE_ROW) {
    747 		row = emalloc(sizeof(struct sql_row));
    748 		row->device = sqlite3_column_int64(stmt, 0);
    749 		row->inode = sqlite3_column_int64(stmt, 1);
    750 		row->mtime = sqlite3_column_int64(stmt, 2);
    751 		row->parent = estrdup((const char *) sqlite3_column_text(stmt, 3));
    752 		row->file = estrdup((const char *) sqlite3_column_text(stmt, 4));
    753 		row->next = rows;
    754 		rows = row;
    755 		total_count++;
    756 	}
    757 	sqlite3_finalize(stmt);
    758 
    759 	for ( ; rows != NULL; free(parent), free(file), free(buf)) {
    760 		int fd;
    761 
    762 		row = rows;
    763 		rows = rows->next;
    764 
    765 		rec->device = row->device;
    766 		rec->inode = row->inode;
    767 		rec->mtime = row->mtime;
    768 		parent = row->parent;
    769 		file = row->file;
    770 		free(row);
    771 
    772 		// XXX: reading twice!
    773 		if (read_and_decompress(file, &buf, &buflen)) {
    774 			err_count++;
    775 			continue;
    776 		}
    777 		if ((fd = mparse_open(mp, file)) == -1) {
    778 			err_count++;
    779 			continue;
    780 		}
    781 
    782 		md5_status = check_md5(file, db, &md5sum, buf, buflen);
    783 		assert(md5sum != NULL);
    784 		if (md5_status == -1) {
    785 			if (mflags.verbosity)
    786 				warnx("An error occurred in checking md5 value"
    787 			      " for file %s", file);
    788 			err_count++;
    789 			close(fd);
    790 			continue;
    791 		}
    792 
    793 		if (md5_status == 0) {
    794 			/*
    795 			 * The MD5 hash is already present in the database,
    796 			 * so simply update the metadata.
    797 			 */
    798 			update_existing_entry(db, file, md5sum, rec,
    799 			    &new_count, &link_count, &err_count);
    800 			free(md5sum);
    801 			close(fd);
    802 			continue;
    803 		}
    804 
    805 		if (md5_status == 1) {
    806 			/*
    807 			 * The MD5 hash was not present in the database.
    808 			 * This means is either a new file or an updated file.
    809 			 * We should go ahead with parsing.
    810 			 */
    811 			if (chdir(parent) == -1) {
    812 				if (mflags.verbosity)
    813 					warn("chdir failed for `%s', could "
    814 					    "not index `%s'", parent, file);
    815 				err_count++;
    816 				free(md5sum);
    817 				close(fd);
    818 				continue;
    819 			}
    820 
    821 			if (mflags.verbosity == 2)
    822 				printf("Parsing: %s\n", file);
    823 			rec->md5_hash = md5sum;
    824 			rec->file_path = estrdup(file);
    825 			// file_path is freed by insert_into_db itself.
    826 			begin_parse(file, mp, rec, fd);
    827 			if (insert_into_db(db, rec) < 0) {
    828 				if (mflags.verbosity)
    829 					warnx("Error in indexing `%s'", file);
    830 				err_count++;
    831 			} else {
    832 				new_count++;
    833 			}
    834 		}
    835 		close(fd);
    836 	}
    837 
    838 	if (mflags.verbosity == 2) {
    839 		printf("Number of new or updated pages encountered: %d\n"
    840 		    "Number of hard links found: %d\n"
    841 		    "Number of pages that were successfully"
    842 		    " indexed or updated: %d\n"
    843 		    "Number of pages that could not be indexed"
    844 		    " due to errors: %d\n",
    845 		    total_count - link_count, link_count, new_count, err_count);
    846 	}
    847 
    848 	if (mflags.recreate)
    849 		return;
    850 
    851 	if (mflags.verbosity == 2)
    852 		printf("Deleting stale index entries\n");
    853 
    854 	sqlstr = "DELETE FROM mandb_meta WHERE file NOT IN"
    855 		 " (SELECT file FROM metadb.file_cache);"
    856 		 "DELETE FROM mandb_links WHERE md5_hash NOT IN"
    857 		 " (SELECT md5_hash from mandb_meta);"
    858 		 "DROP TABLE metadb.file_cache;"
    859 		 "DELETE FROM mandb WHERE rowid NOT IN"
    860 		 " (SELECT id FROM mandb_meta);";
    861 
    862 	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
    863 	if (errmsg != NULL) {
    864 		warnx("Removing old entries failed: %s", errmsg);
    865 		warnx("Please rebuild database from scratch with -f.");
    866 		free(errmsg);
    867 		return;
    868 	}
    869 }
    870 
    871 /*
    872  * begin_parse --
    873  *  parses the man page using libmandoc
    874  */
    875 static void
    876 begin_parse(const char *file, struct mparse *mp, mandb_rec *rec, int fd)
    877 {
    878 	struct roff_meta *roff;
    879 	mparse_reset(mp);
    880 
    881 	rec->xr_found = 0;
    882 
    883 	mparse_readfd(mp, fd, file);
    884 	roff = mparse_result(mp);
    885 	if (roff == NULL) {
    886 		if (mflags.verbosity == 2)
    887 			warnx("Not a roff(7) page");
    888 		return;
    889 	}
    890 
    891 	if (roff->macroset == MACROSET_MDOC) {
    892 		rec->page_type = MDOC;
    893 		proff_node(roff->first->child, rec, roff, mdocs);
    894 	} else if (roff->macroset == MACROSET_MAN) {
    895 		rec->page_type = MAN;
    896 		proff_node(roff->first->child, rec, roff, mans);
    897 	} else
    898 		warnx("Unknown macroset %d", roff->macroset);
    899 	set_machine(roff, rec);
    900 	set_section(roff, rec);
    901 }
    902 
    903 /*
    904  * set_section --
    905  *  Extracts the section number and normalizes it to only the numeric part
    906  *  (Which should be the first character of the string).
    907  */
    908 static void
    909 set_section(const struct roff_meta *rm, mandb_rec *rec)
    910 {
    911 	if (!rm)
    912 		return;
    913 	const char *s = rm->msec == NULL ? "?" : rm->msec;
    914 	easprintf(&rec->section, "%s", s);
    915 	if (rec->section[0] == '?' && mflags.verbosity == 2)
    916 		warnx("%s: Missing section number", rec->file_path);
    917 }
    918 
    919 /*
    920  * get_machine --
    921  *  Extracts the machine architecture information if available.
    922  */
    923 static void
    924 set_machine(const struct roff_meta *rm, mandb_rec *rec)
    925 {
    926 	if (rm == NULL)
    927 		return;
    928 	if (rm->arch)
    929 		rec->machine = estrdup(rm->arch);
    930 }
    931 
    932 /*
    933  * pmdoc_Nm --
    934  *  Extracts the Name of the manual page from the .Nm macro
    935  */
    936 static void
    937 pmdoc_Nm(const struct roff_node *n, mandb_rec *rec)
    938 {
    939 	if (n->sec != SEC_NAME)
    940 		return;
    941 
    942 	for (n = n->child; n; n = n->next) {
    943 		if (n->type == ROFFT_TEXT) {
    944 			char *escaped_name = parse_escape(n->string);
    945 			concat(&rec->name, escaped_name);
    946 			free(escaped_name);
    947 		}
    948 	}
    949 }
    950 
    951 /*
    952  * pmdoc_Nd --
    953  *  Extracts the one line description of the man page from the .Nd macro
    954  */
    955 static void
    956 pmdoc_Nd(const struct roff_node *n, mandb_rec *rec)
    957 {
    958 	if (n->type == ROFFT_BODY)
    959 		deroff(&rec->name_desc, n);
    960 	if (rec->name_desc)
    961 		replace_hyph(rec->name_desc);
    962 
    963 }
    964 
    965 /*
    966  * pmdoc_macro_handler--
    967  *  This function is a single point of handling all the special macros that we
    968  *  want to handle especially. For example the .Xr macro for properly parsing
    969  *  the referenced page name along with the section number, or the .Pp macro
    970  *  for adding a new line whenever we encounter it.
    971  */
    972 static void
    973 pmdoc_macro_handler(const struct roff_node *n, mandb_rec *rec, int doct)
    974 {
    975 	const struct roff_node *sn;
    976 	assert(n);
    977 
    978 	switch (doct) {
    979 	/*  Parse the man page references.
    980 	 * Basically the .Xr macros are used like:
    981 	 *  .Xr ls 1
    982  	 *  and formatted like this:
    983 	 *  ls(1)
    984 	 *  Prepare a buffer to format the data like the above example and call
    985 	 *  pmdoc_parse_section to append it.
    986 	 */
    987 	case MDOC_Xr:
    988 		n = n->child;
    989 		while (n->type != ROFFT_TEXT && n->next)
    990 			n = n->next;
    991 
    992 		if (n && n->type != ROFFT_TEXT)
    993 			return;
    994 		sn = n;
    995 		if (n->next)
    996 			n = n->next;
    997 
    998 		while (n->type != ROFFT_TEXT && n->next)
    999 			n = n->next;
   1000 
   1001 		if (n && n->type == ROFFT_TEXT) {
   1002 			char *buf;
   1003 			easprintf(&buf, "%s(%s)", sn->string, n->string);
   1004 			mdoc_parse_section(n->sec, buf, rec);
   1005 			free(buf);
   1006 		}
   1007 
   1008 		break;
   1009 
   1010 	/* Parse the .Pp macro to add a new line */
   1011 	case MDOC_Pp:
   1012 		if (n->type == ROFFT_TEXT)
   1013 			mdoc_parse_section(n->sec, "\n", rec);
   1014 		break;
   1015 	default:
   1016 		break;
   1017 	}
   1018 
   1019 }
   1020 
   1021 /*
   1022  * pmdoc_Xr, pmdoc_Pp--
   1023  *  Empty stubs.
   1024  *  The parser calls these functions each time it encounters
   1025  *  a .Xr or .Pp macro. We are parsing all the data from
   1026  *  the pmdoc_Sh function, so don't do anything here.
   1027  *  (See if else blocks in pmdoc_Sh.)
   1028  */
   1029 static void
   1030 pmdoc_Xr(const struct roff_node *n, mandb_rec *rec)
   1031 {
   1032 }
   1033 
   1034 static void
   1035 pmdoc_Pp(const struct roff_node *n, mandb_rec *rec)
   1036 {
   1037 }
   1038 
   1039 /*
   1040  * pmdoc_Sh --
   1041  *  Called when a .Sh macro is encountered and tries to parse its body
   1042  */
   1043 static void
   1044 pmdoc_Sh(const struct roff_node *n, mandb_rec *rec)
   1045 {
   1046 	if (n == NULL)
   1047 		return;
   1048 
   1049 	switch (n->sec) {
   1050 	case SEC_NAME:
   1051 	case SEC_SYNOPSIS:
   1052 	case SEC_EXAMPLES:
   1053 	case SEC_STANDARDS:
   1054 	case SEC_HISTORY:
   1055 	case SEC_AUTHORS:
   1056 	case SEC_BUGS:
   1057 		/*
   1058 		 * We don't care about text from these sections
   1059 		 */
   1060 		return;
   1061 	default:
   1062 		break;
   1063 	}
   1064 
   1065 	if (n->type == ROFFT_BLOCK)
   1066 		mdoc_parse_Sh(n->body, rec);
   1067 }
   1068 
   1069 /*
   1070  *  Called from pmdoc_Sh to parse body of a .Sh macro. It calls
   1071  *  mdoc_parse_section to append the data to the section specific buffer.
   1072  *  The .Xr macro needs special handling, thus the separate if branch for it.
   1073  */
   1074 static void
   1075 mdoc_parse_Sh(const struct roff_node *n, mandb_rec *rec)
   1076 {
   1077 	if (n == NULL || (n->type != ROFFT_TEXT && n->tok == MDOC_MAX))
   1078 		return;
   1079 	int xr_found = 0;
   1080 
   1081 	if (n->type == ROFFT_TEXT) {
   1082 		mdoc_parse_section(n->sec, n->string, rec);
   1083 	} else if (mdocs[n->tok] == pmdoc_Xr) {
   1084 		/*
   1085 		 * When encountering other inline macros,
   1086 		 * call pmdoc_macro_handler.
   1087 		 */
   1088 		pmdoc_macro_handler(n, rec, MDOC_Xr);
   1089 		xr_found = 1;
   1090 	} else if (mdocs[n->tok] == pmdoc_Pp) {
   1091 		pmdoc_macro_handler(n, rec, MDOC_Pp);
   1092 	}
   1093 
   1094 	/*
   1095 	 * If an Xr macro was encountered then the child node has
   1096 	 * already been explored by pmdoc_macro_handler.
   1097 	 */
   1098 	if (xr_found == 0)
   1099 		mdoc_parse_Sh(n->child, rec);
   1100 	mdoc_parse_Sh(n->next, rec);
   1101 }
   1102 
   1103 /*
   1104  * mdoc_parse_section--
   1105  *  Utility function for parsing sections of the mdoc type pages.
   1106  *  Takes two params:
   1107  *   1. sec is an enum which indicates the section in which we are present
   1108  *   2. string is the string which we need to append to the secbuff for this
   1109  *      particular section.
   1110  *  The function appends string to the global section buffer and returns.
   1111  */
   1112 static void
   1113 mdoc_parse_section(enum roff_sec sec, const char *string, mandb_rec *rec)
   1114 {
   1115 	/*
   1116 	 * If the user specified the 'l' flag, then parse and store only the
   1117 	 * NAME section. Ignore the rest.
   1118 	 */
   1119 	if (mflags.limit)
   1120 		return;
   1121 
   1122 	switch (sec) {
   1123 	case SEC_LIBRARY:
   1124 		append(&rec->lib, string);
   1125 		break;
   1126 	case SEC_RETURN_VALUES:
   1127 		append(&rec->return_vals, string);
   1128 		break;
   1129 	case SEC_ENVIRONMENT:
   1130 		append(&rec->env, string);
   1131 		break;
   1132 	case SEC_FILES:
   1133 		append(&rec->files, string);
   1134 		break;
   1135 	case SEC_EXIT_STATUS:
   1136 		append(&rec->exit_status, string);
   1137 		break;
   1138 	case SEC_DIAGNOSTICS:
   1139 		append(&rec->diagnostics, string);
   1140 		break;
   1141 	case SEC_ERRORS:
   1142 		append(&rec->errors, string);
   1143 		break;
   1144 	default:
   1145 		append(&rec->desc, string);
   1146 		break;
   1147 	}
   1148 }
   1149 
   1150 static void
   1151 proff_node(const struct roff_node * n, mandb_rec * rec,
   1152     struct roff_meta * roff, const proff_nf * func)
   1153 {
   1154 	if (n == NULL)
   1155 		return;
   1156 
   1157 	int tok_idx;
   1158 
   1159 	switch (n->type) {
   1160 	case (ROFFT_BODY):
   1161 		/* FALLTHROUGH */
   1162 	case (ROFFT_BLOCK):
   1163 		/* FALLTHROUGH */
   1164 	case (ROFFT_ELEM):
   1165 		if (roff->macroset == MACROSET_MAN)
   1166 			tok_idx = n->tok - MAN_TH;
   1167 		else if (roff->macroset == MACROSET_MDOC)
   1168 			tok_idx = n->tok - MDOC_Dd;
   1169 		else
   1170 			tok_idx = -1;
   1171 		if (tok_idx >= 0 && func[tok_idx] != NULL)
   1172 			(*func[tok_idx]) (n, rec);
   1173 		break;
   1174 	default:
   1175 		break;
   1176 	}
   1177 
   1178 	proff_node(n->child, rec, roff, func);
   1179 	proff_node(n->next, rec, roff, func);
   1180 }
   1181 
   1182 /*
   1183  * pman_parse_name --
   1184  *  Parses the NAME section and puts the complete content in the name_desc
   1185  *  variable.
   1186  */
   1187 static void
   1188 pman_parse_name(const struct roff_node *n, mandb_rec *rec)
   1189 {
   1190 	if (n == NULL)
   1191 		return;
   1192 
   1193 	if (n->type == ROFFT_TEXT) {
   1194 		char *tmp = parse_escape(n->string);
   1195 		concat(&rec->name_desc, tmp);
   1196 		free(tmp);
   1197 	}
   1198 
   1199 	if (n->child)
   1200 		pman_parse_name(n->child, rec);
   1201 
   1202 	if(n->next)
   1203 		pman_parse_name(n->next, rec);
   1204 }
   1205 
   1206 /*
   1207  * A stub function to be able to parse the macros like .B embedded inside
   1208  * a section.
   1209  */
   1210 static void
   1211 pman_block(const struct roff_node *n, mandb_rec *rec)
   1212 {
   1213 }
   1214 
   1215 /*
   1216  * pman_sh --
   1217  * This function does one of the two things:
   1218  *  1. If the present section is NAME, then it will:
   1219  *    (a) Extract the name of the page (in case of multiple comma separated
   1220  *        names, it will pick up the first one).
   1221  *    (b) Build a space spearated list of all the symlinks/hardlinks to
   1222  *        this page and store in the buffer 'links'. These are extracted from
   1223  *        the comma separated list of names in the NAME section as well.
   1224  *    (c) Move on to the one line description section, which is after the list
   1225  *        of names in the NAME section.
   1226  *  2. Otherwise, it will check the section name and call the man_parse_section
   1227  *     function, passing the enum corresponding to that section.
   1228  */
   1229 static void
   1230 pman_sh(const struct roff_node *n, mandb_rec *rec)
   1231 {
   1232 	static const struct {
   1233 		enum man_sec section;
   1234 		const char *header;
   1235 	} mapping[] = {
   1236 	    { MANSEC_DESCRIPTION, "DESCRIPTION" },
   1237 	    { MANSEC_SYNOPSIS, "SYNOPSIS" },
   1238 	    { MANSEC_LIBRARY, "LIBRARY" },
   1239 	    { MANSEC_ERRORS, "ERRORS" },
   1240 	    { MANSEC_FILES, "FILES" },
   1241 	    { MANSEC_RETURN_VALUES, "RETURN VALUE" },
   1242 	    { MANSEC_RETURN_VALUES, "RETURN VALUES" },
   1243 	    { MANSEC_EXIT_STATUS, "EXIT STATUS" },
   1244 	    { MANSEC_EXAMPLES, "EXAMPLES" },
   1245 	    { MANSEC_EXAMPLES, "EXAMPLE" },
   1246 	    { MANSEC_STANDARDS, "STANDARDS" },
   1247 	    { MANSEC_HISTORY, "HISTORY" },
   1248 	    { MANSEC_BUGS, "BUGS" },
   1249 	    { MANSEC_AUTHORS, "AUTHORS" },
   1250 	    { MANSEC_COPYRIGHT, "COPYRIGHT" },
   1251 	};
   1252 	const struct roff_node *head;
   1253 	char *name_desc;
   1254 	size_t sz;
   1255 	size_t i;
   1256 
   1257 	if ((head = n->parent->head) == NULL || (head = head->child) == NULL ||
   1258 	    head->type != ROFFT_TEXT)
   1259 		return;
   1260 
   1261 	/*
   1262 	 * Check if this section should be extracted and
   1263 	 * where it should be stored. Handled the trival cases first.
   1264 	 */
   1265 	for (i = 0; i < sizeof(mapping) / sizeof(mapping[0]); ++i) {
   1266 		if (strcmp(head->string, mapping[i].header) == 0) {
   1267 			man_parse_section(mapping[i].section, n, rec);
   1268 			return;
   1269 		}
   1270 	}
   1271 
   1272 	if (strcmp(head->string, "NAME") == 0) {
   1273 		/*
   1274 		 * We are in the NAME section.
   1275 		 * pman_parse_name will put the complete content in name_desc.
   1276 		 */
   1277 		pman_parse_name(n, rec);
   1278 
   1279 		name_desc = rec->name_desc;
   1280 		if (name_desc == NULL)
   1281 			return;
   1282 
   1283 		/* Remove any leading spaces. */
   1284 		while (name_desc[0] == ' ')
   1285 			name_desc++;
   1286 
   1287 		/* If the line begins with a "\&", avoid those */
   1288 		if (name_desc[0] == '\\' && name_desc[1] == '&')
   1289 			name_desc += 2;
   1290 
   1291 		/* Now name_desc should be left with a comma-space
   1292 		 * separated list of names and the one line description
   1293 		 * of the page:
   1294 		 *     "a, b, c \- sample description"
   1295 		 * Take out the first name, before the first comma
   1296 		 * (or space) and store it in rec->name.
   1297 		 * If the page has aliases then they should be
   1298 		 * in the form of a comma separated list.
   1299 		 * Keep looping while there is a comma in name_desc,
   1300 		 * extract the alias name and store in rec->links.
   1301 		 * When there are no more commas left, break out.
   1302 		 */
   1303 		int has_alias = 0;	// Any more aliases left?
   1304 		while (*name_desc) {
   1305 			/* Remove any leading spaces or hyphens. */
   1306 			if (name_desc[0] == ' ' || name_desc[0] == '-') {
   1307 				name_desc++;
   1308 				continue;
   1309 			}
   1310 			sz = strcspn(name_desc, ", ");
   1311 
   1312 			/* Extract the first term and store it in rec->name. */
   1313 			if (rec->name == NULL) {
   1314 				if (name_desc[sz] == ',')
   1315 					has_alias = 1;
   1316 				rec->name = estrndup(name_desc, sz);
   1317 				/* XXX This would only happen with a poorly
   1318 				 * written man page, maybe warn? */
   1319 				if (name_desc[sz] == '\0')
   1320 					break;
   1321 				name_desc += sz + 1;
   1322 				continue;
   1323 			}
   1324 
   1325 			/*
   1326 			 * Once rec->name is set, rest of the names
   1327 			 * are to be treated as links or aliases.
   1328 			 */
   1329 			if (rec->name && has_alias) {
   1330 				if (name_desc[sz] != ',') {
   1331 					/* No more commas left --> no more
   1332 					 * aliases to take out */
   1333 					has_alias = 0;
   1334 				}
   1335 				concat2(&rec->links, name_desc, sz);
   1336 				/* XXX This would only happen with a poorly
   1337 				 * written man page, maybe warn? */
   1338 				if (name_desc[sz] == '\0')
   1339 					break;
   1340 				name_desc += sz + 1;
   1341 				continue;
   1342 			}
   1343 			break;
   1344 		}
   1345 
   1346 		/* Parse any escape sequences that might be there */
   1347 		char *temp = parse_escape(name_desc);
   1348 		free(rec->name_desc);
   1349 		rec->name_desc = temp;
   1350 		temp = parse_escape(rec->name);
   1351 		free(rec->name);
   1352 		rec->name = temp;
   1353 		return;
   1354 	}
   1355 
   1356 	/* The RETURN VALUE section might be specified in multiple ways */
   1357 	if (strcmp(head->string, "RETURN") == 0 &&
   1358 	    head->next != NULL && head->next->type == ROFFT_TEXT &&
   1359 	    (strcmp(head->next->string, "VALUE") == 0 ||
   1360 	    strcmp(head->next->string, "VALUES") == 0)) {
   1361 		man_parse_section(MANSEC_RETURN_VALUES, n, rec);
   1362 		return;
   1363 	}
   1364 
   1365 	/*
   1366 	 * EXIT STATUS section can also be specified all on one line or on two
   1367 	 * separate lines.
   1368 	 */
   1369 	if (strcmp(head->string, "EXIT") == 0 &&
   1370 	    head->next != NULL && head->next->type == ROFFT_TEXT &&
   1371 	    strcmp(head->next->string, "STATUS") == 0) {
   1372 		man_parse_section(MANSEC_EXIT_STATUS, n, rec);
   1373 		return;
   1374 	}
   1375 
   1376 	/* Store the rest of the content in desc. */
   1377 	man_parse_section(MANSEC_NONE, n, rec);
   1378 }
   1379 
   1380 /*
   1381  * pman_parse_node --
   1382  *  Generic function to iterate through a node. Usually called from
   1383  *  man_parse_section to parse a particular section of the man page.
   1384  */
   1385 static void
   1386 pman_parse_node(const struct roff_node *n, secbuff *s)
   1387 {
   1388 	if (n == NULL)
   1389 		return;
   1390 
   1391 	if (n->type == ROFFT_TEXT)
   1392 		append(s, n->string);
   1393 
   1394 	pman_parse_node(n->child, s);
   1395 	pman_parse_node(n->next, s);
   1396 }
   1397 
   1398 /*
   1399  * man_parse_section --
   1400  *  Takes two parameters:
   1401  *   sec: Tells which section we are present in
   1402  *   n: Is the present node of the AST.
   1403  * Depending on the section, we call pman_parse_node to parse that section and
   1404  * concatenate the content from that section into the buffer for that section.
   1405  */
   1406 static void
   1407 man_parse_section(enum man_sec sec, const struct roff_node *n, mandb_rec *rec)
   1408 {
   1409 	/*
   1410 	 * If the user sepecified the 'l' flag then just parse
   1411 	 * the NAME section, ignore the rest.
   1412 	 */
   1413 	if (mflags.limit)
   1414 		return;
   1415 
   1416 	switch (sec) {
   1417 	case MANSEC_LIBRARY:
   1418 		pman_parse_node(n, &rec->lib);
   1419 		break;
   1420 	case MANSEC_RETURN_VALUES:
   1421 		pman_parse_node(n, &rec->return_vals);
   1422 		break;
   1423 	case MANSEC_ENVIRONMENT:
   1424 		pman_parse_node(n, &rec->env);
   1425 		break;
   1426 	case MANSEC_FILES:
   1427 		pman_parse_node(n, &rec->files);
   1428 		break;
   1429 	case MANSEC_EXIT_STATUS:
   1430 		pman_parse_node(n, &rec->exit_status);
   1431 		break;
   1432 	case MANSEC_DIAGNOSTICS:
   1433 		pman_parse_node(n, &rec->diagnostics);
   1434 		break;
   1435 	case MANSEC_ERRORS:
   1436 		pman_parse_node(n, &rec->errors);
   1437 		break;
   1438 	case MANSEC_NAME:
   1439 	case MANSEC_SYNOPSIS:
   1440 	case MANSEC_EXAMPLES:
   1441 	case MANSEC_STANDARDS:
   1442 	case MANSEC_HISTORY:
   1443 	case MANSEC_BUGS:
   1444 	case MANSEC_AUTHORS:
   1445 	case MANSEC_COPYRIGHT:
   1446 		break;
   1447 	default:
   1448 		pman_parse_node(n, &rec->desc);
   1449 		break;
   1450 	}
   1451 
   1452 }
   1453 
   1454 /*
   1455  * insert_into_db --
   1456  *  Inserts the parsed data of the man page in the Sqlite databse.
   1457  *  If any of the values is NULL, then we cleanup and return -1 indicating
   1458  *  an error.
   1459  *  Otherwise, store the data in the database and return 0.
   1460  */
   1461 static int
   1462 insert_into_db(sqlite3 *db, mandb_rec *rec)
   1463 {
   1464 	int rc = 0;
   1465 	int idx = -1;
   1466 	const char *sqlstr = NULL;
   1467 	sqlite3_stmt *stmt = NULL;
   1468 	char *ln = NULL;
   1469 	char *errmsg = NULL;
   1470 	long int mandb_rowid;
   1471 
   1472 	/*
   1473 	 * At the very minimum we want to make sure that we store
   1474 	 * the following data:
   1475 	 *   Name, one line description, and the MD5 hash
   1476 	 */
   1477 	if (rec->name == NULL || rec->name_desc == NULL ||
   1478 	    rec->md5_hash == NULL) {
   1479 		cleanup(rec);
   1480 		return -1;
   1481 	}
   1482 
   1483 	/* Write null byte at the end of all the sec_buffs */
   1484 	rec->desc.data[rec->desc.offset] = 0;
   1485 	rec->lib.data[rec->lib.offset] = 0;
   1486 	rec->env.data[rec->env.offset] = 0;
   1487 	rec->return_vals.data[rec->return_vals.offset] = 0;
   1488 	rec->exit_status.data[rec->exit_status.offset] = 0;
   1489 	rec->files.data[rec->files.offset] = 0;
   1490 	rec->diagnostics.data[rec->diagnostics.offset] = 0;
   1491 	rec->errors.data[rec->errors.offset] = 0;
   1492 
   1493 	/*
   1494 	 * In case of a mdoc page: (sorry, no better place to put this code)
   1495 	 * parse the comma separated list of names of man pages,
   1496 	 * the first name will be stored in the mandb table, rest will be
   1497 	 * treated as links and put in the mandb_links table.
   1498 	 */
   1499 	if (rec->page_type == MDOC) {
   1500 		char *tmp;
   1501 		rec->links = estrdup(rec->name);
   1502 		free(rec->name);
   1503 		size_t sz = strcspn(rec->links, " \0");
   1504 		rec->name = emalloc(sz + 1);
   1505 		memcpy(rec->name, rec->links, sz);
   1506 		if(rec->name[sz - 1] == ',')
   1507 			rec->name[sz - 1] = 0;
   1508 		else
   1509 			rec->name[sz] = 0;
   1510 		while (rec->links[sz] == ' ')
   1511 			++sz;
   1512 		tmp = estrdup(rec->links + sz);
   1513 		free(rec->links);
   1514 		rec->links = tmp;
   1515 	}
   1516 
   1517 /*------------------------ Populate the mandb table---------------------------*/
   1518 	sqlstr = "INSERT INTO mandb VALUES (:section, :name, :name_desc, :desc,"
   1519 		 " :lib, :return_vals, :env, :files, :exit_status,"
   1520 		 " :diagnostics, :errors, :md5_hash, :machine)";
   1521 
   1522 	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
   1523 	if (rc != SQLITE_OK)
   1524 		goto Out;
   1525 
   1526 	idx = sqlite3_bind_parameter_index(stmt, ":name");
   1527 	rc = sqlite3_bind_text(stmt, idx, rec->name, -1, NULL);
   1528 	if (rc != SQLITE_OK) {
   1529 		sqlite3_finalize(stmt);
   1530 		goto Out;
   1531 	}
   1532 
   1533 	idx = sqlite3_bind_parameter_index(stmt, ":section");
   1534 	rc = sqlite3_bind_text(stmt, idx, rec->section, -1, NULL);
   1535 	if (rc != SQLITE_OK) {
   1536 		sqlite3_finalize(stmt);
   1537 		goto Out;
   1538 	}
   1539 
   1540 	idx = sqlite3_bind_parameter_index(stmt, ":name_desc");
   1541 	rc = sqlite3_bind_text(stmt, idx, rec->name_desc, -1, NULL);
   1542 	if (rc != SQLITE_OK) {
   1543 		sqlite3_finalize(stmt);
   1544 		goto Out;
   1545 	}
   1546 
   1547 	idx = sqlite3_bind_parameter_index(stmt, ":desc");
   1548 	rc = sqlite3_bind_text(stmt, idx, rec->desc.data,
   1549 	                       rec->desc.offset + 1, NULL);
   1550 	if (rc != SQLITE_OK) {
   1551 		sqlite3_finalize(stmt);
   1552 		goto Out;
   1553 	}
   1554 
   1555 	idx = sqlite3_bind_parameter_index(stmt, ":lib");
   1556 	rc = sqlite3_bind_text(stmt, idx, rec->lib.data,
   1557 	    rec->lib.offset + 1, NULL);
   1558 	if (rc != SQLITE_OK) {
   1559 		sqlite3_finalize(stmt);
   1560 		goto Out;
   1561 	}
   1562 
   1563 	idx = sqlite3_bind_parameter_index(stmt, ":return_vals");
   1564 	rc = sqlite3_bind_text(stmt, idx, rec->return_vals.data,
   1565 	                      rec->return_vals.offset + 1, NULL);
   1566 	if (rc != SQLITE_OK) {
   1567 		sqlite3_finalize(stmt);
   1568 		goto Out;
   1569 	}
   1570 
   1571 	idx = sqlite3_bind_parameter_index(stmt, ":env");
   1572 	rc = sqlite3_bind_text(stmt, idx, rec->env.data,
   1573 	    rec->env.offset + 1, NULL);
   1574 	if (rc != SQLITE_OK) {
   1575 		sqlite3_finalize(stmt);
   1576 		goto Out;
   1577 	}
   1578 
   1579 	idx = sqlite3_bind_parameter_index(stmt, ":files");
   1580 	rc = sqlite3_bind_text(stmt, idx, rec->files.data,
   1581 	                       rec->files.offset + 1, NULL);
   1582 	if (rc != SQLITE_OK) {
   1583 		sqlite3_finalize(stmt);
   1584 		goto Out;
   1585 	}
   1586 
   1587 	idx = sqlite3_bind_parameter_index(stmt, ":exit_status");
   1588 	rc = sqlite3_bind_text(stmt, idx, rec->exit_status.data,
   1589 	                       rec->exit_status.offset + 1, NULL);
   1590 	if (rc != SQLITE_OK) {
   1591 		sqlite3_finalize(stmt);
   1592 		goto Out;
   1593 	}
   1594 
   1595 	idx = sqlite3_bind_parameter_index(stmt, ":diagnostics");
   1596 	rc = sqlite3_bind_text(stmt, idx, rec->diagnostics.data,
   1597 	                       rec->diagnostics.offset + 1, NULL);
   1598 	if (rc != SQLITE_OK) {
   1599 		sqlite3_finalize(stmt);
   1600 		goto Out;
   1601 	}
   1602 
   1603 	idx = sqlite3_bind_parameter_index(stmt, ":errors");
   1604 	rc = sqlite3_bind_text(stmt, idx, rec->errors.data,
   1605 	                       rec->errors.offset + 1, NULL);
   1606 	if (rc != SQLITE_OK) {
   1607 		sqlite3_finalize(stmt);
   1608 		goto Out;
   1609 	}
   1610 
   1611 	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
   1612 	rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
   1613 	if (rc != SQLITE_OK) {
   1614 		sqlite3_finalize(stmt);
   1615 		goto Out;
   1616 	}
   1617 
   1618 	idx = sqlite3_bind_parameter_index(stmt, ":machine");
   1619 	if (rec->machine)
   1620 		rc = sqlite3_bind_text(stmt, idx, rec->machine, -1, NULL);
   1621 	else
   1622 		rc = sqlite3_bind_null(stmt, idx);
   1623 	if (rc != SQLITE_OK) {
   1624 		sqlite3_finalize(stmt);
   1625 		goto Out;
   1626 	}
   1627 
   1628 	rc = sqlite3_step(stmt);
   1629 	if (rc != SQLITE_DONE) {
   1630 		sqlite3_finalize(stmt);
   1631 		goto Out;
   1632 	}
   1633 
   1634 	sqlite3_finalize(stmt);
   1635 
   1636 	/* Get the row id of the last inserted row */
   1637 	mandb_rowid = sqlite3_last_insert_rowid(db);
   1638 
   1639 /*------------------------Populate the mandb_meta table-----------------------*/
   1640 	sqlstr = "INSERT INTO mandb_meta VALUES (:device, :inode, :mtime,"
   1641 		 " :file, :md5_hash, :id)";
   1642 	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
   1643 	if (rc != SQLITE_OK)
   1644 		goto Out;
   1645 
   1646 	idx = sqlite3_bind_parameter_index(stmt, ":device");
   1647 	rc = sqlite3_bind_int64(stmt, idx, rec->device);
   1648 	if (rc != SQLITE_OK) {
   1649 		sqlite3_finalize(stmt);
   1650 		goto Out;
   1651 	}
   1652 
   1653 	idx = sqlite3_bind_parameter_index(stmt, ":inode");
   1654 	rc = sqlite3_bind_int64(stmt, idx, rec->inode);
   1655 	if (rc != SQLITE_OK) {
   1656 		sqlite3_finalize(stmt);
   1657 		goto Out;
   1658 	}
   1659 
   1660 	idx = sqlite3_bind_parameter_index(stmt, ":mtime");
   1661 	rc = sqlite3_bind_int64(stmt, idx, rec->mtime);
   1662 	if (rc != SQLITE_OK) {
   1663 		sqlite3_finalize(stmt);
   1664 		goto Out;
   1665 	}
   1666 
   1667 	idx = sqlite3_bind_parameter_index(stmt, ":file");
   1668 	rc = sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
   1669 	if (rc != SQLITE_OK) {
   1670 		sqlite3_finalize(stmt);
   1671 		goto Out;
   1672 	}
   1673 
   1674 	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
   1675 	rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
   1676 	if (rc != SQLITE_OK) {
   1677 		sqlite3_finalize(stmt);
   1678 		goto Out;
   1679 	}
   1680 
   1681 	idx = sqlite3_bind_parameter_index(stmt, ":id");
   1682 	rc = sqlite3_bind_int64(stmt, idx, mandb_rowid);
   1683 	if (rc != SQLITE_OK) {
   1684 		sqlite3_finalize(stmt);
   1685 		goto Out;
   1686 	}
   1687 
   1688 	rc = sqlite3_step(stmt);
   1689 	sqlite3_finalize(stmt);
   1690 	if (rc == SQLITE_CONSTRAINT_UNIQUE) {
   1691 		/* The *most* probable reason for reaching here is that
   1692 		 * the UNIQUE contraint on the file column of the mandb_meta
   1693 		 * table was violated.
   1694 		 * This can happen when a file was updated/modified.
   1695 		 * To fix this we need to do two things:
   1696 		 * 1. Delete the row for the older version of this file
   1697 		 *    from mandb table.
   1698 		 * 2. Run an UPDATE query to update the row for this file
   1699 		 *    in the mandb_meta table.
   1700 		 */
   1701 		warnx("Trying to update index for %s", rec->file_path);
   1702 		char *sql = sqlite3_mprintf("DELETE FROM mandb "
   1703 					    "WHERE rowid = (SELECT id"
   1704 					    "  FROM mandb_meta"
   1705 					    "  WHERE file = %Q)",
   1706 					    rec->file_path);
   1707 		sqlite3_exec(db, sql, NULL, NULL, &errmsg);
   1708 		sqlite3_free(sql);
   1709 		if (errmsg != NULL) {
   1710 			if (mflags.verbosity)
   1711 				warnx("%s", errmsg);
   1712 			free(errmsg);
   1713 		}
   1714 		sqlstr = "UPDATE mandb_meta SET device = :device,"
   1715 			 " inode = :inode, mtime = :mtime, id = :id,"
   1716 			 " md5_hash = :md5 WHERE file = :file";
   1717 		rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
   1718 		if (rc != SQLITE_OK) {
   1719 			if (mflags.verbosity)
   1720 				warnx("Update failed with error: %s",
   1721 			    sqlite3_errmsg(db));
   1722 			close_db(db);
   1723 			cleanup(rec);
   1724 			errx(EXIT_FAILURE,
   1725 			    "Consider running makemandb with -f option");
   1726 		}
   1727 
   1728 		idx = sqlite3_bind_parameter_index(stmt, ":device");
   1729 		sqlite3_bind_int64(stmt, idx, rec->device);
   1730 		idx = sqlite3_bind_parameter_index(stmt, ":inode");
   1731 		sqlite3_bind_int64(stmt, idx, rec->inode);
   1732 		idx = sqlite3_bind_parameter_index(stmt, ":mtime");
   1733 		sqlite3_bind_int64(stmt, idx, rec->mtime);
   1734 		idx = sqlite3_bind_parameter_index(stmt, ":id");
   1735 		sqlite3_bind_int64(stmt, idx, mandb_rowid);
   1736 		idx = sqlite3_bind_parameter_index(stmt, ":md5");
   1737 		sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
   1738 		idx = sqlite3_bind_parameter_index(stmt, ":file");
   1739 		sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
   1740 		rc = sqlite3_step(stmt);
   1741 		sqlite3_finalize(stmt);
   1742 
   1743 		if (rc != SQLITE_DONE) {
   1744 			if (mflags.verbosity)
   1745 				warnx("%s", sqlite3_errmsg(db));
   1746 			close_db(db);
   1747 			cleanup(rec);
   1748 			errx(EXIT_FAILURE,
   1749 			    "Consider running makemandb with -f option");
   1750 		}
   1751 	} else if (rc != SQLITE_DONE) {
   1752 		/* Otherwise make this error fatal */
   1753 		warnx("Failed at %s\n%s", rec->file_path, sqlite3_errmsg(db));
   1754 		cleanup(rec);
   1755 		close_db(db);
   1756 		exit(EXIT_FAILURE);
   1757 	}
   1758 
   1759 /*------------------------ Populate the mandb_links table---------------------*/
   1760 	char *str = NULL;
   1761 	char *links;
   1762 	if (rec->links && strlen(rec->links)) {
   1763 		links = rec->links;
   1764 		for(ln = strtok(links, " "); ln; ln = strtok(NULL, " ")) {
   1765 			if (ln[0] == ',')
   1766 				ln++;
   1767 			if(ln[strlen(ln) - 1] == ',')
   1768 				ln[strlen(ln) - 1] = 0;
   1769 
   1770 			str = sqlite3_mprintf("INSERT INTO mandb_links"
   1771 					      " VALUES (%Q, %Q, %Q, %Q, %Q)",
   1772 					      ln, rec->name, rec->section,
   1773 					      rec->machine, rec->md5_hash);
   1774 			sqlite3_exec(db, str, NULL, NULL, &errmsg);
   1775 			sqlite3_free(str);
   1776 			if (errmsg != NULL) {
   1777 				warnx("%s", errmsg);
   1778 				cleanup(rec);
   1779 				free(errmsg);
   1780 				return -1;
   1781 			}
   1782 		}
   1783 	}
   1784 
   1785 	cleanup(rec);
   1786 	return 0;
   1787 
   1788   Out:
   1789 	if (mflags.verbosity)
   1790 		warnx("%s", sqlite3_errmsg(db));
   1791 	cleanup(rec);
   1792 	return -1;
   1793 }
   1794 
   1795 /*
   1796  * check_md5--
   1797  *  Generates the md5 hash of the file and checks if it already doesn't exist
   1798  *  in the table.
   1799  *  This function is being used to avoid hardlinks.
   1800  *  On successful completion it will also set the value of the fourth parameter
   1801  *  to the md5 hash of the file (computed previously). It is the responsibility
   1802  *  of the caller to free this buffer.
   1803  *  Return values:
   1804  *  -1: If an error occurs somewhere and sets the md5 return buffer to NULL.
   1805  *  0: If the md5 hash does not exist in the table.
   1806  *  1: If the hash exists in the database.
   1807  */
   1808 static int
   1809 check_md5(const char *file, sqlite3 *db, char **md5, void *buf, size_t buflen)
   1810 {
   1811 	int rc = 0;
   1812 	int idx = -1;
   1813 	char *sqlstr = NULL;
   1814 	char *mymd5;
   1815 	sqlite3_stmt *stmt = NULL;
   1816 	*md5 = NULL;
   1817 
   1818 	assert(file != NULL);
   1819 	if ((mymd5 = MD5Data(buf, buflen, NULL)) == NULL) {
   1820 		if (mflags.verbosity)
   1821 			warn("md5 failed: %s", file);
   1822 		return -1;
   1823 	}
   1824 
   1825 	easprintf(&sqlstr, "SELECT * FROM mandb_meta WHERE md5_hash = :md5_hash");
   1826 	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
   1827 	if (rc != SQLITE_OK) {
   1828 		free(sqlstr);
   1829 		free(mymd5);
   1830 		return -1;
   1831 	}
   1832 
   1833 	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
   1834 	rc = sqlite3_bind_text(stmt, idx, mymd5, -1, NULL);
   1835 	if (rc != SQLITE_OK) {
   1836 		if (mflags.verbosity)
   1837 			warnx("%s", sqlite3_errmsg(db));
   1838 		sqlite3_finalize(stmt);
   1839 		free(sqlstr);
   1840 		free(mymd5);
   1841 		return -1;
   1842 	}
   1843 
   1844 	*md5 = mymd5;
   1845 	if (sqlite3_step(stmt) == SQLITE_ROW) {
   1846 		sqlite3_finalize(stmt);
   1847 		free(sqlstr);
   1848 		return 0;
   1849 	}
   1850 
   1851 	sqlite3_finalize(stmt);
   1852 	free(sqlstr);
   1853 	return 1;
   1854 }
   1855 
   1856 /* Optimize the index for faster search */
   1857 static void
   1858 optimize(sqlite3 *db)
   1859 {
   1860 	const char *sqlstr;
   1861 	char *errmsg = NULL;
   1862 
   1863 	if (mflags.verbosity == 2)
   1864 		printf("Optimizing the database index\n");
   1865 	sqlstr = "INSERT INTO mandb(mandb) VALUES (\'optimize\');"
   1866 		 "VACUUM";
   1867 	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
   1868 	if (errmsg != NULL) {
   1869 		if (mflags.verbosity)
   1870 			warnx("%s", errmsg);
   1871 		free(errmsg);
   1872 		return;
   1873 	}
   1874 }
   1875 
   1876 /*
   1877  * cleanup --
   1878  *  cleans up the global buffers
   1879  */
   1880 static void
   1881 cleanup(mandb_rec *rec)
   1882 {
   1883 	rec->desc.offset = 0;
   1884 	rec->lib.offset = 0;
   1885 	rec->return_vals.offset = 0;
   1886 	rec->env.offset = 0;
   1887 	rec->exit_status.offset = 0;
   1888 	rec->diagnostics.offset = 0;
   1889 	rec->errors.offset = 0;
   1890 	rec->files.offset = 0;
   1891 
   1892 	free(rec->machine);
   1893 	rec->machine = NULL;
   1894 
   1895 	free(rec->links);
   1896 	rec->links = NULL;
   1897 
   1898 	free(rec->file_path);
   1899 	rec->file_path = NULL;
   1900 
   1901 	free(rec->name);
   1902 	rec->name = NULL;
   1903 
   1904 	free(rec->name_desc);
   1905 	rec->name_desc = NULL;
   1906 
   1907 	free(rec->md5_hash);
   1908 	rec->md5_hash = NULL;
   1909 
   1910 	free(rec->section);
   1911 	rec->section = NULL;
   1912 }
   1913 
   1914 /*
   1915  * init_secbuffs--
   1916  *  Sets the value of buflen for all the sec_buff field of rec. And then
   1917  *  allocate memory to each sec_buff member of rec.
   1918  */
   1919 static void
   1920 init_secbuffs(mandb_rec *rec)
   1921 {
   1922 	/*
   1923 	 * Some sec_buff might need more memory, for example desc,
   1924 	 * which stores the data of the DESCRIPTION section,
   1925 	 * while some might need very small amount of memory.
   1926 	 * Therefore explicitly setting the value of buflen field for
   1927 	 * each sec_buff.
   1928 	 */
   1929 	rec->desc.buflen = 10 * BUFLEN;
   1930 	rec->desc.data = emalloc(rec->desc.buflen);
   1931 	rec->desc.offset = 0;
   1932 
   1933 	rec->lib.buflen = BUFLEN / 2;
   1934 	rec->lib.data = emalloc(rec->lib.buflen);
   1935 	rec->lib.offset = 0;
   1936 
   1937 	rec->return_vals.buflen = BUFLEN;
   1938 	rec->return_vals.data = emalloc(rec->return_vals.buflen);
   1939 	rec->return_vals.offset = 0;
   1940 
   1941 	rec->exit_status.buflen = BUFLEN;
   1942 	rec->exit_status.data = emalloc(rec->exit_status.buflen);
   1943 	rec->exit_status.offset = 0;
   1944 
   1945 	rec->env.buflen = BUFLEN;
   1946 	rec->env.data = emalloc(rec->env.buflen);
   1947 	rec->env.offset = 0;
   1948 
   1949 	rec->files.buflen = BUFLEN;
   1950 	rec->files.data = emalloc(rec->files.buflen);
   1951 	rec->files.offset = 0;
   1952 
   1953 	rec->diagnostics.buflen = BUFLEN;
   1954 	rec->diagnostics.data = emalloc(rec->diagnostics.buflen);
   1955 	rec->diagnostics.offset = 0;
   1956 
   1957 	rec->errors.buflen = BUFLEN;
   1958 	rec->errors.data = emalloc(rec->errors.buflen);
   1959 	rec->errors.offset = 0;
   1960 }
   1961 
   1962 /*
   1963  * free_secbuffs--
   1964  *  This function should be called at the end, when all the pages have been
   1965  *  parsed.
   1966  *  It frees the memory allocated to sec_buffs by init_secbuffs in the starting.
   1967  */
   1968 static void
   1969 free_secbuffs(mandb_rec *rec)
   1970 {
   1971 	free(rec->desc.data);
   1972 	free(rec->lib.data);
   1973 	free(rec->return_vals.data);
   1974 	free(rec->exit_status.data);
   1975 	free(rec->env.data);
   1976 	free(rec->files.data);
   1977 	free(rec->diagnostics.data);
   1978 	free(rec->errors.data);
   1979 }
   1980 
   1981 static void
   1982 replace_hyph(char *str)
   1983 {
   1984 	char *iter = str;
   1985 	while ((iter = strchr(iter, ASCII_HYPH)) != NULL)
   1986 		*iter = '-';
   1987 
   1988 	iter = str;
   1989 	while ((iter = strchr(iter, ASCII_NBRSP)) != NULL)
   1990 		*iter = '-';
   1991 }
   1992 
   1993 static char *
   1994 parse_escape(const char *str)
   1995 {
   1996 	const char *backslash, *last_backslash;
   1997 	char *result, *iter;
   1998 	size_t len;
   1999 
   2000 	assert(str);
   2001 
   2002 	last_backslash = str;
   2003 	backslash = strchr(str, '\\');
   2004 	if (backslash == NULL) {
   2005 		result = estrdup(str);
   2006 		replace_hyph(result);
   2007 		return result;
   2008 	}
   2009 
   2010 	result = emalloc(strlen(str) + 1);
   2011 	iter = result;
   2012 
   2013 	do {
   2014 		len = backslash - last_backslash;
   2015 		memcpy(iter, last_backslash, len);
   2016 		iter += len;
   2017 		if (backslash[1] == '-' || backslash[1] == ' ') {
   2018 			*iter++ = backslash[1];
   2019 			last_backslash = backslash + 2;
   2020 			backslash = strchr(last_backslash, '\\');
   2021 		} else {
   2022 			++backslash;
   2023 			mandoc_escape(&backslash, NULL, NULL);
   2024 			last_backslash = backslash;
   2025 			if (backslash == NULL)
   2026 				break;
   2027 			backslash = strchr(last_backslash, '\\');
   2028 		}
   2029 	} while (backslash != NULL);
   2030 	if (last_backslash != NULL)
   2031 		strcpy(iter, last_backslash);
   2032 
   2033 	replace_hyph(result);
   2034 	return result;
   2035 }
   2036 
   2037 /*
   2038  * append--
   2039  *  Concatenates a space and src at the end of sbuff->data (much like concat in
   2040  *  apropos-utils.c).
   2041  *  Rather than reallocating space for writing data, it uses the value of the
   2042  *  offset field of sec_buff to write new data at the free space left in the
   2043  *  buffer.
   2044  *  In case the size of the data to be appended exceeds the number of bytes left
   2045  *  in the buffer, it reallocates buflen number of bytes and then continues.
   2046  *  Value of offset field should be adjusted as new data is written.
   2047  *
   2048  *  NOTE: This function does not write the null byte at the end of the buffers,
   2049  *  write a null byte at the position pointed to by offset before inserting data
   2050  *  in the db.
   2051  */
   2052 static void
   2053 append(secbuff *sbuff, const char *src)
   2054 {
   2055 	short flag = 0;
   2056 	size_t srclen, newlen;
   2057 	char *temp;
   2058 
   2059 	assert(src != NULL);
   2060 	temp = parse_escape(src);
   2061 	srclen = strlen(temp);
   2062 
   2063 	if (sbuff->data == NULL) {
   2064 		sbuff->data = emalloc(sbuff->buflen);
   2065 		sbuff->offset = 0;
   2066 	}
   2067 
   2068 	newlen = sbuff->offset + srclen + 2;
   2069 	if (newlen >= sbuff->buflen) {
   2070 		while (sbuff->buflen < newlen)
   2071 			sbuff->buflen += sbuff->buflen;
   2072 		sbuff->data = erealloc(sbuff->data, sbuff->buflen);
   2073 		flag = 1;
   2074 	}
   2075 
   2076 	/* Append a space at the end of the buffer. */
   2077 	if (sbuff->offset || flag)
   2078 		sbuff->data[sbuff->offset++] = ' ';
   2079 	/* Now, copy src at the end of the buffer. */
   2080 	memcpy(sbuff->data + sbuff->offset, temp, srclen);
   2081 	sbuff->offset += srclen;
   2082 	free(temp);
   2083 }
   2084 
   2085 static void
   2086 usage(void)
   2087 {
   2088 	fprintf(stderr, "Usage: %s [-floQqv] [-C path]\n", getprogname());
   2089 	exit(1);
   2090 }
   2091