makemandb.c revision 1.5
1 /* $NetBSD: makemandb.c,v 1.5 2012/02/16 20:58:55 joerg Exp $ */ 2 /* 3 * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay (at) gmail.com> 4 * Copyright (c) 2011 Kristaps Dzonsons <kristaps (at) bsd.lv> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/cdefs.h> 20 __RCSID("$NetBSD: makemandb.c,v 1.5 2012/02/16 20:58:55 joerg Exp $"); 21 22 #include <sys/stat.h> 23 #include <sys/types.h> 24 25 #include <assert.h> 26 #include <ctype.h> 27 #include <dirent.h> 28 #include <err.h> 29 #include <archive.h> 30 #include <libgen.h> 31 #include <md5.h> 32 #include <stdio.h> 33 #include <stdlib.h> 34 #include <string.h> 35 #include <unistd.h> 36 #include <util.h> 37 38 #include "apropos-utils.h" 39 #include "man.h" 40 #include "mandoc.h" 41 #include "mdoc.h" 42 #include "sqlite3.h" 43 44 #define BUFLEN 1024 45 #define MDOC 0 //If the page is of mdoc(7) type 46 #define MAN 1 //If the page is of man(7) type 47 48 /* 49 * A data structure for holding section specific data. 50 */ 51 typedef struct secbuff { 52 char *data; 53 size_t buflen; //Total length of buffer allocated initially 54 size_t offset; // Current offset in the buffer. 55 } secbuff; 56 57 typedef struct makemandb_flags { 58 int optimize; 59 int limit; // limit the indexing to only NAME section 60 int recreate; // Database was created from scratch 61 int verbosity; // 0: quiet, 1: default, 2: verbose 62 } makemandb_flags; 63 64 typedef struct mandb_rec { 65 /* Fields for mandb table */ 66 char *name; // for storing the name of the man page 67 char *name_desc; // for storing the one line description (.Nd) 68 secbuff desc; // for storing the DESCRIPTION section 69 secbuff lib; // for the LIBRARY section 70 secbuff return_vals; // RETURN VALUES 71 secbuff env; // ENVIRONMENT 72 secbuff files; // FILES 73 secbuff exit_status; // EXIT STATUS 74 secbuff diagnostics; // DIAGNOSTICS 75 secbuff errors; // ERRORS 76 char section[2]; 77 78 int xr_found; 79 80 /* Fields for mandb_meta table */ 81 char *md5_hash; 82 dev_t device; 83 ino_t inode; 84 time_t mtime; 85 86 /* Fields for mandb_links table */ 87 char *machine; 88 char *links; //all the links to a page in a space separated form 89 char *file_path; 90 91 /* Non-db fields */ 92 int page_type; //Indicates the type of page: mdoc or man 93 } mandb_rec; 94 95 static void append(secbuff *sbuff, const char *src); 96 static void init_secbuffs(mandb_rec *); 97 static void free_secbuffs(mandb_rec *); 98 static int check_md5(const char *, sqlite3 *, const char *, char **, void *, size_t); 99 static void cleanup(mandb_rec *); 100 static void set_section(const struct mdoc *, const struct man *, mandb_rec *); 101 static void set_machine(const struct mdoc *, mandb_rec *); 102 static int insert_into_db(sqlite3 *, mandb_rec *); 103 static void begin_parse(const char *, struct mparse *, mandb_rec *, 104 const void *, size_t len); 105 static void pmdoc_node(const struct mdoc_node *, mandb_rec *); 106 static void pmdoc_Nm(const struct mdoc_node *, mandb_rec *); 107 static void pmdoc_Nd(const struct mdoc_node *, mandb_rec *); 108 static void pmdoc_Sh(const struct mdoc_node *, mandb_rec *); 109 static void pmdoc_Xr(const struct mdoc_node *, mandb_rec *); 110 static void pmdoc_Pp(const struct mdoc_node *, mandb_rec *); 111 static void pmdoc_macro_handler(const struct mdoc_node *, mandb_rec *, 112 enum mdoct); 113 static void pman_node(const struct man_node *n, mandb_rec *); 114 static void pman_parse_node(const struct man_node *, secbuff *); 115 static void pman_parse_name(const struct man_node *, mandb_rec *); 116 static void pman_sh(const struct man_node *, mandb_rec *); 117 static void pman_block(const struct man_node *, mandb_rec *); 118 static void traversedir(const char *, sqlite3 *, struct mparse *); 119 static void mdoc_parse_section(enum mdoc_sec, const char *, mandb_rec *); 120 static void man_parse_section(enum man_sec, const struct man_node *, mandb_rec *); 121 static void build_file_cache(sqlite3 *, const char *, struct stat *); 122 static void update_db(sqlite3 *, struct mparse *, mandb_rec *); 123 __dead static void usage(void); 124 static void optimize(sqlite3 *); 125 static char *parse_escape(const char *); 126 static makemandb_flags mflags = { .verbosity = 1 }; 127 128 typedef void (*pman_nf)(const struct man_node *n, mandb_rec *); 129 typedef void (*pmdoc_nf)(const struct mdoc_node *n, mandb_rec *); 130 static const pmdoc_nf mdocs[MDOC_MAX] = { 131 NULL, /* Ap */ 132 NULL, /* Dd */ 133 NULL, /* Dt */ 134 NULL, /* Os */ 135 pmdoc_Sh, /* Sh */ 136 NULL, /* Ss */ 137 pmdoc_Pp, /* Pp */ 138 NULL, /* D1 */ 139 NULL, /* Dl */ 140 NULL, /* Bd */ 141 NULL, /* Ed */ 142 NULL, /* Bl */ 143 NULL, /* El */ 144 NULL, /* It */ 145 NULL, /* Ad */ 146 NULL, /* An */ 147 NULL, /* Ar */ 148 NULL, /* Cd */ 149 NULL, /* Cm */ 150 NULL, /* Dv */ 151 NULL, /* Er */ 152 NULL, /* Ev */ 153 NULL, /* Ex */ 154 NULL, /* Fa */ 155 NULL, /* Fd */ 156 NULL, /* Fl */ 157 NULL, /* Fn */ 158 NULL, /* Ft */ 159 NULL, /* Ic */ 160 NULL, /* In */ 161 NULL, /* Li */ 162 pmdoc_Nd, /* Nd */ 163 pmdoc_Nm, /* Nm */ 164 NULL, /* Op */ 165 NULL, /* Ot */ 166 NULL, /* Pa */ 167 NULL, /* Rv */ 168 NULL, /* St */ 169 NULL, /* Va */ 170 NULL, /* Vt */ 171 pmdoc_Xr, /* Xr */ 172 NULL, /* %A */ 173 NULL, /* %B */ 174 NULL, /* %D */ 175 NULL, /* %I */ 176 NULL, /* %J */ 177 NULL, /* %N */ 178 NULL, /* %O */ 179 NULL, /* %P */ 180 NULL, /* %R */ 181 NULL, /* %T */ 182 NULL, /* %V */ 183 NULL, /* Ac */ 184 NULL, /* Ao */ 185 NULL, /* Aq */ 186 NULL, /* At */ 187 NULL, /* Bc */ 188 NULL, /* Bf */ 189 NULL, /* Bo */ 190 NULL, /* Bq */ 191 NULL, /* Bsx */ 192 NULL, /* Bx */ 193 NULL, /* Db */ 194 NULL, /* Dc */ 195 NULL, /* Do */ 196 NULL, /* Dq */ 197 NULL, /* Ec */ 198 NULL, /* Ef */ 199 NULL, /* Em */ 200 NULL, /* Eo */ 201 NULL, /* Fx */ 202 NULL, /* Ms */ 203 NULL, /* No */ 204 NULL, /* Ns */ 205 NULL, /* Nx */ 206 NULL, /* Ox */ 207 NULL, /* Pc */ 208 NULL, /* Pf */ 209 NULL, /* Po */ 210 NULL, /* Pq */ 211 NULL, /* Qc */ 212 NULL, /* Ql */ 213 NULL, /* Qo */ 214 NULL, /* Qq */ 215 NULL, /* Re */ 216 NULL, /* Rs */ 217 NULL, /* Sc */ 218 NULL, /* So */ 219 NULL, /* Sq */ 220 NULL, /* Sm */ 221 NULL, /* Sx */ 222 NULL, /* Sy */ 223 NULL, /* Tn */ 224 NULL, /* Ux */ 225 NULL, /* Xc */ 226 NULL, /* Xo */ 227 NULL, /* Fo */ 228 NULL, /* Fc */ 229 NULL, /* Oo */ 230 NULL, /* Oc */ 231 NULL, /* Bk */ 232 NULL, /* Ek */ 233 NULL, /* Bt */ 234 NULL, /* Hf */ 235 NULL, /* Fr */ 236 NULL, /* Ud */ 237 NULL, /* Lb */ 238 NULL, /* Lp */ 239 NULL, /* Lk */ 240 NULL, /* Mt */ 241 NULL, /* Brq */ 242 NULL, /* Bro */ 243 NULL, /* Brc */ 244 NULL, /* %C */ 245 NULL, /* Es */ 246 NULL, /* En */ 247 NULL, /* Dx */ 248 NULL, /* %Q */ 249 NULL, /* br */ 250 NULL, /* sp */ 251 NULL, /* %U */ 252 NULL, /* Ta */ 253 }; 254 255 static const pman_nf mans[MAN_MAX] = { 256 NULL, //br 257 NULL, //TH 258 pman_sh, //SH 259 NULL, //SS 260 NULL, //TP 261 NULL, //LP 262 NULL, //PP 263 NULL, //P 264 NULL, //IP 265 NULL, //HP 266 NULL, //SM 267 NULL, //SB 268 NULL, //BI 269 NULL, //IB 270 NULL, //BR 271 NULL, //RB 272 NULL, //R 273 pman_block, //B 274 NULL, //I 275 NULL, //IR 276 NULL, //RI 277 NULL, //na 278 NULL, //sp 279 NULL, //nf 280 NULL, //fi 281 NULL, //RE 282 NULL, //RS 283 NULL, //DT 284 NULL, //UC 285 NULL, //PD 286 NULL, //AT 287 NULL, //in 288 NULL, //ft 289 }; 290 291 292 int 293 main(int argc, char *argv[]) 294 { 295 FILE *file; 296 const char *sqlstr, *manconf = NULL; 297 char *line, *command, *parent; 298 char *errmsg; 299 int ch; 300 struct mparse *mp; 301 sqlite3 *db; 302 ssize_t len; 303 size_t linesize; 304 struct mandb_rec rec; 305 306 while ((ch = getopt(argc, argv, "C:floqv")) != -1) { 307 switch (ch) { 308 case 'C': 309 manconf = optarg; 310 break; 311 case 'f': 312 remove(DBPATH); 313 mflags.recreate = 1; 314 break; 315 case 'l': 316 mflags.limit = 1; 317 break; 318 case 'o': 319 mflags.optimize = 1; 320 break; 321 case 'q': 322 mflags.verbosity = 0; 323 break; 324 case 'v': 325 mflags.verbosity = 2; 326 break; 327 default: 328 usage(); 329 } 330 } 331 332 memset(&rec, 0, sizeof(rec)); 333 334 init_secbuffs(&rec); 335 mp = mparse_alloc(MPARSE_AUTO, MANDOCLEVEL_FATAL, NULL, NULL); 336 337 if ((db = init_db(MANDB_CREATE)) == NULL) 338 exit(EXIT_FAILURE); 339 340 sqlite3_exec(db, "PRAGMA synchronous = 0", NULL, NULL, &errmsg); 341 if (errmsg != NULL) { 342 warnx("%s", errmsg); 343 free(errmsg); 344 close_db(db); 345 exit(EXIT_FAILURE); 346 } 347 348 sqlite3_exec(db, "ATTACH DATABASE \':memory:\' AS metadb", NULL, NULL, 349 &errmsg); 350 if (errmsg != NULL) { 351 warnx("%s", errmsg); 352 free(errmsg); 353 close_db(db); 354 exit(EXIT_FAILURE); 355 } 356 357 if (manconf) { 358 char *arg; 359 size_t command_len = shquote(manconf, NULL, 0) + 1; 360 arg = malloc(command_len ); 361 shquote(manconf, arg, command_len); 362 easprintf(&command, "man -p -C %s", arg); 363 free(arg); 364 } else { 365 command = estrdup("man -p"); 366 } 367 368 /* Call man -p to get the list of man page dirs */ 369 if ((file = popen(command, "r")) == NULL) { 370 close_db(db); 371 err(EXIT_FAILURE, "fopen failed"); 372 } 373 free(command); 374 375 /* Begin the transaction for indexing the pages */ 376 sqlite3_exec(db, "BEGIN", NULL, NULL, &errmsg); 377 if (errmsg != NULL) { 378 warnx("%s", errmsg); 379 free(errmsg); 380 exit(EXIT_FAILURE); 381 } 382 383 sqlstr = "CREATE TABLE IF NOT EXISTS metadb.file_cache(device, inode," 384 " mtime, file PRIMARY KEY);" 385 "CREATE UNIQUE INDEX IF NOT EXISTS metadb.index_file_cache_dev" 386 " ON file_cache (device, inode)"; 387 388 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg); 389 if (errmsg != NULL) { 390 warnx("%s", errmsg); 391 free(errmsg); 392 close_db(db); 393 exit(EXIT_FAILURE); 394 } 395 396 if (mflags.verbosity) 397 printf("Building temporary file cache\n"); 398 line = NULL; 399 linesize = 0; 400 while ((len = getline(&line, &linesize, file)) != -1) { 401 /* Replace the new line character at the end of string with '\0' */ 402 line[len - 1] = '\0'; 403 parent = estrdup(line); 404 chdir(dirname(parent)); 405 free(parent); 406 /* Traverse the man page directories and parse the pages */ 407 traversedir(line, db, mp); 408 } 409 free(line); 410 411 if (pclose(file) == -1) { 412 close_db(db); 413 cleanup(&rec); 414 free_secbuffs(&rec); 415 err(EXIT_FAILURE, "pclose error"); 416 } 417 418 update_db(db, mp, &rec); 419 mparse_free(mp); 420 free_secbuffs(&rec); 421 422 /* Commit the transaction */ 423 sqlite3_exec(db, "COMMIT", NULL, NULL, &errmsg); 424 if (errmsg != NULL) { 425 warnx("%s", errmsg); 426 free(errmsg); 427 exit(EXIT_FAILURE); 428 } 429 430 if (mflags.optimize) 431 optimize(db); 432 433 close_db(db); 434 return 0; 435 } 436 437 /* 438 * traversedir -- 439 * Traverses the given directory recursively and passes all the man page files 440 * in the way to build_file_cache() 441 */ 442 static void 443 traversedir(const char *file, sqlite3 *db, struct mparse *mp) 444 { 445 struct stat sb; 446 struct dirent *dirp; 447 DIR *dp; 448 char *buf; 449 450 if (stat(file, &sb) < 0) { 451 warn("stat failed: %s", file); 452 return; 453 } 454 455 /* If it is a regular file or a symlink, pass it to build_cache() */ 456 if (S_ISREG(sb.st_mode) || S_ISLNK(sb.st_mode)) { 457 build_file_cache(db, file, &sb); 458 return; 459 } 460 461 /* If it is a directory, traverse it recursively */ 462 if (S_ISDIR(sb.st_mode)) { 463 if ((dp = opendir(file)) == NULL) { 464 warn("opendir error: %s", file); 465 return; 466 } 467 468 while ((dirp = readdir(dp)) != NULL) { 469 /* Avoid . and .. entries in a directory */ 470 if (strncmp(dirp->d_name, ".", 1)) { 471 easprintf(&buf, "%s/%s", file, dirp->d_name); 472 traversedir(buf, db, mp); 473 free(buf); 474 } 475 } 476 closedir(dp); 477 } 478 } 479 480 /* build_file_cache -- 481 * This function generates an md5 hash of the file passed as it's 2nd parameter 482 * and stores it in a temporary table file_cache along with the full file path. 483 * This is done to support incremental updation of the database. 484 * The temporary table file_cache is dropped thereafter in the function 485 * update_db(), once the database has been updated. 486 */ 487 static void 488 build_file_cache(sqlite3 *db, const char *file, struct stat *sb) 489 { 490 const char *sqlstr; 491 sqlite3_stmt *stmt = NULL; 492 int rc, idx; 493 assert(file != NULL); 494 dev_t device_cache = sb->st_dev; 495 ino_t inode_cache = sb->st_ino; 496 time_t mtime_cache = sb->st_mtime; 497 498 sqlstr = "INSERT INTO metadb.file_cache VALUES (:device, :inode," 499 " :mtime, :file)"; 500 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); 501 if (rc != SQLITE_OK) { 502 warnx("%s", sqlite3_errmsg(db)); 503 return; 504 } 505 506 idx = sqlite3_bind_parameter_index(stmt, ":device"); 507 rc = sqlite3_bind_int64(stmt, idx, device_cache); 508 if (rc != SQLITE_OK) { 509 warnx("%s", sqlite3_errmsg(db)); 510 sqlite3_finalize(stmt); 511 return; 512 } 513 514 idx = sqlite3_bind_parameter_index(stmt, ":inode"); 515 rc = sqlite3_bind_int64(stmt, idx, inode_cache); 516 if (rc != SQLITE_OK) { 517 warnx("%s", sqlite3_errmsg(db)); 518 sqlite3_finalize(stmt); 519 return; 520 } 521 522 idx = sqlite3_bind_parameter_index(stmt, ":mtime"); 523 rc = sqlite3_bind_int64(stmt, idx, mtime_cache); 524 if (rc != SQLITE_OK) { 525 warnx("%s", sqlite3_errmsg(db)); 526 sqlite3_finalize(stmt); 527 return; 528 } 529 530 idx = sqlite3_bind_parameter_index(stmt, ":file"); 531 rc = sqlite3_bind_text(stmt, idx, file, -1, NULL); 532 if (rc != SQLITE_OK) { 533 warnx("%s", sqlite3_errmsg(db)); 534 sqlite3_finalize(stmt); 535 return; 536 } 537 538 sqlite3_step(stmt); 539 sqlite3_finalize(stmt); 540 } 541 542 static void 543 update_existing_entry(sqlite3 *db, const char *file, const char *hash, 544 mandb_rec *rec, int *new_count, int *link_count, int *err_count) 545 { 546 int update_count, rc, idx; 547 const char *inner_sqlstr; 548 sqlite3_stmt *inner_stmt; 549 550 update_count = sqlite3_total_changes(db); 551 inner_sqlstr = "UPDATE mandb_meta SET device = :device," 552 " inode = :inode, mtime = :mtime WHERE" 553 " md5_hash = :md5 AND file = :file AND" 554 " (device <> :device2 OR inode <> " 555 " :inode2 OR mtime <> :mtime2)"; 556 rc = sqlite3_prepare_v2(db, inner_sqlstr, -1, &inner_stmt, NULL); 557 if (rc != SQLITE_OK) { 558 warnx("%s", sqlite3_errmsg(db)); 559 return; 560 } 561 idx = sqlite3_bind_parameter_index(inner_stmt, ":device"); 562 sqlite3_bind_int64(inner_stmt, idx, rec->device); 563 idx = sqlite3_bind_parameter_index(inner_stmt, ":inode"); 564 sqlite3_bind_int64(inner_stmt, idx, rec->inode); 565 idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime"); 566 sqlite3_bind_int64(inner_stmt, idx, rec->mtime); 567 idx = sqlite3_bind_parameter_index(inner_stmt, ":md5"); 568 sqlite3_bind_text(inner_stmt, idx, hash, -1, NULL); 569 idx = sqlite3_bind_parameter_index(inner_stmt, ":file"); 570 sqlite3_bind_text(inner_stmt, idx, file, -1, NULL); 571 idx = sqlite3_bind_parameter_index(inner_stmt, ":device2"); 572 sqlite3_bind_int64(inner_stmt, idx, rec->device); 573 idx = sqlite3_bind_parameter_index(inner_stmt, ":inode2"); 574 sqlite3_bind_int64(inner_stmt, idx, rec->inode); 575 idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime2"); 576 sqlite3_bind_int64(inner_stmt, idx, rec->mtime); 577 578 rc = sqlite3_step(inner_stmt); 579 if (rc == SQLITE_DONE) { 580 /* Check if an update has been performed. */ 581 if (update_count != sqlite3_total_changes(db)) { 582 if (mflags.verbosity) 583 printf("Updated %s\n", file); 584 (*new_count)++; 585 } else { 586 /* Otherwise it was a hardlink. */ 587 (*link_count)++; 588 } 589 } else { 590 warnx("Could not update the meta data for %s", file); 591 (*err_count)++; 592 } 593 sqlite3_finalize(inner_stmt); 594 } 595 596 /* read_and_decompress -- 597 * Reads the given file into memory. If it is compressed, decompres 598 * it before returning to the caller. 599 */ 600 static int 601 read_and_decompress(const char *file, void **buf, size_t *len) 602 { 603 size_t off; 604 ssize_t r; 605 struct archive *a; 606 struct archive_entry *ae; 607 608 if ((a = archive_read_new()) == NULL) 609 errx(EXIT_FAILURE, "memory allocation failed"); 610 611 if (archive_read_support_compression_all(a) != ARCHIVE_OK || 612 archive_read_support_format_raw(a) != ARCHIVE_OK || 613 archive_read_open_filename(a, file, 65536) != ARCHIVE_OK || 614 archive_read_next_header(a, &ae) != ARCHIVE_OK) 615 goto archive_error; 616 *len = 65536; 617 *buf = emalloc(*len); 618 off = 0; 619 for (;;) { 620 r = archive_read_data(a, (char *)*buf + off, *len - off); 621 if (r == ARCHIVE_OK) { 622 archive_read_close(a); 623 *len = off; 624 return 0; 625 } 626 if (r <= 0) { 627 free(*buf); 628 break; 629 } 630 off += r; 631 if (off == *len) { 632 *len *= 2; 633 if (*len < off) { 634 warnx("File too large: %s", file); 635 free(*buf); 636 archive_read_close(a); 637 return -1; 638 } 639 *buf = erealloc(*buf, *len); 640 } 641 } 642 643 archive_error: 644 warnx("Error while reading `%s': %s", file, archive_error_string(a)); 645 archive_read_close(a); 646 return -1; 647 } 648 649 /* update_db -- 650 * Does an incremental updation of the database by checking the file_cache. 651 * It parses and adds the pages which are present in file_cache, 652 * but not in the database. 653 * It also removes the pages which are present in the databse, 654 * but not in the file_cache. 655 */ 656 static void 657 update_db(sqlite3 *db, struct mparse *mp, mandb_rec *rec) 658 { 659 const char *sqlstr; 660 sqlite3_stmt *stmt = NULL; 661 const char *file; 662 char *errmsg = NULL; 663 char *md5sum; 664 void *buf; 665 size_t buflen; 666 int new_count = 0; /* Counter for newly indexed/updated pages */ 667 int total_count = 0; /* Counter for total number of pages */ 668 int err_count = 0; /* Counter for number of failed pages */ 669 int link_count = 0; /* Counter for number of hard/sym links */ 670 int md5_status; 671 int rc; 672 673 sqlstr = "SELECT device, inode, mtime, file FROM metadb.file_cache" 674 " EXCEPT SELECT device, inode, mtime, file from mandb_meta"; 675 676 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); 677 if (rc != SQLITE_OK) { 678 warnx("%s", sqlite3_errmsg(db)); 679 close_db(db); 680 errx(EXIT_FAILURE, "Could not query file cache"); 681 } 682 683 buf = NULL; 684 while (sqlite3_step(stmt) == SQLITE_ROW) { 685 free(buf); 686 total_count++; 687 rec->device = sqlite3_column_int64(stmt, 0); 688 rec->inode = sqlite3_column_int64(stmt, 1); 689 rec->mtime = sqlite3_column_int64(stmt, 2); 690 file = (const char *) sqlite3_column_text(stmt, 3); 691 if (read_and_decompress(file, &buf, &buflen)) { 692 err_count++; 693 buf = NULL; 694 continue; 695 } 696 md5_status = check_md5(file, db, "mandb_meta", &md5sum, buf, buflen); 697 assert(md5sum != NULL); 698 if (md5_status == -1) { 699 warnx("An error occurred in checking md5 value" 700 " for file %s", file); 701 err_count++; 702 continue; 703 } 704 705 if (md5_status == 0) { 706 /* 707 * The MD5 hash is already present in the database, 708 * so simply update the metadata, ignoring symlinks. 709 */ 710 struct stat sb; 711 stat(file, &sb); 712 if (S_ISLNK(sb.st_mode)) { 713 free(md5sum); 714 link_count++; 715 continue; 716 } 717 update_existing_entry(db, file, md5sum, rec, 718 &new_count, &link_count, &err_count); 719 free(md5sum); 720 continue; 721 } 722 723 if (md5_status == 1) { 724 /* 725 * The MD5 hash was not present in the database. 726 * This means is either a new file or an updated file. 727 * We should go ahead with parsing. 728 */ 729 if (mflags.verbosity > 1) 730 printf("Parsing: %s\n", file); 731 rec->md5_hash = md5sum; 732 rec->file_path = estrdup(file); 733 // file_path is freed by insert_into_db itself. 734 begin_parse(file, mp, rec, buf, buflen); 735 if (insert_into_db(db, rec) < 0) { 736 warnx("Error in indexing %s", file); 737 err_count++; 738 } else { 739 new_count++; 740 } 741 } 742 } 743 free(buf); 744 745 sqlite3_finalize(stmt); 746 747 if (mflags.verbosity) { 748 printf("Total Number of new or updated pages enountered = %d\n" 749 "Total number of pages that were successfully" 750 " indexed/updated = %d\n" 751 "Total number of (hard or symbolic) links found = %d\n" 752 "Total number of pages that could not be indexed" 753 " due to errors = %d\n", 754 total_count, new_count, link_count, err_count); 755 } 756 757 if (mflags.recreate == 0) 758 return; 759 760 if (mflags.verbosity) 761 printf("Deleting stale index entries\n"); 762 763 sqlstr = "DELETE FROM mandb_meta WHERE file NOT IN" 764 " (SELECT file FROM metadb.file_cache);" 765 "DROP TABLE metadb.file_cache;" 766 "DELETE FROM mandb WHERE rowid NOT IN" 767 " (SELECT id FROM mandb_meta);"; 768 769 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg); 770 if (errmsg != NULL) { 771 warnx("Removing old entries failed: %s", errmsg); 772 warnx("Please rebuild database from scratch with -f."); 773 free(errmsg); 774 return; 775 } 776 } 777 778 /* 779 * begin_parse -- 780 * parses the man page using libmandoc 781 */ 782 static void 783 begin_parse(const char *file, struct mparse *mp, mandb_rec *rec, 784 const void *buf, size_t len) 785 { 786 struct mdoc *mdoc; 787 struct man *man; 788 mparse_reset(mp); 789 790 rec->xr_found = 0; 791 792 if (mparse_readmem(mp, buf, len, file) >= MANDOCLEVEL_FATAL) { 793 warnx("%s: Parse failure", file); 794 return; 795 } 796 797 mparse_result(mp, &mdoc, &man); 798 if (mdoc == NULL && man == NULL) { 799 warnx("Not a man(7) or mdoc(7) page"); 800 return; 801 } 802 803 set_machine(mdoc, rec); 804 set_section(mdoc, man, rec); 805 if (mdoc) { 806 rec->page_type = MDOC; 807 pmdoc_node(mdoc_node(mdoc), rec); 808 } else { 809 rec->page_type = MAN; 810 pman_node(man_node(man), rec); 811 } 812 } 813 814 /* 815 * set_section -- 816 * Extracts the section number and normalizes it to only the numeric part 817 * (Which should be the first character of the string). 818 */ 819 static void 820 set_section(const struct mdoc *md, const struct man *m, mandb_rec *rec) 821 { 822 if (md) { 823 const struct mdoc_meta *md_meta = mdoc_meta(md); 824 rec->section[0] = md_meta->msec[0]; 825 } else if (m) { 826 const struct man_meta *m_meta = man_meta(m); 827 rec->section[0] = m_meta->msec[0]; 828 } 829 } 830 831 /* 832 * get_machine -- 833 * Extracts the machine architecture information if available. 834 */ 835 static void 836 set_machine(const struct mdoc *md, mandb_rec *rec) 837 { 838 if (md == NULL) 839 return; 840 const struct mdoc_meta *md_meta = mdoc_meta(md); 841 if (md_meta->arch) 842 rec->machine = estrdup(md_meta->arch); 843 } 844 845 static void 846 pmdoc_node(const struct mdoc_node *n, mandb_rec *rec) 847 { 848 849 if (n == NULL) 850 return; 851 852 switch (n->type) { 853 case (MDOC_BODY): 854 /* FALLTHROUGH */ 855 case (MDOC_TAIL): 856 /* FALLTHROUGH */ 857 case (MDOC_ELEM): 858 if (mdocs[n->tok] == NULL) 859 break; 860 (*mdocs[n->tok])(n, rec); 861 break; 862 default: 863 break; 864 } 865 866 pmdoc_node(n->child, rec); 867 pmdoc_node(n->next, rec); 868 } 869 870 /* 871 * pmdoc_Nm -- 872 * Extracts the Name of the manual page from the .Nm macro 873 */ 874 static void 875 pmdoc_Nm(const struct mdoc_node *n, mandb_rec *rec) 876 { 877 if (n->sec != SEC_NAME) 878 return; 879 880 for (n = n->child; n; n = n->next) { 881 if (n->type == MDOC_TEXT) { 882 concat(&rec->name, n->string); 883 } 884 } 885 } 886 887 /* 888 * pmdoc_Nd -- 889 * Extracts the one line description of the man page from the .Nd macro 890 */ 891 static void 892 pmdoc_Nd(const struct mdoc_node *n, mandb_rec *rec) 893 { 894 /* 895 * A static variable for keeping track of whether a Xr macro was seen 896 * previously. 897 */ 898 char *buf = NULL; 899 char *temp; 900 901 if (n == NULL) 902 return; 903 904 if (n->type == MDOC_TEXT) { 905 if (rec->xr_found && n->next) { 906 /* 907 * An Xr macro was seen previously, so parse this 908 * and the next node. 909 */ 910 temp = estrdup(n->string); 911 n = n->next; 912 easprintf(&buf, "%s(%s)", temp, n->string); 913 concat(&rec->name_desc, buf); 914 free(buf); 915 free(temp); 916 } else { 917 concat(&rec->name_desc, n->string); 918 } 919 rec->xr_found = 0; 920 } else if (mdocs[n->tok] == pmdoc_Xr) { 921 /* Remember that we have encountered an Xr macro */ 922 rec->xr_found = 1; 923 } 924 925 if (n->child) 926 pmdoc_Nd(n->child, rec); 927 928 if(n->next) 929 pmdoc_Nd(n->next, rec); 930 } 931 932 /* 933 * pmdoc_macro_handler-- 934 * This function is a single point of handling all the special macros that we 935 * want to handle especially. For example the .Xr macro for properly parsing 936 * the referenced page name along with the section number, or the .Pp macro 937 * for adding a new line whenever we encounter it. 938 */ 939 static void 940 pmdoc_macro_handler(const struct mdoc_node *n, mandb_rec *rec, enum mdoct doct) 941 { 942 const struct mdoc_node *sn; 943 assert(n); 944 945 switch (doct) { 946 /* Parse the man page references. 947 * Basically the .Xr macros are used like: 948 * .Xr ls 1 949 * and formatted like this: 950 * ls(1) 951 * Prepare a buffer to format the data like the above example and call 952 * pmdoc_parse_section to append it. 953 */ 954 case MDOC_Xr: 955 n = n->child; 956 while (n->type != MDOC_TEXT && n->next) 957 n = n->next; 958 959 if (n && n->type != MDOC_TEXT) 960 return; 961 sn = n; 962 if (n->next) 963 n = n->next; 964 965 while (n->type != MDOC_TEXT && n->next) 966 n = n->next; 967 968 if (n && n->type == MDOC_TEXT) { 969 size_t len = strlen(sn->string); 970 char *buf = emalloc(len + 4); 971 memcpy(buf, sn->string, len); 972 buf[len] = '('; 973 buf[len + 1] = n->string[0]; 974 buf[len + 2] = ')'; 975 buf[len + 3] = 0; 976 mdoc_parse_section(n->sec, buf, rec); 977 free(buf); 978 } 979 980 break; 981 982 /* Parse the .Pp macro to add a new line */ 983 case MDOC_Pp: 984 if (n->type == MDOC_TEXT) 985 mdoc_parse_section(n->sec, "\n", rec); 986 break; 987 default: 988 break; 989 } 990 991 } 992 993 /* 994 * pmdoc_Xr, pmdoc_Pp-- 995 * Empty stubs. 996 * The parser calls these functions each time it encounters 997 * a .Xr or .Pp macro. We are parsing all the data from 998 * the pmdoc_Sh function, so don't do anything here. 999 * (See if else blocks in pmdoc_Sh.) 1000 */ 1001 static void 1002 pmdoc_Xr(const struct mdoc_node *n, mandb_rec *rec) 1003 { 1004 } 1005 1006 static void 1007 pmdoc_Pp(const struct mdoc_node *n, mandb_rec *rec) 1008 { 1009 } 1010 1011 /* 1012 * pmdoc_Sh -- 1013 * Called when a .Sh macro is encountered and loops through its body, calling 1014 * mdoc_parse_section to append the data to the section specific buffer. 1015 * Two special macros which may occur inside the body of Sh are .Nm and .Xr and 1016 * they need special handling, thus the separate if branches for them. 1017 */ 1018 static void 1019 pmdoc_Sh(const struct mdoc_node *n, mandb_rec *rec) 1020 { 1021 if (n == NULL) 1022 return; 1023 int xr_found = 0; 1024 1025 if (n->type == MDOC_TEXT) { 1026 mdoc_parse_section(n->sec, n->string, rec); 1027 } else if (mdocs[n->tok] == pmdoc_Nm && rec->name != NULL) { 1028 /* 1029 * When encountering a .Nm macro, substitute it 1030 * with its previously cached value of the argument. 1031 */ 1032 mdoc_parse_section(n->sec, rec->name, rec); 1033 } else if (mdocs[n->tok] == pmdoc_Xr) { 1034 /* 1035 * When encountering other inline macros, 1036 * call pmdoc_macro_handler. 1037 */ 1038 pmdoc_macro_handler(n, rec, MDOC_Xr); 1039 xr_found = 1; 1040 } else if (mdocs[n->tok] == pmdoc_Pp) { 1041 pmdoc_macro_handler(n, rec, MDOC_Pp); 1042 } 1043 1044 /* 1045 * If an Xr macro was encountered then the child node has 1046 * already been explored by pmdoc_macro_handler. 1047 */ 1048 if (xr_found == 0) 1049 pmdoc_Sh(n->child, rec); 1050 pmdoc_Sh(n->next, rec); 1051 } 1052 1053 /* 1054 * mdoc_parse_section-- 1055 * Utility function for parsing sections of the mdoc type pages. 1056 * Takes two params: 1057 * 1. sec is an enum which indicates the section in which we are present 1058 * 2. string is the string which we need to append to the secbuff for this 1059 * particular section. 1060 * The function appends string to the global section buffer and returns. 1061 */ 1062 static void 1063 mdoc_parse_section(enum mdoc_sec sec, const char *string, mandb_rec *rec) 1064 { 1065 /* 1066 * If the user specified the 'l' flag, then parse and store only the 1067 * NAME section. Ignore the rest. 1068 */ 1069 if (mflags.limit) 1070 return; 1071 1072 switch (sec) { 1073 case SEC_LIBRARY: 1074 append(&rec->lib, string); 1075 break; 1076 case SEC_RETURN_VALUES: 1077 append(&rec->return_vals, string); 1078 break; 1079 case SEC_ENVIRONMENT: 1080 append(&rec->env, string); 1081 break; 1082 case SEC_FILES: 1083 append(&rec->files, string); 1084 break; 1085 case SEC_EXIT_STATUS: 1086 append(&rec->exit_status, string); 1087 break; 1088 case SEC_DIAGNOSTICS: 1089 append(&rec->diagnostics, string); 1090 break; 1091 case SEC_ERRORS: 1092 append(&rec->errors, string); 1093 break; 1094 case SEC_NAME: 1095 case SEC_SYNOPSIS: 1096 case SEC_EXAMPLES: 1097 case SEC_STANDARDS: 1098 case SEC_HISTORY: 1099 case SEC_AUTHORS: 1100 case SEC_BUGS: 1101 break; 1102 default: 1103 append(&rec->desc, string); 1104 break; 1105 } 1106 } 1107 1108 static void 1109 pman_node(const struct man_node *n, mandb_rec *rec) 1110 { 1111 if (n == NULL) 1112 return; 1113 1114 switch (n->type) { 1115 case (MAN_BODY): 1116 /* FALLTHROUGH */ 1117 case (MAN_TAIL): 1118 /* FALLTHROUGH */ 1119 case (MAN_BLOCK): 1120 /* FALLTHROUGH */ 1121 case (MAN_ELEM): 1122 if (mans[n->tok] != NULL) 1123 (*mans[n->tok])(n, rec); 1124 break; 1125 default: 1126 break; 1127 } 1128 1129 pman_node(n->child, rec); 1130 pman_node(n->next, rec); 1131 } 1132 1133 /* 1134 * pman_parse_name -- 1135 * Parses the NAME section and puts the complete content in the name_desc 1136 * variable. 1137 */ 1138 static void 1139 pman_parse_name(const struct man_node *n, mandb_rec *rec) 1140 { 1141 if (n == NULL) 1142 return; 1143 1144 if (n->type == MAN_TEXT) { 1145 char *tmp = parse_escape(n->string); 1146 concat(&rec->name_desc, tmp); 1147 free(tmp); 1148 } 1149 1150 if (n->child) 1151 pman_parse_name(n->child, rec); 1152 1153 if(n->next) 1154 pman_parse_name(n->next, rec); 1155 } 1156 1157 /* 1158 * A stub function to be able to parse the macros like .B embedded inside 1159 * a section. 1160 */ 1161 static void 1162 pman_block(const struct man_node *n, mandb_rec *rec) 1163 { 1164 } 1165 1166 /* 1167 * pman_sh -- 1168 * This function does one of the two things: 1169 * 1. If the present section is NAME, then it will: 1170 * (a) Extract the name of the page (in case of multiple comma separated 1171 * names, it will pick up the first one). 1172 * (b) Build a space spearated list of all the symlinks/hardlinks to 1173 * this page and store in the buffer 'links'. These are extracted from 1174 * the comma separated list of names in the NAME section as well. 1175 * (c) Move on to the one line description section, which is after the list 1176 * of names in the NAME section. 1177 * 2. Otherwise, it will check the section name and call the man_parse_section 1178 * function, passing the enum corresponding that section. 1179 */ 1180 static void 1181 pman_sh(const struct man_node *n, mandb_rec *rec) 1182 { 1183 static const struct { 1184 enum man_sec section; 1185 const char *header; 1186 } mapping[] = { 1187 { MANSEC_DESCRIPTION, "DESCRIPTION" }, 1188 { MANSEC_SYNOPSIS, "SYNOPSIS" }, 1189 { MANSEC_LIBRARY, "LIBRARY" }, 1190 { MANSEC_ERRORS, "ERRORS" }, 1191 { MANSEC_FILES, "FILES" }, 1192 { MANSEC_RETURN_VALUES, "RETURN VALUE" }, 1193 { MANSEC_RETURN_VALUES, "RETURN VALUES" }, 1194 { MANSEC_EXIT_STATUS, "EXIT STATUS" }, 1195 { MANSEC_EXAMPLES, "EXAMPLES" }, 1196 { MANSEC_EXAMPLES, "EXAMPLE" }, 1197 { MANSEC_STANDARDS, "STANDARDS" }, 1198 { MANSEC_HISTORY, "HISTORY" }, 1199 { MANSEC_BUGS, "BUGS" }, 1200 { MANSEC_AUTHORS, "AUTHORS" }, 1201 { MANSEC_COPYRIGHT, "COPYRIGHT" }, 1202 }; 1203 const struct man_node *head; 1204 char *name_desc; 1205 int sz; 1206 size_t i; 1207 1208 if ((head = n->parent->head) == NULL || (head = head->child) == NULL || 1209 head->type != MAN_TEXT) 1210 return; 1211 1212 /* 1213 * Check if this section should be extracted and 1214 * where it should be stored. Handled the trival cases first. 1215 */ 1216 for (i = 0; i < sizeof(mapping) / sizeof(mapping[0]); ++i) { 1217 if (strcmp(head->string, mapping[i].header) == 0) { 1218 man_parse_section(mapping[i].section, n, rec); 1219 return; 1220 } 1221 } 1222 1223 if (strcmp(head->string, "NAME") == 0) { 1224 /* 1225 * We are in the NAME section. 1226 * pman_parse_name will put the complete content in name_desc. 1227 */ 1228 pman_parse_name(n, rec); 1229 1230 name_desc = rec->name_desc; 1231 1232 /* Remove any leading spaces. */ 1233 while (name_desc[0] == ' ') 1234 name_desc++; 1235 1236 /* If the line begins with a "\&", avoid those */ 1237 if (name_desc[0] == '\\' && name_desc[1] == '&') 1238 name_desc += 2; 1239 1240 /* Now name_desc should be left with a comma-space 1241 * separated list of names and the one line description 1242 * of the page: 1243 * "a, b, c \- sample description" 1244 * Take out the first name, before the first comma 1245 * (or space) and store it in rec->name. 1246 * If the page has aliases then they should be 1247 * in the form of a comma separated list. 1248 * Keep looping while there is a comma in name_desc, 1249 * extract the alias name and store in rec->links. 1250 * When there are no more commas left, break out. 1251 */ 1252 int has_alias = 0; // Any more aliases left? 1253 while (*name_desc) { 1254 /* Remove any leading spaces. */ 1255 if (name_desc[0] == ' ') { 1256 name_desc++; 1257 continue; 1258 } 1259 sz = strcspn(name_desc, ", "); 1260 1261 /* Extract the first term and store it in rec->name. */ 1262 if (rec->name == NULL) { 1263 if (name_desc[sz] == ',') 1264 has_alias = 1; 1265 name_desc[sz] = 0; 1266 rec->name = emalloc(sz + 1); 1267 memcpy(rec->name, name_desc, sz + 1); 1268 name_desc += sz + 1; 1269 continue; 1270 } 1271 1272 /* 1273 * Once rec->name is set, rest of the names 1274 * are to be treated as links or aliases. 1275 */ 1276 if (rec->name && has_alias) { 1277 if (name_desc[sz] != ',') { 1278 /* No more commas left --> 1279 * no more aliases to take out 1280 */ 1281 has_alias = 0; 1282 } 1283 name_desc[sz] = 0; 1284 concat2(&rec->links, name_desc, sz); 1285 name_desc += sz + 1; 1286 continue; 1287 } 1288 break; 1289 } 1290 1291 /* Parse any escape sequences that might be there */ 1292 char *temp = parse_escape(name_desc); 1293 free(rec->name_desc); 1294 rec->name_desc = temp; 1295 temp = parse_escape(rec->name); 1296 free(rec->name); 1297 rec->name = temp; 1298 return; 1299 } 1300 1301 /* The RETURN VALUE section might be specified in multiple ways */ 1302 if (strcmp(head->string, "RETURN") == 0 && 1303 head->next != NULL && head->next->type == MAN_TEXT && 1304 (strcmp(head->next->string, "VALUE") == 0 || 1305 strcmp(head->next->string, "VALUES") == 0)) { 1306 man_parse_section(MANSEC_RETURN_VALUES, n, rec); 1307 return; 1308 } 1309 1310 /* 1311 * EXIT STATUS section can also be specified all on one line or on two 1312 * separate lines. 1313 */ 1314 if (strcmp(head->string, "EXIT") == 0 && 1315 head->next != NULL && head->next->type == MAN_TEXT && 1316 strcmp(head->next->string, "STATUS") == 0) { 1317 man_parse_section(MANSEC_EXIT_STATUS, n, rec); 1318 return; 1319 } 1320 1321 /* Store the rest of the content in desc. */ 1322 man_parse_section(MANSEC_NONE, n, rec); 1323 } 1324 1325 /* 1326 * pman_parse_node -- 1327 * Generic function to iterate through a node. Usually called from 1328 * man_parse_section to parse a particular section of the man page. 1329 */ 1330 static void 1331 pman_parse_node(const struct man_node *n, secbuff *s) 1332 { 1333 if (n == NULL) 1334 return; 1335 1336 if (n->type == MAN_TEXT) 1337 append(s, n->string); 1338 1339 pman_parse_node(n->child, s); 1340 pman_parse_node(n->next, s); 1341 } 1342 1343 /* 1344 * man_parse_section -- 1345 * Takes two parameters: 1346 * sec: Tells which section we are present in 1347 * n: Is the present node of the AST. 1348 * Depending on the section, we call pman_parse_node to parse that section and 1349 * concatenate the content from that section into the buffer for that section. 1350 */ 1351 static void 1352 man_parse_section(enum man_sec sec, const struct man_node *n, mandb_rec *rec) 1353 { 1354 /* 1355 * If the user sepecified the 'l' flag then just parse 1356 * the NAME section, ignore the rest. 1357 */ 1358 if (mflags.limit) 1359 return; 1360 1361 switch (sec) { 1362 case MANSEC_LIBRARY: 1363 pman_parse_node(n, &rec->lib); 1364 break; 1365 case MANSEC_RETURN_VALUES: 1366 pman_parse_node(n, &rec->return_vals); 1367 break; 1368 case MANSEC_ENVIRONMENT: 1369 pman_parse_node(n, &rec->env); 1370 break; 1371 case MANSEC_FILES: 1372 pman_parse_node(n, &rec->files); 1373 break; 1374 case MANSEC_EXIT_STATUS: 1375 pman_parse_node(n, &rec->exit_status); 1376 break; 1377 case MANSEC_DIAGNOSTICS: 1378 pman_parse_node(n, &rec->diagnostics); 1379 break; 1380 case MANSEC_ERRORS: 1381 pman_parse_node(n, &rec->errors); 1382 break; 1383 case MANSEC_NAME: 1384 case MANSEC_SYNOPSIS: 1385 case MANSEC_EXAMPLES: 1386 case MANSEC_STANDARDS: 1387 case MANSEC_HISTORY: 1388 case MANSEC_BUGS: 1389 case MANSEC_AUTHORS: 1390 case MANSEC_COPYRIGHT: 1391 break; 1392 default: 1393 pman_parse_node(n, &rec->desc); 1394 break; 1395 } 1396 1397 } 1398 1399 /* 1400 * insert_into_db -- 1401 * Inserts the parsed data of the man page in the Sqlite databse. 1402 * If any of the values is NULL, then we cleanup and return -1 indicating 1403 * an error. 1404 * Otherwise, store the data in the database and return 0. 1405 */ 1406 static int 1407 insert_into_db(sqlite3 *db, mandb_rec *rec) 1408 { 1409 int rc = 0; 1410 int idx = -1; 1411 const char *sqlstr = NULL; 1412 sqlite3_stmt *stmt = NULL; 1413 char *ln = NULL; 1414 char *errmsg = NULL; 1415 long int mandb_rowid; 1416 1417 /* 1418 * At the very minimum we want to make sure that we store 1419 * the following data: 1420 * Name, one line description, and the MD5 hash 1421 */ 1422 if (rec->name == NULL || rec->name_desc == NULL || 1423 rec->md5_hash == NULL) { 1424 cleanup(rec); 1425 return -1; 1426 } 1427 1428 /* Write null byte at the end of all the sec_buffs */ 1429 rec->desc.data[rec->desc.offset] = 0; 1430 rec->lib.data[rec->lib.offset] = 0; 1431 rec->env.data[rec->env.offset] = 0; 1432 rec->return_vals.data[rec->return_vals.offset] = 0; 1433 rec->exit_status.data[rec->exit_status.offset] = 0; 1434 rec->files.data[rec->files.offset] = 0; 1435 rec->diagnostics.data[rec->diagnostics.offset] = 0; 1436 rec->errors.data[rec->errors.offset] = 0; 1437 1438 /* 1439 * In case of a mdoc page: (sorry, no better place to put this code) 1440 * parse the comma separated list of names of man pages, 1441 * the first name will be stored in the mandb table, rest will be 1442 * treated as links and put in the mandb_links table. 1443 */ 1444 if (rec->page_type == MDOC) { 1445 char *tmp; 1446 rec->links = estrdup(rec->name); 1447 free(rec->name); 1448 int sz = strcspn(rec->links, " \0"); 1449 rec->name = emalloc(sz + 1); 1450 memcpy(rec->name, rec->links, sz); 1451 if(rec->name[sz - 1] == ',') 1452 rec->name[sz - 1] = 0; 1453 else 1454 rec->name[sz] = 0; 1455 while (rec->links[sz] == ' ') 1456 ++sz; 1457 tmp = estrdup(rec->links + sz); 1458 free(rec->links); 1459 rec->links = tmp; 1460 } 1461 1462 /*------------------------ Populate the mandb table---------------------------*/ 1463 sqlstr = "INSERT INTO mandb VALUES (:section, :name, :name_desc, :desc," 1464 " :lib, :return_vals, :env, :files, :exit_status," 1465 " :diagnostics, :errors, :md5_hash, :machine)"; 1466 1467 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); 1468 if (rc != SQLITE_OK) 1469 goto Out; 1470 1471 idx = sqlite3_bind_parameter_index(stmt, ":name"); 1472 rc = sqlite3_bind_text(stmt, idx, rec->name, -1, NULL); 1473 if (rc != SQLITE_OK) { 1474 sqlite3_finalize(stmt); 1475 goto Out; 1476 } 1477 1478 idx = sqlite3_bind_parameter_index(stmt, ":section"); 1479 rc = sqlite3_bind_text(stmt, idx, rec->section, -1, NULL); 1480 if (rc != SQLITE_OK) { 1481 sqlite3_finalize(stmt); 1482 goto Out; 1483 } 1484 1485 idx = sqlite3_bind_parameter_index(stmt, ":name_desc"); 1486 rc = sqlite3_bind_text(stmt, idx, rec->name_desc, -1, NULL); 1487 if (rc != SQLITE_OK) { 1488 sqlite3_finalize(stmt); 1489 goto Out; 1490 } 1491 1492 idx = sqlite3_bind_parameter_index(stmt, ":desc"); 1493 rc = sqlite3_bind_text(stmt, idx, rec->desc.data, 1494 rec->desc.offset + 1, NULL); 1495 if (rc != SQLITE_OK) { 1496 sqlite3_finalize(stmt); 1497 goto Out; 1498 } 1499 1500 idx = sqlite3_bind_parameter_index(stmt, ":lib"); 1501 rc = sqlite3_bind_text(stmt, idx, rec->lib.data, rec->lib.offset + 1, NULL); 1502 if (rc != SQLITE_OK) { 1503 sqlite3_finalize(stmt); 1504 goto Out; 1505 } 1506 1507 idx = sqlite3_bind_parameter_index(stmt, ":return_vals"); 1508 rc = sqlite3_bind_text(stmt, idx, rec->return_vals.data, 1509 rec->return_vals.offset + 1, NULL); 1510 if (rc != SQLITE_OK) { 1511 sqlite3_finalize(stmt); 1512 goto Out; 1513 } 1514 1515 idx = sqlite3_bind_parameter_index(stmt, ":env"); 1516 rc = sqlite3_bind_text(stmt, idx, rec->env.data, rec->env.offset + 1, NULL); 1517 if (rc != SQLITE_OK) { 1518 sqlite3_finalize(stmt); 1519 goto Out; 1520 } 1521 1522 idx = sqlite3_bind_parameter_index(stmt, ":files"); 1523 rc = sqlite3_bind_text(stmt, idx, rec->files.data, 1524 rec->files.offset + 1, NULL); 1525 if (rc != SQLITE_OK) { 1526 sqlite3_finalize(stmt); 1527 goto Out; 1528 } 1529 1530 idx = sqlite3_bind_parameter_index(stmt, ":exit_status"); 1531 rc = sqlite3_bind_text(stmt, idx, rec->exit_status.data, 1532 rec->exit_status.offset + 1, NULL); 1533 if (rc != SQLITE_OK) { 1534 sqlite3_finalize(stmt); 1535 goto Out; 1536 } 1537 1538 idx = sqlite3_bind_parameter_index(stmt, ":diagnostics"); 1539 rc = sqlite3_bind_text(stmt, idx, rec->diagnostics.data, 1540 rec->diagnostics.offset + 1, NULL); 1541 if (rc != SQLITE_OK) { 1542 sqlite3_finalize(stmt); 1543 goto Out; 1544 } 1545 1546 idx = sqlite3_bind_parameter_index(stmt, ":errors"); 1547 rc = sqlite3_bind_text(stmt, idx, rec->errors.data, 1548 rec->errors.offset + 1, NULL); 1549 if (rc != SQLITE_OK) { 1550 sqlite3_finalize(stmt); 1551 goto Out; 1552 } 1553 1554 idx = sqlite3_bind_parameter_index(stmt, ":md5_hash"); 1555 rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL); 1556 if (rc != SQLITE_OK) { 1557 sqlite3_finalize(stmt); 1558 goto Out; 1559 } 1560 1561 idx = sqlite3_bind_parameter_index(stmt, ":machine"); 1562 if (rec->machine) 1563 rc = sqlite3_bind_text(stmt, idx, rec->machine, -1, NULL); 1564 else 1565 rc = sqlite3_bind_null(stmt, idx); 1566 if (rc != SQLITE_OK) { 1567 sqlite3_finalize(stmt); 1568 goto Out; 1569 } 1570 1571 rc = sqlite3_step(stmt); 1572 if (rc != SQLITE_DONE) { 1573 sqlite3_finalize(stmt); 1574 goto Out; 1575 } 1576 1577 sqlite3_finalize(stmt); 1578 1579 /* Get the row id of the last inserted row */ 1580 mandb_rowid = sqlite3_last_insert_rowid(db); 1581 1582 /*------------------------Populate the mandb_meta table-----------------------*/ 1583 sqlstr = "INSERT INTO mandb_meta VALUES (:device, :inode, :mtime," 1584 " :file, :md5_hash, :id)"; 1585 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); 1586 if (rc != SQLITE_OK) 1587 goto Out; 1588 1589 idx = sqlite3_bind_parameter_index(stmt, ":device"); 1590 rc = sqlite3_bind_int64(stmt, idx, rec->device); 1591 if (rc != SQLITE_OK) { 1592 sqlite3_finalize(stmt); 1593 goto Out; 1594 } 1595 1596 idx = sqlite3_bind_parameter_index(stmt, ":inode"); 1597 rc = sqlite3_bind_int64(stmt, idx, rec->inode); 1598 if (rc != SQLITE_OK) { 1599 sqlite3_finalize(stmt); 1600 goto Out; 1601 } 1602 1603 idx = sqlite3_bind_parameter_index(stmt, ":mtime"); 1604 rc = sqlite3_bind_int64(stmt, idx, rec->mtime); 1605 if (rc != SQLITE_OK) { 1606 sqlite3_finalize(stmt); 1607 goto Out; 1608 } 1609 1610 idx = sqlite3_bind_parameter_index(stmt, ":file"); 1611 rc = sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL); 1612 if (rc != SQLITE_OK) { 1613 sqlite3_finalize(stmt); 1614 goto Out; 1615 } 1616 1617 idx = sqlite3_bind_parameter_index(stmt, ":md5_hash"); 1618 rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL); 1619 if (rc != SQLITE_OK) { 1620 sqlite3_finalize(stmt); 1621 goto Out; 1622 } 1623 1624 idx = sqlite3_bind_parameter_index(stmt, ":id"); 1625 rc = sqlite3_bind_int64(stmt, idx, mandb_rowid); 1626 if (rc != SQLITE_OK) { 1627 sqlite3_finalize(stmt); 1628 goto Out; 1629 } 1630 1631 rc = sqlite3_step(stmt); 1632 sqlite3_finalize(stmt); 1633 if (rc == SQLITE_CONSTRAINT) { 1634 /* The *most* probable reason for reaching here is that 1635 * the UNIQUE contraint on the file column of the mandb_meta 1636 * table was violated. 1637 * This can happen when a file was updated/modified. 1638 * To fix this we need to do two things: 1639 * 1. Delete the row for the older version of this file 1640 * from mandb table. 1641 * 2. Run an UPDATE query to update the row for this file 1642 * in the mandb_meta table. 1643 */ 1644 warnx("Trying to update index for %s", rec->file_path); 1645 char *sql = sqlite3_mprintf("DELETE FROM mandb " 1646 "WHERE rowid = (SELECT id" 1647 " FROM mandb_meta" 1648 " WHERE file = %Q)", 1649 rec->file_path); 1650 sqlite3_exec(db, sql, NULL, NULL, &errmsg); 1651 sqlite3_free(sql); 1652 if (errmsg != NULL) { 1653 warnx("%s", errmsg); 1654 free(errmsg); 1655 } 1656 sqlstr = "UPDATE mandb_meta SET device = :device," 1657 " inode = :inode, mtime = :mtime, id = :id," 1658 " md5_hash = :md5 WHERE file = :file"; 1659 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); 1660 if (rc != SQLITE_OK) { 1661 warnx("Update failed with error: %s", 1662 sqlite3_errmsg(db)); 1663 close_db(db); 1664 cleanup(rec); 1665 errx(EXIT_FAILURE, 1666 "Consider running makemandb with -f option"); 1667 } 1668 1669 idx = sqlite3_bind_parameter_index(stmt, ":device"); 1670 sqlite3_bind_int64(stmt, idx, rec->device); 1671 idx = sqlite3_bind_parameter_index(stmt, ":inode"); 1672 sqlite3_bind_int64(stmt, idx, rec->inode); 1673 idx = sqlite3_bind_parameter_index(stmt, ":mtime"); 1674 sqlite3_bind_int64(stmt, idx, rec->mtime); 1675 idx = sqlite3_bind_parameter_index(stmt, ":id"); 1676 sqlite3_bind_int64(stmt, idx, mandb_rowid); 1677 idx = sqlite3_bind_parameter_index(stmt, ":md5"); 1678 sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL); 1679 idx = sqlite3_bind_parameter_index(stmt, ":file"); 1680 sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL); 1681 rc = sqlite3_step(stmt); 1682 sqlite3_finalize(stmt); 1683 1684 if (rc != SQLITE_DONE) { 1685 warnx("%s", sqlite3_errmsg(db)); 1686 close_db(db); 1687 cleanup(rec); 1688 errx(EXIT_FAILURE, 1689 "Consider running makemandb with -f option"); 1690 } 1691 } else if (rc != SQLITE_DONE) { 1692 /* Otherwise make this error fatal */ 1693 warnx("Failed at %s\n%s", rec->file_path, sqlite3_errmsg(db)); 1694 cleanup(rec); 1695 close_db(db); 1696 exit(EXIT_FAILURE); 1697 } 1698 1699 /*------------------------ Populate the mandb_links table---------------------*/ 1700 char *str = NULL; 1701 char *links; 1702 if (rec->links && strlen(rec->links)) { 1703 links = rec->links; 1704 for(ln = strtok(links, " "); ln; ln = strtok(NULL, " ")) { 1705 if (ln[0] == ',') 1706 ln++; 1707 if(ln[strlen(ln) - 1] == ',') 1708 ln[strlen(ln) - 1] = 0; 1709 1710 str = sqlite3_mprintf("INSERT INTO mandb_links" 1711 " VALUES (%Q, %Q, %Q, %Q)", 1712 ln, rec->name, rec->section, 1713 rec->machine); 1714 sqlite3_exec(db, str, NULL, NULL, &errmsg); 1715 sqlite3_free(str); 1716 if (errmsg != NULL) { 1717 warnx("%s", errmsg); 1718 cleanup(rec); 1719 free(errmsg); 1720 return -1; 1721 } 1722 } 1723 } 1724 1725 cleanup(rec); 1726 return 0; 1727 1728 Out: 1729 warnx("%s", sqlite3_errmsg(db)); 1730 cleanup(rec); 1731 return -1; 1732 } 1733 1734 /* 1735 * check_md5-- 1736 * Generates the md5 hash of the file and checks if it already doesn't exist 1737 * in the table (passed as the 3rd parameter). 1738 * This function is being used to avoid hardlinks. 1739 * On successful completion it will also set the value of the fourth parameter 1740 * to the md5 hash of the file (computed previously). It is the responsibility 1741 * of the caller to free this buffer. 1742 * Return values: 1743 * -1: If an error occurs somewhere and sets the md5 return buffer to NULL. 1744 * 0: If the md5 hash does not exist in the table. 1745 * 1: If the hash exists in the database. 1746 */ 1747 static int 1748 check_md5(const char *file, sqlite3 *db, const char *table, char **md5sum, 1749 void *buf, size_t buflen) 1750 { 1751 int rc = 0; 1752 int idx = -1; 1753 char *sqlstr = NULL; 1754 sqlite3_stmt *stmt = NULL; 1755 1756 assert(file != NULL); 1757 *md5sum = MD5Data(buf, buflen, NULL); 1758 if (*md5sum == NULL) { 1759 warn("md5 failed: %s", file); 1760 return -1; 1761 } 1762 1763 easprintf(&sqlstr, "SELECT * FROM %s WHERE md5_hash = :md5_hash", 1764 table); 1765 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); 1766 if (rc != SQLITE_OK) { 1767 free(sqlstr); 1768 free(*md5sum); 1769 *md5sum = NULL; 1770 return -1; 1771 } 1772 1773 idx = sqlite3_bind_parameter_index(stmt, ":md5_hash"); 1774 rc = sqlite3_bind_text(stmt, idx, *md5sum, -1, NULL); 1775 if (rc != SQLITE_OK) { 1776 warnx("%s", sqlite3_errmsg(db)); 1777 sqlite3_finalize(stmt); 1778 free(sqlstr); 1779 free(*md5sum); 1780 *md5sum = NULL; 1781 return -1; 1782 } 1783 1784 if (sqlite3_step(stmt) == SQLITE_ROW) { 1785 sqlite3_finalize(stmt); 1786 free(sqlstr); 1787 return 0; 1788 } 1789 1790 sqlite3_finalize(stmt); 1791 free(sqlstr); 1792 return 1; 1793 } 1794 1795 /* Optimize the index for faster search */ 1796 static void 1797 optimize(sqlite3 *db) 1798 { 1799 const char *sqlstr; 1800 char *errmsg = NULL; 1801 1802 if (mflags.verbosity) 1803 printf("Optimizing the database index\n"); 1804 sqlstr = "INSERT INTO mandb(mandb) VALUES (\'optimize\');" 1805 "VACUUM"; 1806 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg); 1807 if (errmsg != NULL) { 1808 warnx("%s", errmsg); 1809 free(errmsg); 1810 return; 1811 } 1812 } 1813 1814 /* 1815 * cleanup -- 1816 * cleans up the global buffers 1817 */ 1818 static void 1819 cleanup(mandb_rec *rec) 1820 { 1821 rec->desc.offset = 0; 1822 rec->lib.offset = 0; 1823 rec->return_vals.offset = 0; 1824 rec->env.offset = 0; 1825 rec->exit_status.offset = 0; 1826 rec->diagnostics.offset = 0; 1827 rec->errors.offset = 0; 1828 rec->files.offset = 0; 1829 1830 free(rec->machine); 1831 rec->machine = NULL; 1832 1833 free(rec->links); 1834 rec->links = NULL; 1835 1836 free(rec->file_path); 1837 rec->file_path = NULL; 1838 1839 free(rec->name); 1840 rec->name = NULL; 1841 1842 free(rec->name_desc); 1843 rec->name_desc = NULL; 1844 1845 free(rec->md5_hash); 1846 rec->md5_hash = NULL; 1847 } 1848 1849 /* 1850 * init_secbuffs-- 1851 * Sets the value of buflen for all the sec_buff field of rec. And then 1852 * allocate memory to each sec_buff member of rec. 1853 */ 1854 static void 1855 init_secbuffs(mandb_rec *rec) 1856 { 1857 /* 1858 * Some sec_buff might need more memory, for example desc, 1859 * which stores the data of the DESCRIPTION section, 1860 * while some might need very small amount of memory. 1861 * Therefore explicitly setting the value of buflen field for 1862 * each sec_buff. 1863 */ 1864 rec->desc.buflen = 10 * BUFLEN; 1865 rec->desc.data = emalloc(rec->desc.buflen); 1866 rec->desc.offset = 0; 1867 1868 rec->lib.buflen = BUFLEN / 2; 1869 rec->lib.data = emalloc(rec->lib.buflen); 1870 rec->lib.offset = 0; 1871 1872 rec->return_vals.buflen = BUFLEN; 1873 rec->return_vals.data = emalloc(rec->return_vals.buflen); 1874 rec->return_vals.offset = 0; 1875 1876 rec->exit_status.buflen = BUFLEN; 1877 rec->exit_status.data = emalloc(rec->exit_status.buflen); 1878 rec->exit_status.offset = 0; 1879 1880 rec->env.buflen = BUFLEN; 1881 rec->env.data = emalloc(rec->env.buflen); 1882 rec->env.offset = 0; 1883 1884 rec->files.buflen = BUFLEN; 1885 rec->files.data = emalloc(rec->files.buflen); 1886 rec->files.offset = 0; 1887 1888 rec->diagnostics.buflen = BUFLEN; 1889 rec->diagnostics.data = emalloc(rec->diagnostics.buflen); 1890 rec->diagnostics.offset = 0; 1891 1892 rec->errors.buflen = BUFLEN; 1893 rec->errors.data = emalloc(rec->errors.buflen); 1894 rec->errors.offset = 0; 1895 } 1896 1897 /* 1898 * free_secbuffs-- 1899 * This function should be called at the end, when all the pages have been 1900 * parsed. 1901 * It frees the memory allocated to sec_buffs by init_secbuffs in the starting. 1902 */ 1903 static void 1904 free_secbuffs(mandb_rec *rec) 1905 { 1906 free(rec->desc.data); 1907 free(rec->lib.data); 1908 free(rec->return_vals.data); 1909 free(rec->exit_status.data); 1910 free(rec->env.data); 1911 free(rec->files.data); 1912 free(rec->diagnostics.data); 1913 free(rec->errors.data); 1914 } 1915 1916 static void 1917 replace_hyph(char *str) 1918 { 1919 char *iter = str; 1920 while ((iter = strchr(iter, ASCII_HYPH)) != NULL) 1921 *iter = '-'; 1922 } 1923 1924 static char * 1925 parse_escape(const char *str) 1926 { 1927 const char *backslash, *last_backslash; 1928 char *result, *iter; 1929 size_t len; 1930 1931 assert(str); 1932 1933 last_backslash = str; 1934 backslash = strchr(str, '\\'); 1935 if (backslash == NULL) { 1936 result = estrdup(str); 1937 replace_hyph(result); 1938 return result; 1939 } 1940 1941 result = emalloc(strlen(str) + 1); 1942 iter = result; 1943 1944 do { 1945 len = backslash - last_backslash; 1946 memcpy(iter, last_backslash, len); 1947 iter += len; 1948 if (backslash[1] == '-' || backslash[1] == ' ') { 1949 *iter++ = backslash[1]; 1950 last_backslash = backslash + 2; 1951 backslash = strchr(backslash + 2, '\\'); 1952 } else { 1953 ++backslash; 1954 mandoc_escape(&backslash, NULL, NULL); 1955 last_backslash = backslash; 1956 if (backslash == NULL) 1957 break; 1958 backslash = strchr(last_backslash, '\\'); 1959 } 1960 } while (backslash != NULL); 1961 if (last_backslash != NULL) 1962 strcpy(iter, last_backslash); 1963 1964 replace_hyph(result); 1965 return result; 1966 } 1967 1968 /* 1969 * append-- 1970 * Concatenates a space and src at the end of sbuff->data (much like concat in 1971 * apropos-utils.c). 1972 * Rather than reallocating space for writing data, it uses the value of the 1973 * offset field of sec_buff to write new data at the free space left in the 1974 * buffer. 1975 * In case the size of the data to be appended exceeds the number of bytes left 1976 * in the buffer, it reallocates buflen number of bytes and then continues. 1977 * Value of offset field should be adjusted as new data is written. 1978 * 1979 * NOTE: This function does not write the null byte at the end of the buffers, 1980 * write a null byte at the position pointed to by offset before inserting data 1981 * in the db. 1982 */ 1983 static void 1984 append(secbuff *sbuff, const char *src) 1985 { 1986 short flag = 0; 1987 size_t srclen, newlen; 1988 char *temp; 1989 1990 assert(src != NULL); 1991 temp = parse_escape(src); 1992 srclen = strlen(temp); 1993 1994 if (sbuff->data == NULL) { 1995 sbuff->data = emalloc(sbuff->buflen); 1996 sbuff->offset = 0; 1997 } 1998 1999 newlen = sbuff->offset + srclen + 2; 2000 if (newlen >= sbuff->buflen) { 2001 while (sbuff->buflen < newlen) 2002 sbuff->buflen += sbuff->buflen; 2003 sbuff->data = erealloc(sbuff->data, sbuff->buflen); 2004 flag = 1; 2005 } 2006 2007 /* Append a space at the end of the buffer. */ 2008 if (sbuff->offset || flag) 2009 sbuff->data[sbuff->offset++] = ' '; 2010 /* Now, copy src at the end of the buffer. */ 2011 memcpy(sbuff->data + sbuff->offset, temp, srclen); 2012 sbuff->offset += srclen; 2013 free(temp); 2014 } 2015 2016 static void 2017 usage(void) 2018 { 2019 fprintf(stderr, "Usage: %s [-flo]\n", getprogname()); 2020 exit(1); 2021 } 2022
Indexes created Mon Sep 29 21:09:56 GMT 2025