makemandb.c revision 1.13
1 /* $NetBSD: makemandb.c,v 1.13 2012/08/29 20:33:01 wiz Exp $ */ 2 /* 3 * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay (at) gmail.com> 4 * Copyright (c) 2011 Kristaps Dzonsons <kristaps (at) bsd.lv> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/cdefs.h> 20 __RCSID("$NetBSD: makemandb.c,v 1.13 2012/08/29 20:33:01 wiz Exp $"); 21 22 #include <sys/stat.h> 23 #include <sys/types.h> 24 25 #include <assert.h> 26 #include <ctype.h> 27 #include <dirent.h> 28 #include <err.h> 29 #include <archive.h> 30 #include <libgen.h> 31 #include <md5.h> 32 #include <stdio.h> 33 #include <stdlib.h> 34 #include <string.h> 35 #include <unistd.h> 36 #include <util.h> 37 38 #include "apropos-utils.h" 39 #include "man.h" 40 #include "mandoc.h" 41 #include "mdoc.h" 42 #include "sqlite3.h" 43 44 #define BUFLEN 1024 45 #define MDOC 0 //If the page is of mdoc(7) type 46 #define MAN 1 //If the page is of man(7) type 47 48 /* 49 * A data structure for holding section specific data. 50 */ 51 typedef struct secbuff { 52 char *data; 53 size_t buflen; //Total length of buffer allocated initially 54 size_t offset; // Current offset in the buffer. 55 } secbuff; 56 57 typedef struct makemandb_flags { 58 int optimize; 59 int limit; // limit the indexing to only NAME section 60 int recreate; // Database was created from scratch 61 int verbosity; // 0: quiet, 1: default, 2: verbose 62 } makemandb_flags; 63 64 typedef struct mandb_rec { 65 /* Fields for mandb table */ 66 char *name; // for storing the name of the man page 67 char *name_desc; // for storing the one line description (.Nd) 68 secbuff desc; // for storing the DESCRIPTION section 69 secbuff lib; // for the LIBRARY section 70 secbuff return_vals; // RETURN VALUES 71 secbuff env; // ENVIRONMENT 72 secbuff files; // FILES 73 secbuff exit_status; // EXIT STATUS 74 secbuff diagnostics; // DIAGNOSTICS 75 secbuff errors; // ERRORS 76 char section[2]; 77 78 int xr_found; 79 80 /* Fields for mandb_meta table */ 81 char *md5_hash; 82 dev_t device; 83 ino_t inode; 84 time_t mtime; 85 86 /* Fields for mandb_links table */ 87 char *machine; 88 char *links; //all the links to a page in a space separated form 89 char *file_path; 90 91 /* Non-db fields */ 92 int page_type; //Indicates the type of page: mdoc or man 93 } mandb_rec; 94 95 static void append(secbuff *sbuff, const char *src); 96 static void init_secbuffs(mandb_rec *); 97 static void free_secbuffs(mandb_rec *); 98 static int check_md5(const char *, sqlite3 *, const char *, char **, void *, size_t); 99 static void cleanup(mandb_rec *); 100 static void set_section(const struct mdoc *, const struct man *, mandb_rec *); 101 static void set_machine(const struct mdoc *, mandb_rec *); 102 static int insert_into_db(sqlite3 *, mandb_rec *); 103 static void begin_parse(const char *, struct mparse *, mandb_rec *, 104 const void *, size_t len); 105 static void pmdoc_node(const struct mdoc_node *, mandb_rec *); 106 static void pmdoc_Nm(const struct mdoc_node *, mandb_rec *); 107 static void pmdoc_Nd(const struct mdoc_node *, mandb_rec *); 108 static void pmdoc_Sh(const struct mdoc_node *, mandb_rec *); 109 static void pmdoc_Xr(const struct mdoc_node *, mandb_rec *); 110 static void pmdoc_Pp(const struct mdoc_node *, mandb_rec *); 111 static void pmdoc_macro_handler(const struct mdoc_node *, mandb_rec *, 112 enum mdoct); 113 static void pman_node(const struct man_node *n, mandb_rec *); 114 static void pman_parse_node(const struct man_node *, secbuff *); 115 static void pman_parse_name(const struct man_node *, mandb_rec *); 116 static void pman_sh(const struct man_node *, mandb_rec *); 117 static void pman_block(const struct man_node *, mandb_rec *); 118 static void traversedir(const char *, const char *, sqlite3 *, struct mparse *); 119 static void mdoc_parse_section(enum mdoc_sec, const char *, mandb_rec *); 120 static void man_parse_section(enum man_sec, const struct man_node *, mandb_rec *); 121 static void build_file_cache(sqlite3 *, const char *, const char *, 122 struct stat *); 123 static void update_db(sqlite3 *, struct mparse *, mandb_rec *); 124 __dead static void usage(void); 125 static void optimize(sqlite3 *); 126 static char *parse_escape(const char *); 127 static makemandb_flags mflags = { .verbosity = 1 }; 128 129 typedef void (*pman_nf)(const struct man_node *n, mandb_rec *); 130 typedef void (*pmdoc_nf)(const struct mdoc_node *n, mandb_rec *); 131 static const pmdoc_nf mdocs[MDOC_MAX] = { 132 NULL, /* Ap */ 133 NULL, /* Dd */ 134 NULL, /* Dt */ 135 NULL, /* Os */ 136 pmdoc_Sh, /* Sh */ 137 NULL, /* Ss */ 138 pmdoc_Pp, /* Pp */ 139 NULL, /* D1 */ 140 NULL, /* Dl */ 141 NULL, /* Bd */ 142 NULL, /* Ed */ 143 NULL, /* Bl */ 144 NULL, /* El */ 145 NULL, /* It */ 146 NULL, /* Ad */ 147 NULL, /* An */ 148 NULL, /* Ar */ 149 NULL, /* Cd */ 150 NULL, /* Cm */ 151 NULL, /* Dv */ 152 NULL, /* Er */ 153 NULL, /* Ev */ 154 NULL, /* Ex */ 155 NULL, /* Fa */ 156 NULL, /* Fd */ 157 NULL, /* Fl */ 158 NULL, /* Fn */ 159 NULL, /* Ft */ 160 NULL, /* Ic */ 161 NULL, /* In */ 162 NULL, /* Li */ 163 pmdoc_Nd, /* Nd */ 164 pmdoc_Nm, /* Nm */ 165 NULL, /* Op */ 166 NULL, /* Ot */ 167 NULL, /* Pa */ 168 NULL, /* Rv */ 169 NULL, /* St */ 170 NULL, /* Va */ 171 NULL, /* Vt */ 172 pmdoc_Xr, /* Xr */ 173 NULL, /* %A */ 174 NULL, /* %B */ 175 NULL, /* %D */ 176 NULL, /* %I */ 177 NULL, /* %J */ 178 NULL, /* %N */ 179 NULL, /* %O */ 180 NULL, /* %P */ 181 NULL, /* %R */ 182 NULL, /* %T */ 183 NULL, /* %V */ 184 NULL, /* Ac */ 185 NULL, /* Ao */ 186 NULL, /* Aq */ 187 NULL, /* At */ 188 NULL, /* Bc */ 189 NULL, /* Bf */ 190 NULL, /* Bo */ 191 NULL, /* Bq */ 192 NULL, /* Bsx */ 193 NULL, /* Bx */ 194 NULL, /* Db */ 195 NULL, /* Dc */ 196 NULL, /* Do */ 197 NULL, /* Dq */ 198 NULL, /* Ec */ 199 NULL, /* Ef */ 200 NULL, /* Em */ 201 NULL, /* Eo */ 202 NULL, /* Fx */ 203 NULL, /* Ms */ 204 NULL, /* No */ 205 NULL, /* Ns */ 206 NULL, /* Nx */ 207 NULL, /* Ox */ 208 NULL, /* Pc */ 209 NULL, /* Pf */ 210 NULL, /* Po */ 211 NULL, /* Pq */ 212 NULL, /* Qc */ 213 NULL, /* Ql */ 214 NULL, /* Qo */ 215 NULL, /* Qq */ 216 NULL, /* Re */ 217 NULL, /* Rs */ 218 NULL, /* Sc */ 219 NULL, /* So */ 220 NULL, /* Sq */ 221 NULL, /* Sm */ 222 NULL, /* Sx */ 223 NULL, /* Sy */ 224 NULL, /* Tn */ 225 NULL, /* Ux */ 226 NULL, /* Xc */ 227 NULL, /* Xo */ 228 NULL, /* Fo */ 229 NULL, /* Fc */ 230 NULL, /* Oo */ 231 NULL, /* Oc */ 232 NULL, /* Bk */ 233 NULL, /* Ek */ 234 NULL, /* Bt */ 235 NULL, /* Hf */ 236 NULL, /* Fr */ 237 NULL, /* Ud */ 238 NULL, /* Lb */ 239 NULL, /* Lp */ 240 NULL, /* Lk */ 241 NULL, /* Mt */ 242 NULL, /* Brq */ 243 NULL, /* Bro */ 244 NULL, /* Brc */ 245 NULL, /* %C */ 246 NULL, /* Es */ 247 NULL, /* En */ 248 NULL, /* Dx */ 249 NULL, /* %Q */ 250 NULL, /* br */ 251 NULL, /* sp */ 252 NULL, /* %U */ 253 NULL, /* Ta */ 254 }; 255 256 static const pman_nf mans[MAN_MAX] = { 257 NULL, //br 258 NULL, //TH 259 pman_sh, //SH 260 NULL, //SS 261 NULL, //TP 262 NULL, //LP 263 NULL, //PP 264 NULL, //P 265 NULL, //IP 266 NULL, //HP 267 NULL, //SM 268 NULL, //SB 269 NULL, //BI 270 NULL, //IB 271 NULL, //BR 272 NULL, //RB 273 NULL, //R 274 pman_block, //B 275 NULL, //I 276 NULL, //IR 277 NULL, //RI 278 NULL, //na 279 NULL, //sp 280 NULL, //nf 281 NULL, //fi 282 NULL, //RE 283 NULL, //RS 284 NULL, //DT 285 NULL, //UC 286 NULL, //PD 287 NULL, //AT 288 NULL, //in 289 NULL, //ft 290 }; 291 292 293 int 294 main(int argc, char *argv[]) 295 { 296 FILE *file; 297 const char *sqlstr, *manconf = NULL; 298 char *line, *command, *parent; 299 char *errmsg; 300 int ch; 301 struct mparse *mp; 302 sqlite3 *db; 303 ssize_t len; 304 size_t linesize; 305 struct mandb_rec rec; 306 307 while ((ch = getopt(argc, argv, "C:floQqv")) != -1) { 308 switch (ch) { 309 case 'C': 310 manconf = optarg; 311 break; 312 case 'f': 313 remove(DBPATH); 314 mflags.recreate = 1; 315 break; 316 case 'l': 317 mflags.limit = 1; 318 break; 319 case 'o': 320 mflags.optimize = 1; 321 break; 322 case 'Q': 323 mflags.verbosity = 0; 324 break; 325 case 'q': 326 mflags.verbosity = 1; 327 break; 328 case 'v': 329 mflags.verbosity = 2; 330 break; 331 default: 332 usage(); 333 } 334 } 335 336 memset(&rec, 0, sizeof(rec)); 337 338 init_secbuffs(&rec); 339 mp = mparse_alloc(MPARSE_AUTO, MANDOCLEVEL_FATAL, NULL, NULL); 340 341 if ((db = init_db(MANDB_CREATE)) == NULL) 342 exit(EXIT_FAILURE); 343 344 sqlite3_exec(db, "PRAGMA synchronous = 0", NULL, NULL, &errmsg); 345 if (errmsg != NULL) { 346 warnx("%s", errmsg); 347 free(errmsg); 348 close_db(db); 349 exit(EXIT_FAILURE); 350 } 351 352 sqlite3_exec(db, "ATTACH DATABASE \':memory:\' AS metadb", NULL, NULL, 353 &errmsg); 354 if (errmsg != NULL) { 355 warnx("%s", errmsg); 356 free(errmsg); 357 close_db(db); 358 exit(EXIT_FAILURE); 359 } 360 361 if (manconf) { 362 char *arg; 363 size_t command_len = shquote(manconf, NULL, 0) + 1; 364 arg = malloc(command_len ); 365 shquote(manconf, arg, command_len); 366 easprintf(&command, "man -p -C %s", arg); 367 free(arg); 368 } else { 369 command = estrdup("man -p"); 370 } 371 372 /* Call man -p to get the list of man page dirs */ 373 if ((file = popen(command, "r")) == NULL) { 374 close_db(db); 375 err(EXIT_FAILURE, "fopen failed"); 376 } 377 free(command); 378 379 /* Begin the transaction for indexing the pages */ 380 sqlite3_exec(db, "BEGIN", NULL, NULL, &errmsg); 381 if (errmsg != NULL) { 382 warnx("%s", errmsg); 383 free(errmsg); 384 exit(EXIT_FAILURE); 385 } 386 387 sqlstr = "CREATE TABLE metadb.file_cache(device, inode, mtime, parent," 388 " file PRIMARY KEY);" 389 "CREATE UNIQUE INDEX metadb.index_file_cache_dev" 390 " ON file_cache (device, inode)"; 391 392 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg); 393 if (errmsg != NULL) { 394 warnx("%s", errmsg); 395 free(errmsg); 396 close_db(db); 397 exit(EXIT_FAILURE); 398 } 399 400 if (mflags.verbosity) 401 printf("Building temporary file cache\n"); 402 line = NULL; 403 linesize = 0; 404 while ((len = getline(&line, &linesize, file)) != -1) { 405 /* Replace the new line character at the end of string with '\0' */ 406 line[len - 1] = '\0'; 407 parent = estrdup(line); 408 char *pdir = estrdup(dirname(parent)); 409 free(parent); 410 /* Traverse the man page directories and parse the pages */ 411 traversedir(pdir, line, db, mp); 412 free(pdir); 413 } 414 free(line); 415 416 if (pclose(file) == -1) { 417 close_db(db); 418 cleanup(&rec); 419 free_secbuffs(&rec); 420 err(EXIT_FAILURE, "pclose error"); 421 } 422 423 if (mflags.verbosity) 424 printf("Performing index update\n"); 425 update_db(db, mp, &rec); 426 mparse_free(mp); 427 free_secbuffs(&rec); 428 429 /* Commit the transaction */ 430 sqlite3_exec(db, "COMMIT", NULL, NULL, &errmsg); 431 if (errmsg != NULL) { 432 warnx("%s", errmsg); 433 free(errmsg); 434 exit(EXIT_FAILURE); 435 } 436 437 if (mflags.optimize) 438 optimize(db); 439 440 close_db(db); 441 return 0; 442 } 443 444 /* 445 * traversedir -- 446 * Traverses the given directory recursively and passes all the man page files 447 * in the way to build_file_cache() 448 */ 449 static void 450 traversedir(const char *parent, const char *file, sqlite3 *db, 451 struct mparse *mp) 452 { 453 struct stat sb; 454 struct dirent *dirp; 455 DIR *dp; 456 char *buf; 457 458 if (stat(file, &sb) < 0) { 459 if (mflags.verbosity) 460 warn("stat failed: %s", file); 461 return; 462 } 463 464 /* If it is a regular file or a symlink, pass it to build_cache() */ 465 if (S_ISREG(sb.st_mode) || S_ISLNK(sb.st_mode)) { 466 build_file_cache(db, parent, file, &sb); 467 return; 468 } 469 470 /* If it is a directory, traverse it recursively */ 471 if (S_ISDIR(sb.st_mode)) { 472 if ((dp = opendir(file)) == NULL) { 473 if (mflags.verbosity) 474 warn("opendir error: %s", file); 475 return; 476 } 477 478 while ((dirp = readdir(dp)) != NULL) { 479 /* Avoid . and .. entries in a directory */ 480 if (strncmp(dirp->d_name, ".", 1)) { 481 easprintf(&buf, "%s/%s", file, dirp->d_name); 482 traversedir(parent, buf, db, mp); 483 free(buf); 484 } 485 } 486 closedir(dp); 487 } 488 } 489 490 /* build_file_cache -- 491 * This function generates an md5 hash of the file passed as it's 2nd parameter 492 * and stores it in a temporary table file_cache along with the full file path. 493 * This is done to support incremental updation of the database. 494 * The temporary table file_cache is dropped thereafter in the function 495 * update_db(), once the database has been updated. 496 */ 497 static void 498 build_file_cache(sqlite3 *db, const char *parent, const char *file, 499 struct stat *sb) 500 { 501 const char *sqlstr; 502 sqlite3_stmt *stmt = NULL; 503 int rc, idx; 504 assert(file != NULL); 505 dev_t device_cache = sb->st_dev; 506 ino_t inode_cache = sb->st_ino; 507 time_t mtime_cache = sb->st_mtime; 508 509 sqlstr = "INSERT INTO metadb.file_cache VALUES (:device, :inode," 510 " :mtime, :parent, :file)"; 511 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); 512 if (rc != SQLITE_OK) { 513 if (mflags.verbosity) 514 warnx("%s", sqlite3_errmsg(db)); 515 return; 516 } 517 518 idx = sqlite3_bind_parameter_index(stmt, ":device"); 519 rc = sqlite3_bind_int64(stmt, idx, device_cache); 520 if (rc != SQLITE_OK) { 521 if (mflags.verbosity) 522 warnx("%s", sqlite3_errmsg(db)); 523 sqlite3_finalize(stmt); 524 return; 525 } 526 527 idx = sqlite3_bind_parameter_index(stmt, ":inode"); 528 rc = sqlite3_bind_int64(stmt, idx, inode_cache); 529 if (rc != SQLITE_OK) { 530 if (mflags.verbosity) 531 warnx("%s", sqlite3_errmsg(db)); 532 sqlite3_finalize(stmt); 533 return; 534 } 535 536 idx = sqlite3_bind_parameter_index(stmt, ":mtime"); 537 rc = sqlite3_bind_int64(stmt, idx, mtime_cache); 538 if (rc != SQLITE_OK) { 539 if (mflags.verbosity) 540 warnx("%s", sqlite3_errmsg(db)); 541 sqlite3_finalize(stmt); 542 return; 543 } 544 545 idx = sqlite3_bind_parameter_index(stmt, ":parent"); 546 rc = sqlite3_bind_text(stmt, idx, parent, -1, NULL); 547 if (rc != SQLITE_OK) { 548 if (mflags.verbosity) 549 warnx("%s", sqlite3_errmsg(db)); 550 sqlite3_finalize(stmt); 551 return; 552 } 553 554 idx = sqlite3_bind_parameter_index(stmt, ":file"); 555 rc = sqlite3_bind_text(stmt, idx, file, -1, NULL); 556 if (rc != SQLITE_OK) { 557 if (mflags.verbosity) 558 warnx("%s", sqlite3_errmsg(db)); 559 sqlite3_finalize(stmt); 560 return; 561 } 562 563 sqlite3_step(stmt); 564 sqlite3_finalize(stmt); 565 } 566 567 static void 568 update_existing_entry(sqlite3 *db, const char *file, const char *hash, 569 mandb_rec *rec, int *new_count, int *link_count, int *err_count) 570 { 571 int update_count, rc, idx; 572 const char *inner_sqlstr; 573 sqlite3_stmt *inner_stmt; 574 575 update_count = sqlite3_total_changes(db); 576 inner_sqlstr = "UPDATE mandb_meta SET device = :device," 577 " inode = :inode, mtime = :mtime WHERE" 578 " md5_hash = :md5 AND file = :file AND" 579 " (device <> :device2 OR inode <> " 580 " :inode2 OR mtime <> :mtime2)"; 581 rc = sqlite3_prepare_v2(db, inner_sqlstr, -1, &inner_stmt, NULL); 582 if (rc != SQLITE_OK) { 583 if (mflags.verbosity) 584 warnx("%s", sqlite3_errmsg(db)); 585 return; 586 } 587 idx = sqlite3_bind_parameter_index(inner_stmt, ":device"); 588 sqlite3_bind_int64(inner_stmt, idx, rec->device); 589 idx = sqlite3_bind_parameter_index(inner_stmt, ":inode"); 590 sqlite3_bind_int64(inner_stmt, idx, rec->inode); 591 idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime"); 592 sqlite3_bind_int64(inner_stmt, idx, rec->mtime); 593 idx = sqlite3_bind_parameter_index(inner_stmt, ":md5"); 594 sqlite3_bind_text(inner_stmt, idx, hash, -1, NULL); 595 idx = sqlite3_bind_parameter_index(inner_stmt, ":file"); 596 sqlite3_bind_text(inner_stmt, idx, file, -1, NULL); 597 idx = sqlite3_bind_parameter_index(inner_stmt, ":device2"); 598 sqlite3_bind_int64(inner_stmt, idx, rec->device); 599 idx = sqlite3_bind_parameter_index(inner_stmt, ":inode2"); 600 sqlite3_bind_int64(inner_stmt, idx, rec->inode); 601 idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime2"); 602 sqlite3_bind_int64(inner_stmt, idx, rec->mtime); 603 604 rc = sqlite3_step(inner_stmt); 605 if (rc == SQLITE_DONE) { 606 /* Check if an update has been performed. */ 607 if (update_count != sqlite3_total_changes(db)) { 608 if (mflags.verbosity == 2) 609 printf("Updated %s\n", file); 610 (*new_count)++; 611 } else { 612 /* Otherwise it was a hardlink. */ 613 (*link_count)++; 614 } 615 } else { 616 if (mflags.verbosity == 2) 617 warnx("Could not update the meta data for %s", file); 618 (*err_count)++; 619 } 620 sqlite3_finalize(inner_stmt); 621 } 622 623 /* read_and_decompress -- 624 * Reads the given file into memory. If it is compressed, decompres 625 * it before returning to the caller. 626 */ 627 static int 628 read_and_decompress(const char *file, void **buf, size_t *len) 629 { 630 size_t off; 631 ssize_t r; 632 struct archive *a; 633 struct archive_entry *ae; 634 635 if ((a = archive_read_new()) == NULL) 636 errx(EXIT_FAILURE, "memory allocation failed"); 637 638 if (archive_read_support_compression_all(a) != ARCHIVE_OK || 639 archive_read_support_format_raw(a) != ARCHIVE_OK || 640 archive_read_open_filename(a, file, 65536) != ARCHIVE_OK || 641 archive_read_next_header(a, &ae) != ARCHIVE_OK) 642 goto archive_error; 643 *len = 65536; 644 *buf = emalloc(*len); 645 off = 0; 646 for (;;) { 647 r = archive_read_data(a, (char *)*buf + off, *len - off); 648 if (r == ARCHIVE_OK) { 649 archive_read_close(a); 650 *len = off; 651 return 0; 652 } 653 if (r <= 0) { 654 free(*buf); 655 break; 656 } 657 off += r; 658 if (off == *len) { 659 *len *= 2; 660 if (*len < off) { 661 if (mflags.verbosity) 662 warnx("File too large: %s", file); 663 free(*buf); 664 archive_read_close(a); 665 return -1; 666 } 667 *buf = erealloc(*buf, *len); 668 } 669 } 670 671 archive_error: 672 warnx("Error while reading `%s': %s", file, archive_error_string(a)); 673 archive_read_close(a); 674 return -1; 675 } 676 677 /* update_db -- 678 * Does an incremental updation of the database by checking the file_cache. 679 * It parses and adds the pages which are present in file_cache, 680 * but not in the database. 681 * It also removes the pages which are present in the databse, 682 * but not in the file_cache. 683 */ 684 static void 685 update_db(sqlite3 *db, struct mparse *mp, mandb_rec *rec) 686 { 687 const char *sqlstr; 688 sqlite3_stmt *stmt = NULL; 689 const char *file; 690 const char *parent; 691 char *errmsg = NULL; 692 char *md5sum; 693 void *buf; 694 size_t buflen; 695 int new_count = 0; /* Counter for newly indexed/updated pages */ 696 int total_count = 0; /* Counter for total number of pages */ 697 int err_count = 0; /* Counter for number of failed pages */ 698 int link_count = 0; /* Counter for number of hard/sym links */ 699 int md5_status; 700 int rc; 701 702 sqlstr = "SELECT device, inode, mtime, parent, file" 703 " FROM metadb.file_cache fc" 704 " WHERE NOT EXISTS(SELECT 1 FROM mandb_meta WHERE" 705 " device = fc.device AND inode = fc.inode AND " 706 " mtime = fc.mtime AND file = fc.file)"; 707 708 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); 709 if (rc != SQLITE_OK) { 710 if (mflags.verbosity) 711 warnx("%s", sqlite3_errmsg(db)); 712 close_db(db); 713 errx(EXIT_FAILURE, "Could not query file cache"); 714 } 715 716 buf = NULL; 717 while (sqlite3_step(stmt) == SQLITE_ROW) { 718 free(buf); 719 total_count++; 720 rec->device = sqlite3_column_int64(stmt, 0); 721 rec->inode = sqlite3_column_int64(stmt, 1); 722 rec->mtime = sqlite3_column_int64(stmt, 2); 723 parent = (const char *) sqlite3_column_text(stmt, 3); 724 file = (const char *) sqlite3_column_text(stmt, 4); 725 if (read_and_decompress(file, &buf, &buflen)) { 726 err_count++; 727 buf = NULL; 728 continue; 729 } 730 md5_status = check_md5(file, db, "mandb_meta", &md5sum, buf, buflen); 731 assert(md5sum != NULL); 732 if (md5_status == -1) { 733 if (mflags.verbosity) 734 warnx("An error occurred in checking md5 value" 735 " for file %s", file); 736 err_count++; 737 continue; 738 } 739 740 if (md5_status == 0) { 741 /* 742 * The MD5 hash is already present in the database, 743 * so simply update the metadata, ignoring symlinks. 744 */ 745 struct stat sb; 746 stat(file, &sb); 747 if (S_ISLNK(sb.st_mode)) { 748 free(md5sum); 749 link_count++; 750 continue; 751 } 752 update_existing_entry(db, file, md5sum, rec, 753 &new_count, &link_count, &err_count); 754 free(md5sum); 755 continue; 756 } 757 758 if (md5_status == 1) { 759 /* 760 * The MD5 hash was not present in the database. 761 * This means is either a new file or an updated file. 762 * We should go ahead with parsing. 763 */ 764 if (mflags.verbosity == 2) 765 printf("Parsing: %s\n", file); 766 rec->md5_hash = md5sum; 767 rec->file_path = estrdup(file); 768 // file_path is freed by insert_into_db itself. 769 chdir(parent); 770 begin_parse(file, mp, rec, buf, buflen); 771 if (insert_into_db(db, rec) < 0) { 772 if (mflags.verbosity) 773 warnx("Error in indexing %s", file); 774 err_count++; 775 } else { 776 new_count++; 777 } 778 } 779 } 780 free(buf); 781 782 sqlite3_finalize(stmt); 783 784 if (mflags.verbosity == 2) { 785 printf("Total Number of new or updated pages encountered = %d\n" 786 "Total number of (hard or symbolic) links found = %d\n" 787 "Total number of pages that were successfully" 788 " indexed/updated = %d\n" 789 "Total number of pages that could not be indexed" 790 " due to errors = %d\n", 791 total_count - link_count, link_count, new_count, err_count); 792 } 793 794 if (mflags.recreate) 795 return; 796 797 if (mflags.verbosity == 2) 798 printf("Deleting stale index entries\n"); 799 800 sqlstr = "DELETE FROM mandb_meta WHERE file NOT IN" 801 " (SELECT file FROM metadb.file_cache);" 802 "DELETE FROM mandb_links WHERE md5_hash NOT IN" 803 " (SELECT md5_hash from mandb_meta);" 804 "DROP TABLE metadb.file_cache;" 805 "DELETE FROM mandb WHERE rowid NOT IN" 806 " (SELECT id FROM mandb_meta);"; 807 808 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg); 809 if (errmsg != NULL) { 810 warnx("Removing old entries failed: %s", errmsg); 811 warnx("Please rebuild database from scratch with -f."); 812 free(errmsg); 813 return; 814 } 815 } 816 817 /* 818 * begin_parse -- 819 * parses the man page using libmandoc 820 */ 821 static void 822 begin_parse(const char *file, struct mparse *mp, mandb_rec *rec, 823 const void *buf, size_t len) 824 { 825 struct mdoc *mdoc; 826 struct man *man; 827 mparse_reset(mp); 828 829 rec->xr_found = 0; 830 831 if (mparse_readmem(mp, buf, len, file) >= MANDOCLEVEL_FATAL) { 832 /* Printing this warning at verbosity level 2 833 * because some packages from pkgsrc might trigger several 834 * of such warnings. 835 */ 836 if (mflags.verbosity == 2) 837 warnx("%s: Parse failure", file); 838 return; 839 } 840 841 mparse_result(mp, &mdoc, &man); 842 if (mdoc == NULL && man == NULL) { 843 if (mflags.verbosity == 2) 844 warnx("Not a man(7) or mdoc(7) page"); 845 return; 846 } 847 848 set_machine(mdoc, rec); 849 set_section(mdoc, man, rec); 850 if (mdoc) { 851 rec->page_type = MDOC; 852 pmdoc_node(mdoc_node(mdoc), rec); 853 } else { 854 rec->page_type = MAN; 855 pman_node(man_node(man), rec); 856 } 857 } 858 859 /* 860 * set_section -- 861 * Extracts the section number and normalizes it to only the numeric part 862 * (Which should be the first character of the string). 863 */ 864 static void 865 set_section(const struct mdoc *md, const struct man *m, mandb_rec *rec) 866 { 867 if (md) { 868 const struct mdoc_meta *md_meta = mdoc_meta(md); 869 rec->section[0] = md_meta->msec[0]; 870 } else if (m) { 871 const struct man_meta *m_meta = man_meta(m); 872 rec->section[0] = m_meta->msec[0]; 873 } 874 } 875 876 /* 877 * get_machine -- 878 * Extracts the machine architecture information if available. 879 */ 880 static void 881 set_machine(const struct mdoc *md, mandb_rec *rec) 882 { 883 if (md == NULL) 884 return; 885 const struct mdoc_meta *md_meta = mdoc_meta(md); 886 if (md_meta->arch) 887 rec->machine = estrdup(md_meta->arch); 888 } 889 890 static void 891 pmdoc_node(const struct mdoc_node *n, mandb_rec *rec) 892 { 893 894 if (n == NULL) 895 return; 896 897 switch (n->type) { 898 case (MDOC_BODY): 899 /* FALLTHROUGH */ 900 case (MDOC_TAIL): 901 /* FALLTHROUGH */ 902 case (MDOC_ELEM): 903 if (mdocs[n->tok] == NULL) 904 break; 905 (*mdocs[n->tok])(n, rec); 906 break; 907 default: 908 break; 909 } 910 911 pmdoc_node(n->child, rec); 912 pmdoc_node(n->next, rec); 913 } 914 915 /* 916 * pmdoc_Nm -- 917 * Extracts the Name of the manual page from the .Nm macro 918 */ 919 static void 920 pmdoc_Nm(const struct mdoc_node *n, mandb_rec *rec) 921 { 922 if (n->sec != SEC_NAME) 923 return; 924 925 for (n = n->child; n; n = n->next) { 926 if (n->type == MDOC_TEXT) { 927 concat(&rec->name, n->string); 928 } 929 } 930 } 931 932 /* 933 * pmdoc_Nd -- 934 * Extracts the one line description of the man page from the .Nd macro 935 */ 936 static void 937 pmdoc_Nd(const struct mdoc_node *n, mandb_rec *rec) 938 { 939 /* 940 * A static variable for keeping track of whether a Xr macro was seen 941 * previously. 942 */ 943 char *buf = NULL; 944 char *temp; 945 946 if (n == NULL) 947 return; 948 949 if (n->type == MDOC_TEXT) { 950 if (rec->xr_found && n->next) { 951 /* 952 * An Xr macro was seen previously, so parse this 953 * and the next node. 954 */ 955 temp = estrdup(n->string); 956 n = n->next; 957 easprintf(&buf, "%s(%s)", temp, n->string); 958 concat(&rec->name_desc, buf); 959 free(buf); 960 free(temp); 961 } else { 962 concat(&rec->name_desc, n->string); 963 } 964 rec->xr_found = 0; 965 } else if (mdocs[n->tok] == pmdoc_Xr) { 966 /* Remember that we have encountered an Xr macro */ 967 rec->xr_found = 1; 968 } 969 970 if (n->child) 971 pmdoc_Nd(n->child, rec); 972 973 if(n->next) 974 pmdoc_Nd(n->next, rec); 975 } 976 977 /* 978 * pmdoc_macro_handler-- 979 * This function is a single point of handling all the special macros that we 980 * want to handle especially. For example the .Xr macro for properly parsing 981 * the referenced page name along with the section number, or the .Pp macro 982 * for adding a new line whenever we encounter it. 983 */ 984 static void 985 pmdoc_macro_handler(const struct mdoc_node *n, mandb_rec *rec, enum mdoct doct) 986 { 987 const struct mdoc_node *sn; 988 assert(n); 989 990 switch (doct) { 991 /* Parse the man page references. 992 * Basically the .Xr macros are used like: 993 * .Xr ls 1 994 * and formatted like this: 995 * ls(1) 996 * Prepare a buffer to format the data like the above example and call 997 * pmdoc_parse_section to append it. 998 */ 999 case MDOC_Xr: 1000 n = n->child; 1001 while (n->type != MDOC_TEXT && n->next) 1002 n = n->next; 1003 1004 if (n && n->type != MDOC_TEXT) 1005 return; 1006 sn = n; 1007 if (n->next) 1008 n = n->next; 1009 1010 while (n->type != MDOC_TEXT && n->next) 1011 n = n->next; 1012 1013 if (n && n->type == MDOC_TEXT) { 1014 size_t len = strlen(sn->string); 1015 char *buf = emalloc(len + 4); 1016 memcpy(buf, sn->string, len); 1017 buf[len] = '('; 1018 buf[len + 1] = n->string[0]; 1019 buf[len + 2] = ')'; 1020 buf[len + 3] = 0; 1021 mdoc_parse_section(n->sec, buf, rec); 1022 free(buf); 1023 } 1024 1025 break; 1026 1027 /* Parse the .Pp macro to add a new line */ 1028 case MDOC_Pp: 1029 if (n->type == MDOC_TEXT) 1030 mdoc_parse_section(n->sec, "\n", rec); 1031 break; 1032 default: 1033 break; 1034 } 1035 1036 } 1037 1038 /* 1039 * pmdoc_Xr, pmdoc_Pp-- 1040 * Empty stubs. 1041 * The parser calls these functions each time it encounters 1042 * a .Xr or .Pp macro. We are parsing all the data from 1043 * the pmdoc_Sh function, so don't do anything here. 1044 * (See if else blocks in pmdoc_Sh.) 1045 */ 1046 static void 1047 pmdoc_Xr(const struct mdoc_node *n, mandb_rec *rec) 1048 { 1049 } 1050 1051 static void 1052 pmdoc_Pp(const struct mdoc_node *n, mandb_rec *rec) 1053 { 1054 } 1055 1056 /* 1057 * pmdoc_Sh -- 1058 * Called when a .Sh macro is encountered and loops through its body, calling 1059 * mdoc_parse_section to append the data to the section specific buffer. 1060 * Two special macros which may occur inside the body of Sh are .Nm and .Xr and 1061 * they need special handling, thus the separate if branches for them. 1062 */ 1063 static void 1064 pmdoc_Sh(const struct mdoc_node *n, mandb_rec *rec) 1065 { 1066 if (n == NULL) 1067 return; 1068 int xr_found = 0; 1069 1070 if (n->type == MDOC_TEXT) { 1071 mdoc_parse_section(n->sec, n->string, rec); 1072 } else if (mdocs[n->tok] == pmdoc_Nm && rec->name != NULL) { 1073 /* 1074 * When encountering a .Nm macro, substitute it 1075 * with its previously cached value of the argument. 1076 */ 1077 mdoc_parse_section(n->sec, rec->name, rec); 1078 } else if (mdocs[n->tok] == pmdoc_Xr) { 1079 /* 1080 * When encountering other inline macros, 1081 * call pmdoc_macro_handler. 1082 */ 1083 pmdoc_macro_handler(n, rec, MDOC_Xr); 1084 xr_found = 1; 1085 } else if (mdocs[n->tok] == pmdoc_Pp) { 1086 pmdoc_macro_handler(n, rec, MDOC_Pp); 1087 } 1088 1089 /* 1090 * If an Xr macro was encountered then the child node has 1091 * already been explored by pmdoc_macro_handler. 1092 */ 1093 if (xr_found == 0) 1094 pmdoc_Sh(n->child, rec); 1095 pmdoc_Sh(n->next, rec); 1096 } 1097 1098 /* 1099 * mdoc_parse_section-- 1100 * Utility function for parsing sections of the mdoc type pages. 1101 * Takes two params: 1102 * 1. sec is an enum which indicates the section in which we are present 1103 * 2. string is the string which we need to append to the secbuff for this 1104 * particular section. 1105 * The function appends string to the global section buffer and returns. 1106 */ 1107 static void 1108 mdoc_parse_section(enum mdoc_sec sec, const char *string, mandb_rec *rec) 1109 { 1110 /* 1111 * If the user specified the 'l' flag, then parse and store only the 1112 * NAME section. Ignore the rest. 1113 */ 1114 if (mflags.limit) 1115 return; 1116 1117 switch (sec) { 1118 case SEC_LIBRARY: 1119 append(&rec->lib, string); 1120 break; 1121 case SEC_RETURN_VALUES: 1122 append(&rec->return_vals, string); 1123 break; 1124 case SEC_ENVIRONMENT: 1125 append(&rec->env, string); 1126 break; 1127 case SEC_FILES: 1128 append(&rec->files, string); 1129 break; 1130 case SEC_EXIT_STATUS: 1131 append(&rec->exit_status, string); 1132 break; 1133 case SEC_DIAGNOSTICS: 1134 append(&rec->diagnostics, string); 1135 break; 1136 case SEC_ERRORS: 1137 append(&rec->errors, string); 1138 break; 1139 case SEC_NAME: 1140 case SEC_SYNOPSIS: 1141 case SEC_EXAMPLES: 1142 case SEC_STANDARDS: 1143 case SEC_HISTORY: 1144 case SEC_AUTHORS: 1145 case SEC_BUGS: 1146 break; 1147 default: 1148 append(&rec->desc, string); 1149 break; 1150 } 1151 } 1152 1153 static void 1154 pman_node(const struct man_node *n, mandb_rec *rec) 1155 { 1156 if (n == NULL) 1157 return; 1158 1159 switch (n->type) { 1160 case (MAN_BODY): 1161 /* FALLTHROUGH */ 1162 case (MAN_TAIL): 1163 /* FALLTHROUGH */ 1164 case (MAN_BLOCK): 1165 /* FALLTHROUGH */ 1166 case (MAN_ELEM): 1167 if (mans[n->tok] != NULL) 1168 (*mans[n->tok])(n, rec); 1169 break; 1170 default: 1171 break; 1172 } 1173 1174 pman_node(n->child, rec); 1175 pman_node(n->next, rec); 1176 } 1177 1178 /* 1179 * pman_parse_name -- 1180 * Parses the NAME section and puts the complete content in the name_desc 1181 * variable. 1182 */ 1183 static void 1184 pman_parse_name(const struct man_node *n, mandb_rec *rec) 1185 { 1186 if (n == NULL) 1187 return; 1188 1189 if (n->type == MAN_TEXT) { 1190 char *tmp = parse_escape(n->string); 1191 concat(&rec->name_desc, tmp); 1192 free(tmp); 1193 } 1194 1195 if (n->child) 1196 pman_parse_name(n->child, rec); 1197 1198 if(n->next) 1199 pman_parse_name(n->next, rec); 1200 } 1201 1202 /* 1203 * A stub function to be able to parse the macros like .B embedded inside 1204 * a section. 1205 */ 1206 static void 1207 pman_block(const struct man_node *n, mandb_rec *rec) 1208 { 1209 } 1210 1211 /* 1212 * pman_sh -- 1213 * This function does one of the two things: 1214 * 1. If the present section is NAME, then it will: 1215 * (a) Extract the name of the page (in case of multiple comma separated 1216 * names, it will pick up the first one). 1217 * (b) Build a space spearated list of all the symlinks/hardlinks to 1218 * this page and store in the buffer 'links'. These are extracted from 1219 * the comma separated list of names in the NAME section as well. 1220 * (c) Move on to the one line description section, which is after the list 1221 * of names in the NAME section. 1222 * 2. Otherwise, it will check the section name and call the man_parse_section 1223 * function, passing the enum corresponding that section. 1224 */ 1225 static void 1226 pman_sh(const struct man_node *n, mandb_rec *rec) 1227 { 1228 static const struct { 1229 enum man_sec section; 1230 const char *header; 1231 } mapping[] = { 1232 { MANSEC_DESCRIPTION, "DESCRIPTION" }, 1233 { MANSEC_SYNOPSIS, "SYNOPSIS" }, 1234 { MANSEC_LIBRARY, "LIBRARY" }, 1235 { MANSEC_ERRORS, "ERRORS" }, 1236 { MANSEC_FILES, "FILES" }, 1237 { MANSEC_RETURN_VALUES, "RETURN VALUE" }, 1238 { MANSEC_RETURN_VALUES, "RETURN VALUES" }, 1239 { MANSEC_EXIT_STATUS, "EXIT STATUS" }, 1240 { MANSEC_EXAMPLES, "EXAMPLES" }, 1241 { MANSEC_EXAMPLES, "EXAMPLE" }, 1242 { MANSEC_STANDARDS, "STANDARDS" }, 1243 { MANSEC_HISTORY, "HISTORY" }, 1244 { MANSEC_BUGS, "BUGS" }, 1245 { MANSEC_AUTHORS, "AUTHORS" }, 1246 { MANSEC_COPYRIGHT, "COPYRIGHT" }, 1247 }; 1248 const struct man_node *head; 1249 char *name_desc; 1250 int sz; 1251 size_t i; 1252 1253 if ((head = n->parent->head) == NULL || (head = head->child) == NULL || 1254 head->type != MAN_TEXT) 1255 return; 1256 1257 /* 1258 * Check if this section should be extracted and 1259 * where it should be stored. Handled the trival cases first. 1260 */ 1261 for (i = 0; i < sizeof(mapping) / sizeof(mapping[0]); ++i) { 1262 if (strcmp(head->string, mapping[i].header) == 0) { 1263 man_parse_section(mapping[i].section, n, rec); 1264 return; 1265 } 1266 } 1267 1268 if (strcmp(head->string, "NAME") == 0) { 1269 /* 1270 * We are in the NAME section. 1271 * pman_parse_name will put the complete content in name_desc. 1272 */ 1273 pman_parse_name(n, rec); 1274 1275 name_desc = rec->name_desc; 1276 1277 /* Remove any leading spaces. */ 1278 while (name_desc[0] == ' ') 1279 name_desc++; 1280 1281 /* If the line begins with a "\&", avoid those */ 1282 if (name_desc[0] == '\\' && name_desc[1] == '&') 1283 name_desc += 2; 1284 1285 /* Now name_desc should be left with a comma-space 1286 * separated list of names and the one line description 1287 * of the page: 1288 * "a, b, c \- sample description" 1289 * Take out the first name, before the first comma 1290 * (or space) and store it in rec->name. 1291 * If the page has aliases then they should be 1292 * in the form of a comma separated list. 1293 * Keep looping while there is a comma in name_desc, 1294 * extract the alias name and store in rec->links. 1295 * When there are no more commas left, break out. 1296 */ 1297 int has_alias = 0; // Any more aliases left? 1298 while (*name_desc) { 1299 /* Remove any leading spaces or hyphens. */ 1300 if (name_desc[0] == ' ' || name_desc[0] =='-') { 1301 name_desc++; 1302 continue; 1303 } 1304 sz = strcspn(name_desc, ", "); 1305 1306 /* Extract the first term and store it in rec->name. */ 1307 if (rec->name == NULL) { 1308 if (name_desc[sz] == ',') 1309 has_alias = 1; 1310 name_desc[sz] = 0; 1311 rec->name = emalloc(sz + 1); 1312 memcpy(rec->name, name_desc, sz + 1); 1313 name_desc += sz + 1; 1314 continue; 1315 } 1316 1317 /* 1318 * Once rec->name is set, rest of the names 1319 * are to be treated as links or aliases. 1320 */ 1321 if (rec->name && has_alias) { 1322 if (name_desc[sz] != ',') { 1323 /* No more commas left --> 1324 * no more aliases to take out 1325 */ 1326 has_alias = 0; 1327 } 1328 name_desc[sz] = 0; 1329 concat2(&rec->links, name_desc, sz); 1330 name_desc += sz + 1; 1331 continue; 1332 } 1333 break; 1334 } 1335 1336 /* Parse any escape sequences that might be there */ 1337 char *temp = parse_escape(name_desc); 1338 free(rec->name_desc); 1339 rec->name_desc = temp; 1340 temp = parse_escape(rec->name); 1341 free(rec->name); 1342 rec->name = temp; 1343 return; 1344 } 1345 1346 /* The RETURN VALUE section might be specified in multiple ways */ 1347 if (strcmp(head->string, "RETURN") == 0 && 1348 head->next != NULL && head->next->type == MAN_TEXT && 1349 (strcmp(head->next->string, "VALUE") == 0 || 1350 strcmp(head->next->string, "VALUES") == 0)) { 1351 man_parse_section(MANSEC_RETURN_VALUES, n, rec); 1352 return; 1353 } 1354 1355 /* 1356 * EXIT STATUS section can also be specified all on one line or on two 1357 * separate lines. 1358 */ 1359 if (strcmp(head->string, "EXIT") == 0 && 1360 head->next != NULL && head->next->type == MAN_TEXT && 1361 strcmp(head->next->string, "STATUS") == 0) { 1362 man_parse_section(MANSEC_EXIT_STATUS, n, rec); 1363 return; 1364 } 1365 1366 /* Store the rest of the content in desc. */ 1367 man_parse_section(MANSEC_NONE, n, rec); 1368 } 1369 1370 /* 1371 * pman_parse_node -- 1372 * Generic function to iterate through a node. Usually called from 1373 * man_parse_section to parse a particular section of the man page. 1374 */ 1375 static void 1376 pman_parse_node(const struct man_node *n, secbuff *s) 1377 { 1378 if (n == NULL) 1379 return; 1380 1381 if (n->type == MAN_TEXT) 1382 append(s, n->string); 1383 1384 pman_parse_node(n->child, s); 1385 pman_parse_node(n->next, s); 1386 } 1387 1388 /* 1389 * man_parse_section -- 1390 * Takes two parameters: 1391 * sec: Tells which section we are present in 1392 * n: Is the present node of the AST. 1393 * Depending on the section, we call pman_parse_node to parse that section and 1394 * concatenate the content from that section into the buffer for that section. 1395 */ 1396 static void 1397 man_parse_section(enum man_sec sec, const struct man_node *n, mandb_rec *rec) 1398 { 1399 /* 1400 * If the user sepecified the 'l' flag then just parse 1401 * the NAME section, ignore the rest. 1402 */ 1403 if (mflags.limit) 1404 return; 1405 1406 switch (sec) { 1407 case MANSEC_LIBRARY: 1408 pman_parse_node(n, &rec->lib); 1409 break; 1410 case MANSEC_RETURN_VALUES: 1411 pman_parse_node(n, &rec->return_vals); 1412 break; 1413 case MANSEC_ENVIRONMENT: 1414 pman_parse_node(n, &rec->env); 1415 break; 1416 case MANSEC_FILES: 1417 pman_parse_node(n, &rec->files); 1418 break; 1419 case MANSEC_EXIT_STATUS: 1420 pman_parse_node(n, &rec->exit_status); 1421 break; 1422 case MANSEC_DIAGNOSTICS: 1423 pman_parse_node(n, &rec->diagnostics); 1424 break; 1425 case MANSEC_ERRORS: 1426 pman_parse_node(n, &rec->errors); 1427 break; 1428 case MANSEC_NAME: 1429 case MANSEC_SYNOPSIS: 1430 case MANSEC_EXAMPLES: 1431 case MANSEC_STANDARDS: 1432 case MANSEC_HISTORY: 1433 case MANSEC_BUGS: 1434 case MANSEC_AUTHORS: 1435 case MANSEC_COPYRIGHT: 1436 break; 1437 default: 1438 pman_parse_node(n, &rec->desc); 1439 break; 1440 } 1441 1442 } 1443 1444 /* 1445 * insert_into_db -- 1446 * Inserts the parsed data of the man page in the Sqlite databse. 1447 * If any of the values is NULL, then we cleanup and return -1 indicating 1448 * an error. 1449 * Otherwise, store the data in the database and return 0. 1450 */ 1451 static int 1452 insert_into_db(sqlite3 *db, mandb_rec *rec) 1453 { 1454 int rc = 0; 1455 int idx = -1; 1456 const char *sqlstr = NULL; 1457 sqlite3_stmt *stmt = NULL; 1458 char *ln = NULL; 1459 char *errmsg = NULL; 1460 long int mandb_rowid; 1461 1462 /* 1463 * At the very minimum we want to make sure that we store 1464 * the following data: 1465 * Name, one line description, and the MD5 hash 1466 */ 1467 if (rec->name == NULL || rec->name_desc == NULL || 1468 rec->md5_hash == NULL) { 1469 cleanup(rec); 1470 return -1; 1471 } 1472 1473 /* Write null byte at the end of all the sec_buffs */ 1474 rec->desc.data[rec->desc.offset] = 0; 1475 rec->lib.data[rec->lib.offset] = 0; 1476 rec->env.data[rec->env.offset] = 0; 1477 rec->return_vals.data[rec->return_vals.offset] = 0; 1478 rec->exit_status.data[rec->exit_status.offset] = 0; 1479 rec->files.data[rec->files.offset] = 0; 1480 rec->diagnostics.data[rec->diagnostics.offset] = 0; 1481 rec->errors.data[rec->errors.offset] = 0; 1482 1483 /* 1484 * In case of a mdoc page: (sorry, no better place to put this code) 1485 * parse the comma separated list of names of man pages, 1486 * the first name will be stored in the mandb table, rest will be 1487 * treated as links and put in the mandb_links table. 1488 */ 1489 if (rec->page_type == MDOC) { 1490 char *tmp; 1491 rec->links = estrdup(rec->name); 1492 free(rec->name); 1493 int sz = strcspn(rec->links, " \0"); 1494 rec->name = emalloc(sz + 1); 1495 memcpy(rec->name, rec->links, sz); 1496 if(rec->name[sz - 1] == ',') 1497 rec->name[sz - 1] = 0; 1498 else 1499 rec->name[sz] = 0; 1500 while (rec->links[sz] == ' ') 1501 ++sz; 1502 tmp = estrdup(rec->links + sz); 1503 free(rec->links); 1504 rec->links = tmp; 1505 } 1506 1507 /*------------------------ Populate the mandb table---------------------------*/ 1508 sqlstr = "INSERT INTO mandb VALUES (:section, :name, :name_desc, :desc," 1509 " :lib, :return_vals, :env, :files, :exit_status," 1510 " :diagnostics, :errors, :md5_hash, :machine)"; 1511 1512 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); 1513 if (rc != SQLITE_OK) 1514 goto Out; 1515 1516 idx = sqlite3_bind_parameter_index(stmt, ":name"); 1517 rc = sqlite3_bind_text(stmt, idx, rec->name, -1, NULL); 1518 if (rc != SQLITE_OK) { 1519 sqlite3_finalize(stmt); 1520 goto Out; 1521 } 1522 1523 idx = sqlite3_bind_parameter_index(stmt, ":section"); 1524 rc = sqlite3_bind_text(stmt, idx, rec->section, -1, NULL); 1525 if (rc != SQLITE_OK) { 1526 sqlite3_finalize(stmt); 1527 goto Out; 1528 } 1529 1530 idx = sqlite3_bind_parameter_index(stmt, ":name_desc"); 1531 rc = sqlite3_bind_text(stmt, idx, rec->name_desc, -1, NULL); 1532 if (rc != SQLITE_OK) { 1533 sqlite3_finalize(stmt); 1534 goto Out; 1535 } 1536 1537 idx = sqlite3_bind_parameter_index(stmt, ":desc"); 1538 rc = sqlite3_bind_text(stmt, idx, rec->desc.data, 1539 rec->desc.offset + 1, NULL); 1540 if (rc != SQLITE_OK) { 1541 sqlite3_finalize(stmt); 1542 goto Out; 1543 } 1544 1545 idx = sqlite3_bind_parameter_index(stmt, ":lib"); 1546 rc = sqlite3_bind_text(stmt, idx, rec->lib.data, rec->lib.offset + 1, NULL); 1547 if (rc != SQLITE_OK) { 1548 sqlite3_finalize(stmt); 1549 goto Out; 1550 } 1551 1552 idx = sqlite3_bind_parameter_index(stmt, ":return_vals"); 1553 rc = sqlite3_bind_text(stmt, idx, rec->return_vals.data, 1554 rec->return_vals.offset + 1, NULL); 1555 if (rc != SQLITE_OK) { 1556 sqlite3_finalize(stmt); 1557 goto Out; 1558 } 1559 1560 idx = sqlite3_bind_parameter_index(stmt, ":env"); 1561 rc = sqlite3_bind_text(stmt, idx, rec->env.data, rec->env.offset + 1, NULL); 1562 if (rc != SQLITE_OK) { 1563 sqlite3_finalize(stmt); 1564 goto Out; 1565 } 1566 1567 idx = sqlite3_bind_parameter_index(stmt, ":files"); 1568 rc = sqlite3_bind_text(stmt, idx, rec->files.data, 1569 rec->files.offset + 1, NULL); 1570 if (rc != SQLITE_OK) { 1571 sqlite3_finalize(stmt); 1572 goto Out; 1573 } 1574 1575 idx = sqlite3_bind_parameter_index(stmt, ":exit_status"); 1576 rc = sqlite3_bind_text(stmt, idx, rec->exit_status.data, 1577 rec->exit_status.offset + 1, NULL); 1578 if (rc != SQLITE_OK) { 1579 sqlite3_finalize(stmt); 1580 goto Out; 1581 } 1582 1583 idx = sqlite3_bind_parameter_index(stmt, ":diagnostics"); 1584 rc = sqlite3_bind_text(stmt, idx, rec->diagnostics.data, 1585 rec->diagnostics.offset + 1, NULL); 1586 if (rc != SQLITE_OK) { 1587 sqlite3_finalize(stmt); 1588 goto Out; 1589 } 1590 1591 idx = sqlite3_bind_parameter_index(stmt, ":errors"); 1592 rc = sqlite3_bind_text(stmt, idx, rec->errors.data, 1593 rec->errors.offset + 1, NULL); 1594 if (rc != SQLITE_OK) { 1595 sqlite3_finalize(stmt); 1596 goto Out; 1597 } 1598 1599 idx = sqlite3_bind_parameter_index(stmt, ":md5_hash"); 1600 rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL); 1601 if (rc != SQLITE_OK) { 1602 sqlite3_finalize(stmt); 1603 goto Out; 1604 } 1605 1606 idx = sqlite3_bind_parameter_index(stmt, ":machine"); 1607 if (rec->machine) 1608 rc = sqlite3_bind_text(stmt, idx, rec->machine, -1, NULL); 1609 else 1610 rc = sqlite3_bind_null(stmt, idx); 1611 if (rc != SQLITE_OK) { 1612 sqlite3_finalize(stmt); 1613 goto Out; 1614 } 1615 1616 rc = sqlite3_step(stmt); 1617 if (rc != SQLITE_DONE) { 1618 sqlite3_finalize(stmt); 1619 goto Out; 1620 } 1621 1622 sqlite3_finalize(stmt); 1623 1624 /* Get the row id of the last inserted row */ 1625 mandb_rowid = sqlite3_last_insert_rowid(db); 1626 1627 /*------------------------Populate the mandb_meta table-----------------------*/ 1628 sqlstr = "INSERT INTO mandb_meta VALUES (:device, :inode, :mtime," 1629 " :file, :md5_hash, :id)"; 1630 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); 1631 if (rc != SQLITE_OK) 1632 goto Out; 1633 1634 idx = sqlite3_bind_parameter_index(stmt, ":device"); 1635 rc = sqlite3_bind_int64(stmt, idx, rec->device); 1636 if (rc != SQLITE_OK) { 1637 sqlite3_finalize(stmt); 1638 goto Out; 1639 } 1640 1641 idx = sqlite3_bind_parameter_index(stmt, ":inode"); 1642 rc = sqlite3_bind_int64(stmt, idx, rec->inode); 1643 if (rc != SQLITE_OK) { 1644 sqlite3_finalize(stmt); 1645 goto Out; 1646 } 1647 1648 idx = sqlite3_bind_parameter_index(stmt, ":mtime"); 1649 rc = sqlite3_bind_int64(stmt, idx, rec->mtime); 1650 if (rc != SQLITE_OK) { 1651 sqlite3_finalize(stmt); 1652 goto Out; 1653 } 1654 1655 idx = sqlite3_bind_parameter_index(stmt, ":file"); 1656 rc = sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL); 1657 if (rc != SQLITE_OK) { 1658 sqlite3_finalize(stmt); 1659 goto Out; 1660 } 1661 1662 idx = sqlite3_bind_parameter_index(stmt, ":md5_hash"); 1663 rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL); 1664 if (rc != SQLITE_OK) { 1665 sqlite3_finalize(stmt); 1666 goto Out; 1667 } 1668 1669 idx = sqlite3_bind_parameter_index(stmt, ":id"); 1670 rc = sqlite3_bind_int64(stmt, idx, mandb_rowid); 1671 if (rc != SQLITE_OK) { 1672 sqlite3_finalize(stmt); 1673 goto Out; 1674 } 1675 1676 rc = sqlite3_step(stmt); 1677 sqlite3_finalize(stmt); 1678 if (rc == SQLITE_CONSTRAINT) { 1679 /* The *most* probable reason for reaching here is that 1680 * the UNIQUE contraint on the file column of the mandb_meta 1681 * table was violated. 1682 * This can happen when a file was updated/modified. 1683 * To fix this we need to do two things: 1684 * 1. Delete the row for the older version of this file 1685 * from mandb table. 1686 * 2. Run an UPDATE query to update the row for this file 1687 * in the mandb_meta table. 1688 */ 1689 warnx("Trying to update index for %s", rec->file_path); 1690 char *sql = sqlite3_mprintf("DELETE FROM mandb " 1691 "WHERE rowid = (SELECT id" 1692 " FROM mandb_meta" 1693 " WHERE file = %Q)", 1694 rec->file_path); 1695 sqlite3_exec(db, sql, NULL, NULL, &errmsg); 1696 sqlite3_free(sql); 1697 if (errmsg != NULL) { 1698 if (mflags.verbosity) 1699 warnx("%s", errmsg); 1700 free(errmsg); 1701 } 1702 sqlstr = "UPDATE mandb_meta SET device = :device," 1703 " inode = :inode, mtime = :mtime, id = :id," 1704 " md5_hash = :md5 WHERE file = :file"; 1705 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); 1706 if (rc != SQLITE_OK) { 1707 if (mflags.verbosity) 1708 warnx("Update failed with error: %s", 1709 sqlite3_errmsg(db)); 1710 close_db(db); 1711 cleanup(rec); 1712 errx(EXIT_FAILURE, 1713 "Consider running makemandb with -f option"); 1714 } 1715 1716 idx = sqlite3_bind_parameter_index(stmt, ":device"); 1717 sqlite3_bind_int64(stmt, idx, rec->device); 1718 idx = sqlite3_bind_parameter_index(stmt, ":inode"); 1719 sqlite3_bind_int64(stmt, idx, rec->inode); 1720 idx = sqlite3_bind_parameter_index(stmt, ":mtime"); 1721 sqlite3_bind_int64(stmt, idx, rec->mtime); 1722 idx = sqlite3_bind_parameter_index(stmt, ":id"); 1723 sqlite3_bind_int64(stmt, idx, mandb_rowid); 1724 idx = sqlite3_bind_parameter_index(stmt, ":md5"); 1725 sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL); 1726 idx = sqlite3_bind_parameter_index(stmt, ":file"); 1727 sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL); 1728 rc = sqlite3_step(stmt); 1729 sqlite3_finalize(stmt); 1730 1731 if (rc != SQLITE_DONE) { 1732 if (mflags.verbosity) 1733 warnx("%s", sqlite3_errmsg(db)); 1734 close_db(db); 1735 cleanup(rec); 1736 errx(EXIT_FAILURE, 1737 "Consider running makemandb with -f option"); 1738 } 1739 } else if (rc != SQLITE_DONE) { 1740 /* Otherwise make this error fatal */ 1741 warnx("Failed at %s\n%s", rec->file_path, sqlite3_errmsg(db)); 1742 cleanup(rec); 1743 close_db(db); 1744 exit(EXIT_FAILURE); 1745 } 1746 1747 /*------------------------ Populate the mandb_links table---------------------*/ 1748 char *str = NULL; 1749 char *links; 1750 if (rec->links && strlen(rec->links)) { 1751 links = rec->links; 1752 for(ln = strtok(links, " "); ln; ln = strtok(NULL, " ")) { 1753 if (ln[0] == ',') 1754 ln++; 1755 if(ln[strlen(ln) - 1] == ',') 1756 ln[strlen(ln) - 1] = 0; 1757 1758 str = sqlite3_mprintf("INSERT INTO mandb_links" 1759 " VALUES (%Q, %Q, %Q, %Q, %Q)", 1760 ln, rec->name, rec->section, 1761 rec->machine, rec->md5_hash); 1762 sqlite3_exec(db, str, NULL, NULL, &errmsg); 1763 sqlite3_free(str); 1764 if (errmsg != NULL) { 1765 warnx("%s", errmsg); 1766 cleanup(rec); 1767 free(errmsg); 1768 return -1; 1769 } 1770 } 1771 } 1772 1773 cleanup(rec); 1774 return 0; 1775 1776 Out: 1777 if (mflags.verbosity) 1778 warnx("%s", sqlite3_errmsg(db)); 1779 cleanup(rec); 1780 return -1; 1781 } 1782 1783 /* 1784 * check_md5-- 1785 * Generates the md5 hash of the file and checks if it already doesn't exist 1786 * in the table (passed as the 3rd parameter). 1787 * This function is being used to avoid hardlinks. 1788 * On successful completion it will also set the value of the fourth parameter 1789 * to the md5 hash of the file (computed previously). It is the responsibility 1790 * of the caller to free this buffer. 1791 * Return values: 1792 * -1: If an error occurs somewhere and sets the md5 return buffer to NULL. 1793 * 0: If the md5 hash does not exist in the table. 1794 * 1: If the hash exists in the database. 1795 */ 1796 static int 1797 check_md5(const char *file, sqlite3 *db, const char *table, char **md5sum, 1798 void *buf, size_t buflen) 1799 { 1800 int rc = 0; 1801 int idx = -1; 1802 char *sqlstr = NULL; 1803 sqlite3_stmt *stmt = NULL; 1804 1805 assert(file != NULL); 1806 *md5sum = MD5Data(buf, buflen, NULL); 1807 if (*md5sum == NULL) { 1808 if (mflags.verbosity) 1809 warn("md5 failed: %s", file); 1810 return -1; 1811 } 1812 1813 easprintf(&sqlstr, "SELECT * FROM %s WHERE md5_hash = :md5_hash", 1814 table); 1815 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); 1816 if (rc != SQLITE_OK) { 1817 free(sqlstr); 1818 free(*md5sum); 1819 *md5sum = NULL; 1820 return -1; 1821 } 1822 1823 idx = sqlite3_bind_parameter_index(stmt, ":md5_hash"); 1824 rc = sqlite3_bind_text(stmt, idx, *md5sum, -1, NULL); 1825 if (rc != SQLITE_OK) { 1826 if (mflags.verbosity) 1827 warnx("%s", sqlite3_errmsg(db)); 1828 sqlite3_finalize(stmt); 1829 free(sqlstr); 1830 free(*md5sum); 1831 *md5sum = NULL; 1832 return -1; 1833 } 1834 1835 if (sqlite3_step(stmt) == SQLITE_ROW) { 1836 sqlite3_finalize(stmt); 1837 free(sqlstr); 1838 return 0; 1839 } 1840 1841 sqlite3_finalize(stmt); 1842 free(sqlstr); 1843 return 1; 1844 } 1845 1846 /* Optimize the index for faster search */ 1847 static void 1848 optimize(sqlite3 *db) 1849 { 1850 const char *sqlstr; 1851 char *errmsg = NULL; 1852 1853 if (mflags.verbosity == 2) 1854 printf("Optimizing the database index\n"); 1855 sqlstr = "INSERT INTO mandb(mandb) VALUES (\'optimize\');" 1856 "VACUUM"; 1857 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg); 1858 if (errmsg != NULL) { 1859 if (mflags.verbosity) 1860 warnx("%s", errmsg); 1861 free(errmsg); 1862 return; 1863 } 1864 } 1865 1866 /* 1867 * cleanup -- 1868 * cleans up the global buffers 1869 */ 1870 static void 1871 cleanup(mandb_rec *rec) 1872 { 1873 rec->desc.offset = 0; 1874 rec->lib.offset = 0; 1875 rec->return_vals.offset = 0; 1876 rec->env.offset = 0; 1877 rec->exit_status.offset = 0; 1878 rec->diagnostics.offset = 0; 1879 rec->errors.offset = 0; 1880 rec->files.offset = 0; 1881 1882 free(rec->machine); 1883 rec->machine = NULL; 1884 1885 free(rec->links); 1886 rec->links = NULL; 1887 1888 free(rec->file_path); 1889 rec->file_path = NULL; 1890 1891 free(rec->name); 1892 rec->name = NULL; 1893 1894 free(rec->name_desc); 1895 rec->name_desc = NULL; 1896 1897 free(rec->md5_hash); 1898 rec->md5_hash = NULL; 1899 } 1900 1901 /* 1902 * init_secbuffs-- 1903 * Sets the value of buflen for all the sec_buff field of rec. And then 1904 * allocate memory to each sec_buff member of rec. 1905 */ 1906 static void 1907 init_secbuffs(mandb_rec *rec) 1908 { 1909 /* 1910 * Some sec_buff might need more memory, for example desc, 1911 * which stores the data of the DESCRIPTION section, 1912 * while some might need very small amount of memory. 1913 * Therefore explicitly setting the value of buflen field for 1914 * each sec_buff. 1915 */ 1916 rec->desc.buflen = 10 * BUFLEN; 1917 rec->desc.data = emalloc(rec->desc.buflen); 1918 rec->desc.offset = 0; 1919 1920 rec->lib.buflen = BUFLEN / 2; 1921 rec->lib.data = emalloc(rec->lib.buflen); 1922 rec->lib.offset = 0; 1923 1924 rec->return_vals.buflen = BUFLEN; 1925 rec->return_vals.data = emalloc(rec->return_vals.buflen); 1926 rec->return_vals.offset = 0; 1927 1928 rec->exit_status.buflen = BUFLEN; 1929 rec->exit_status.data = emalloc(rec->exit_status.buflen); 1930 rec->exit_status.offset = 0; 1931 1932 rec->env.buflen = BUFLEN; 1933 rec->env.data = emalloc(rec->env.buflen); 1934 rec->env.offset = 0; 1935 1936 rec->files.buflen = BUFLEN; 1937 rec->files.data = emalloc(rec->files.buflen); 1938 rec->files.offset = 0; 1939 1940 rec->diagnostics.buflen = BUFLEN; 1941 rec->diagnostics.data = emalloc(rec->diagnostics.buflen); 1942 rec->diagnostics.offset = 0; 1943 1944 rec->errors.buflen = BUFLEN; 1945 rec->errors.data = emalloc(rec->errors.buflen); 1946 rec->errors.offset = 0; 1947 } 1948 1949 /* 1950 * free_secbuffs-- 1951 * This function should be called at the end, when all the pages have been 1952 * parsed. 1953 * It frees the memory allocated to sec_buffs by init_secbuffs in the starting. 1954 */ 1955 static void 1956 free_secbuffs(mandb_rec *rec) 1957 { 1958 free(rec->desc.data); 1959 free(rec->lib.data); 1960 free(rec->return_vals.data); 1961 free(rec->exit_status.data); 1962 free(rec->env.data); 1963 free(rec->files.data); 1964 free(rec->diagnostics.data); 1965 free(rec->errors.data); 1966 } 1967 1968 static void 1969 replace_hyph(char *str) 1970 { 1971 char *iter = str; 1972 while ((iter = strchr(iter, ASCII_HYPH)) != NULL) 1973 *iter = '-'; 1974 } 1975 1976 static char * 1977 parse_escape(const char *str) 1978 { 1979 const char *backslash, *last_backslash; 1980 char *result, *iter; 1981 size_t len; 1982 1983 assert(str); 1984 1985 last_backslash = str; 1986 backslash = strchr(str, '\\'); 1987 if (backslash == NULL) { 1988 result = estrdup(str); 1989 replace_hyph(result); 1990 return result; 1991 } 1992 1993 result = emalloc(strlen(str) + 1); 1994 iter = result; 1995 1996 do { 1997 len = backslash - last_backslash; 1998 memcpy(iter, last_backslash, len); 1999 iter += len; 2000 if (backslash[1] == '-' || backslash[1] == ' ') { 2001 *iter++ = backslash[1]; 2002 last_backslash = backslash + 2; 2003 backslash = strchr(backslash + 2, '\\'); 2004 } else { 2005 ++backslash; 2006 mandoc_escape(&backslash, NULL, NULL); 2007 last_backslash = backslash; 2008 if (backslash == NULL) 2009 break; 2010 backslash = strchr(last_backslash, '\\'); 2011 } 2012 } while (backslash != NULL); 2013 if (last_backslash != NULL) 2014 strcpy(iter, last_backslash); 2015 2016 replace_hyph(result); 2017 return result; 2018 } 2019 2020 /* 2021 * append-- 2022 * Concatenates a space and src at the end of sbuff->data (much like concat in 2023 * apropos-utils.c). 2024 * Rather than reallocating space for writing data, it uses the value of the 2025 * offset field of sec_buff to write new data at the free space left in the 2026 * buffer. 2027 * In case the size of the data to be appended exceeds the number of bytes left 2028 * in the buffer, it reallocates buflen number of bytes and then continues. 2029 * Value of offset field should be adjusted as new data is written. 2030 * 2031 * NOTE: This function does not write the null byte at the end of the buffers, 2032 * write a null byte at the position pointed to by offset before inserting data 2033 * in the db. 2034 */ 2035 static void 2036 append(secbuff *sbuff, const char *src) 2037 { 2038 short flag = 0; 2039 size_t srclen, newlen; 2040 char *temp; 2041 2042 assert(src != NULL); 2043 temp = parse_escape(src); 2044 srclen = strlen(temp); 2045 2046 if (sbuff->data == NULL) { 2047 sbuff->data = emalloc(sbuff->buflen); 2048 sbuff->offset = 0; 2049 } 2050 2051 newlen = sbuff->offset + srclen + 2; 2052 if (newlen >= sbuff->buflen) { 2053 while (sbuff->buflen < newlen) 2054 sbuff->buflen += sbuff->buflen; 2055 sbuff->data = erealloc(sbuff->data, sbuff->buflen); 2056 flag = 1; 2057 } 2058 2059 /* Append a space at the end of the buffer. */ 2060 if (sbuff->offset || flag) 2061 sbuff->data[sbuff->offset++] = ' '; 2062 /* Now, copy src at the end of the buffer. */ 2063 memcpy(sbuff->data + sbuff->offset, temp, srclen); 2064 sbuff->offset += srclen; 2065 free(temp); 2066 } 2067 2068 static void 2069 usage(void) 2070 { 2071 fprintf(stderr, "Usage: %s [-floQqv] [-C path]\n", getprogname()); 2072 exit(1); 2073 } 2074
Indexes created Mon Sep 29 21:09:56 GMT 2025