makemandb.c revision 1.9
1 /* $NetBSD: makemandb.c,v 1.9 2012/05/07 11:18:16 wiz Exp $ */ 2 /* 3 * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay (at) gmail.com> 4 * Copyright (c) 2011 Kristaps Dzonsons <kristaps (at) bsd.lv> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/cdefs.h> 20 __RCSID("$NetBSD: makemandb.c,v 1.9 2012/05/07 11:18:16 wiz Exp $"); 21 22 #include <sys/stat.h> 23 #include <sys/types.h> 24 25 #include <assert.h> 26 #include <ctype.h> 27 #include <dirent.h> 28 #include <err.h> 29 #include <archive.h> 30 #include <libgen.h> 31 #include <md5.h> 32 #include <stdio.h> 33 #include <stdlib.h> 34 #include <string.h> 35 #include <unistd.h> 36 #include <util.h> 37 38 #include "apropos-utils.h" 39 #include "man.h" 40 #include "mandoc.h" 41 #include "mdoc.h" 42 #include "sqlite3.h" 43 44 #define BUFLEN 1024 45 #define MDOC 0 //If the page is of mdoc(7) type 46 #define MAN 1 //If the page is of man(7) type 47 48 /* 49 * A data structure for holding section specific data. 50 */ 51 typedef struct secbuff { 52 char *data; 53 size_t buflen; //Total length of buffer allocated initially 54 size_t offset; // Current offset in the buffer. 55 } secbuff; 56 57 typedef struct makemandb_flags { 58 int optimize; 59 int limit; // limit the indexing to only NAME section 60 int recreate; // Database was created from scratch 61 int verbosity; // 0: quiet, 1: default, 2: verbose 62 } makemandb_flags; 63 64 typedef struct mandb_rec { 65 /* Fields for mandb table */ 66 char *name; // for storing the name of the man page 67 char *name_desc; // for storing the one line description (.Nd) 68 secbuff desc; // for storing the DESCRIPTION section 69 secbuff lib; // for the LIBRARY section 70 secbuff return_vals; // RETURN VALUES 71 secbuff env; // ENVIRONMENT 72 secbuff files; // FILES 73 secbuff exit_status; // EXIT STATUS 74 secbuff diagnostics; // DIAGNOSTICS 75 secbuff errors; // ERRORS 76 char section[2]; 77 78 int xr_found; 79 80 /* Fields for mandb_meta table */ 81 char *md5_hash; 82 dev_t device; 83 ino_t inode; 84 time_t mtime; 85 86 /* Fields for mandb_links table */ 87 char *machine; 88 char *links; //all the links to a page in a space separated form 89 char *file_path; 90 91 /* Non-db fields */ 92 int page_type; //Indicates the type of page: mdoc or man 93 } mandb_rec; 94 95 static void append(secbuff *sbuff, const char *src); 96 static void init_secbuffs(mandb_rec *); 97 static void free_secbuffs(mandb_rec *); 98 static int check_md5(const char *, sqlite3 *, const char *, char **, void *, size_t); 99 static void cleanup(mandb_rec *); 100 static void set_section(const struct mdoc *, const struct man *, mandb_rec *); 101 static void set_machine(const struct mdoc *, mandb_rec *); 102 static int insert_into_db(sqlite3 *, mandb_rec *); 103 static void begin_parse(const char *, struct mparse *, mandb_rec *, 104 const void *, size_t len); 105 static void pmdoc_node(const struct mdoc_node *, mandb_rec *); 106 static void pmdoc_Nm(const struct mdoc_node *, mandb_rec *); 107 static void pmdoc_Nd(const struct mdoc_node *, mandb_rec *); 108 static void pmdoc_Sh(const struct mdoc_node *, mandb_rec *); 109 static void pmdoc_Xr(const struct mdoc_node *, mandb_rec *); 110 static void pmdoc_Pp(const struct mdoc_node *, mandb_rec *); 111 static void pmdoc_macro_handler(const struct mdoc_node *, mandb_rec *, 112 enum mdoct); 113 static void pman_node(const struct man_node *n, mandb_rec *); 114 static void pman_parse_node(const struct man_node *, secbuff *); 115 static void pman_parse_name(const struct man_node *, mandb_rec *); 116 static void pman_sh(const struct man_node *, mandb_rec *); 117 static void pman_block(const struct man_node *, mandb_rec *); 118 static void traversedir(const char *, const char *, sqlite3 *, struct mparse *); 119 static void mdoc_parse_section(enum mdoc_sec, const char *, mandb_rec *); 120 static void man_parse_section(enum man_sec, const struct man_node *, mandb_rec *); 121 static void build_file_cache(sqlite3 *, const char *, const char *, 122 struct stat *); 123 static void update_db(sqlite3 *, struct mparse *, mandb_rec *); 124 __dead static void usage(void); 125 static void optimize(sqlite3 *); 126 static char *parse_escape(const char *); 127 static makemandb_flags mflags = { .verbosity = 1 }; 128 129 typedef void (*pman_nf)(const struct man_node *n, mandb_rec *); 130 typedef void (*pmdoc_nf)(const struct mdoc_node *n, mandb_rec *); 131 static const pmdoc_nf mdocs[MDOC_MAX] = { 132 NULL, /* Ap */ 133 NULL, /* Dd */ 134 NULL, /* Dt */ 135 NULL, /* Os */ 136 pmdoc_Sh, /* Sh */ 137 NULL, /* Ss */ 138 pmdoc_Pp, /* Pp */ 139 NULL, /* D1 */ 140 NULL, /* Dl */ 141 NULL, /* Bd */ 142 NULL, /* Ed */ 143 NULL, /* Bl */ 144 NULL, /* El */ 145 NULL, /* It */ 146 NULL, /* Ad */ 147 NULL, /* An */ 148 NULL, /* Ar */ 149 NULL, /* Cd */ 150 NULL, /* Cm */ 151 NULL, /* Dv */ 152 NULL, /* Er */ 153 NULL, /* Ev */ 154 NULL, /* Ex */ 155 NULL, /* Fa */ 156 NULL, /* Fd */ 157 NULL, /* Fl */ 158 NULL, /* Fn */ 159 NULL, /* Ft */ 160 NULL, /* Ic */ 161 NULL, /* In */ 162 NULL, /* Li */ 163 pmdoc_Nd, /* Nd */ 164 pmdoc_Nm, /* Nm */ 165 NULL, /* Op */ 166 NULL, /* Ot */ 167 NULL, /* Pa */ 168 NULL, /* Rv */ 169 NULL, /* St */ 170 NULL, /* Va */ 171 NULL, /* Vt */ 172 pmdoc_Xr, /* Xr */ 173 NULL, /* %A */ 174 NULL, /* %B */ 175 NULL, /* %D */ 176 NULL, /* %I */ 177 NULL, /* %J */ 178 NULL, /* %N */ 179 NULL, /* %O */ 180 NULL, /* %P */ 181 NULL, /* %R */ 182 NULL, /* %T */ 183 NULL, /* %V */ 184 NULL, /* Ac */ 185 NULL, /* Ao */ 186 NULL, /* Aq */ 187 NULL, /* At */ 188 NULL, /* Bc */ 189 NULL, /* Bf */ 190 NULL, /* Bo */ 191 NULL, /* Bq */ 192 NULL, /* Bsx */ 193 NULL, /* Bx */ 194 NULL, /* Db */ 195 NULL, /* Dc */ 196 NULL, /* Do */ 197 NULL, /* Dq */ 198 NULL, /* Ec */ 199 NULL, /* Ef */ 200 NULL, /* Em */ 201 NULL, /* Eo */ 202 NULL, /* Fx */ 203 NULL, /* Ms */ 204 NULL, /* No */ 205 NULL, /* Ns */ 206 NULL, /* Nx */ 207 NULL, /* Ox */ 208 NULL, /* Pc */ 209 NULL, /* Pf */ 210 NULL, /* Po */ 211 NULL, /* Pq */ 212 NULL, /* Qc */ 213 NULL, /* Ql */ 214 NULL, /* Qo */ 215 NULL, /* Qq */ 216 NULL, /* Re */ 217 NULL, /* Rs */ 218 NULL, /* Sc */ 219 NULL, /* So */ 220 NULL, /* Sq */ 221 NULL, /* Sm */ 222 NULL, /* Sx */ 223 NULL, /* Sy */ 224 NULL, /* Tn */ 225 NULL, /* Ux */ 226 NULL, /* Xc */ 227 NULL, /* Xo */ 228 NULL, /* Fo */ 229 NULL, /* Fc */ 230 NULL, /* Oo */ 231 NULL, /* Oc */ 232 NULL, /* Bk */ 233 NULL, /* Ek */ 234 NULL, /* Bt */ 235 NULL, /* Hf */ 236 NULL, /* Fr */ 237 NULL, /* Ud */ 238 NULL, /* Lb */ 239 NULL, /* Lp */ 240 NULL, /* Lk */ 241 NULL, /* Mt */ 242 NULL, /* Brq */ 243 NULL, /* Bro */ 244 NULL, /* Brc */ 245 NULL, /* %C */ 246 NULL, /* Es */ 247 NULL, /* En */ 248 NULL, /* Dx */ 249 NULL, /* %Q */ 250 NULL, /* br */ 251 NULL, /* sp */ 252 NULL, /* %U */ 253 NULL, /* Ta */ 254 }; 255 256 static const pman_nf mans[MAN_MAX] = { 257 NULL, //br 258 NULL, //TH 259 pman_sh, //SH 260 NULL, //SS 261 NULL, //TP 262 NULL, //LP 263 NULL, //PP 264 NULL, //P 265 NULL, //IP 266 NULL, //HP 267 NULL, //SM 268 NULL, //SB 269 NULL, //BI 270 NULL, //IB 271 NULL, //BR 272 NULL, //RB 273 NULL, //R 274 pman_block, //B 275 NULL, //I 276 NULL, //IR 277 NULL, //RI 278 NULL, //na 279 NULL, //sp 280 NULL, //nf 281 NULL, //fi 282 NULL, //RE 283 NULL, //RS 284 NULL, //DT 285 NULL, //UC 286 NULL, //PD 287 NULL, //AT 288 NULL, //in 289 NULL, //ft 290 }; 291 292 293 int 294 main(int argc, char *argv[]) 295 { 296 FILE *file; 297 const char *sqlstr, *manconf = NULL; 298 char *line, *command, *parent; 299 char *errmsg; 300 int ch; 301 struct mparse *mp; 302 sqlite3 *db; 303 ssize_t len; 304 size_t linesize; 305 struct mandb_rec rec; 306 307 while ((ch = getopt(argc, argv, "C:floqv")) != -1) { 308 switch (ch) { 309 case 'C': 310 manconf = optarg; 311 break; 312 case 'f': 313 remove(DBPATH); 314 mflags.recreate = 1; 315 break; 316 case 'l': 317 mflags.limit = 1; 318 break; 319 case 'o': 320 mflags.optimize = 1; 321 break; 322 case 'q': 323 mflags.verbosity = 0; 324 break; 325 case 'v': 326 mflags.verbosity = 2; 327 break; 328 default: 329 usage(); 330 } 331 } 332 333 memset(&rec, 0, sizeof(rec)); 334 335 init_secbuffs(&rec); 336 mp = mparse_alloc(MPARSE_AUTO, MANDOCLEVEL_FATAL, NULL, NULL); 337 338 if ((db = init_db(MANDB_CREATE)) == NULL) 339 exit(EXIT_FAILURE); 340 341 sqlite3_exec(db, "PRAGMA synchronous = 0", NULL, NULL, &errmsg); 342 if (errmsg != NULL) { 343 warnx("%s", errmsg); 344 free(errmsg); 345 close_db(db); 346 exit(EXIT_FAILURE); 347 } 348 349 sqlite3_exec(db, "ATTACH DATABASE \':memory:\' AS metadb", NULL, NULL, 350 &errmsg); 351 if (errmsg != NULL) { 352 warnx("%s", errmsg); 353 free(errmsg); 354 close_db(db); 355 exit(EXIT_FAILURE); 356 } 357 358 if (manconf) { 359 char *arg; 360 size_t command_len = shquote(manconf, NULL, 0) + 1; 361 arg = malloc(command_len ); 362 shquote(manconf, arg, command_len); 363 easprintf(&command, "man -p -C %s", arg); 364 free(arg); 365 } else { 366 command = estrdup("man -p"); 367 } 368 369 /* Call man -p to get the list of man page dirs */ 370 if ((file = popen(command, "r")) == NULL) { 371 close_db(db); 372 err(EXIT_FAILURE, "fopen failed"); 373 } 374 free(command); 375 376 /* Begin the transaction for indexing the pages */ 377 sqlite3_exec(db, "BEGIN", NULL, NULL, &errmsg); 378 if (errmsg != NULL) { 379 warnx("%s", errmsg); 380 free(errmsg); 381 exit(EXIT_FAILURE); 382 } 383 384 sqlstr = "CREATE TABLE metadb.file_cache(device, inode, mtime, parent," 385 " file PRIMARY KEY);" 386 "CREATE UNIQUE INDEX metadb.index_file_cache_dev" 387 " ON file_cache (device, inode)"; 388 389 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg); 390 if (errmsg != NULL) { 391 warnx("%s", errmsg); 392 free(errmsg); 393 close_db(db); 394 exit(EXIT_FAILURE); 395 } 396 397 if (mflags.verbosity) 398 printf("Building temporary file cache\n"); 399 line = NULL; 400 linesize = 0; 401 while ((len = getline(&line, &linesize, file)) != -1) { 402 /* Replace the new line character at the end of string with '\0' */ 403 line[len - 1] = '\0'; 404 parent = estrdup(line); 405 char *pdir = estrdup(dirname(parent)); 406 free(parent); 407 /* Traverse the man page directories and parse the pages */ 408 traversedir(pdir, line, db, mp); 409 free(pdir); 410 } 411 free(line); 412 413 if (pclose(file) == -1) { 414 close_db(db); 415 cleanup(&rec); 416 free_secbuffs(&rec); 417 err(EXIT_FAILURE, "pclose error"); 418 } 419 420 update_db(db, mp, &rec); 421 mparse_free(mp); 422 free_secbuffs(&rec); 423 424 /* Commit the transaction */ 425 sqlite3_exec(db, "COMMIT", NULL, NULL, &errmsg); 426 if (errmsg != NULL) { 427 warnx("%s", errmsg); 428 free(errmsg); 429 exit(EXIT_FAILURE); 430 } 431 432 if (mflags.optimize) 433 optimize(db); 434 435 close_db(db); 436 return 0; 437 } 438 439 /* 440 * traversedir -- 441 * Traverses the given directory recursively and passes all the man page files 442 * in the way to build_file_cache() 443 */ 444 static void 445 traversedir(const char *parent, const char *file, sqlite3 *db, 446 struct mparse *mp) 447 { 448 struct stat sb; 449 struct dirent *dirp; 450 DIR *dp; 451 char *buf; 452 453 if (stat(file, &sb) < 0) { 454 warn("stat failed: %s", file); 455 return; 456 } 457 458 /* If it is a regular file or a symlink, pass it to build_cache() */ 459 if (S_ISREG(sb.st_mode) || S_ISLNK(sb.st_mode)) { 460 build_file_cache(db, parent, file, &sb); 461 return; 462 } 463 464 /* If it is a directory, traverse it recursively */ 465 if (S_ISDIR(sb.st_mode)) { 466 if ((dp = opendir(file)) == NULL) { 467 warn("opendir error: %s", file); 468 return; 469 } 470 471 while ((dirp = readdir(dp)) != NULL) { 472 /* Avoid . and .. entries in a directory */ 473 if (strncmp(dirp->d_name, ".", 1)) { 474 easprintf(&buf, "%s/%s", file, dirp->d_name); 475 traversedir(parent, buf, db, mp); 476 free(buf); 477 } 478 } 479 closedir(dp); 480 } 481 } 482 483 /* build_file_cache -- 484 * This function generates an md5 hash of the file passed as it's 2nd parameter 485 * and stores it in a temporary table file_cache along with the full file path. 486 * This is done to support incremental updation of the database. 487 * The temporary table file_cache is dropped thereafter in the function 488 * update_db(), once the database has been updated. 489 */ 490 static void 491 build_file_cache(sqlite3 *db, const char *parent, const char *file, 492 struct stat *sb) 493 { 494 const char *sqlstr; 495 sqlite3_stmt *stmt = NULL; 496 int rc, idx; 497 assert(file != NULL); 498 dev_t device_cache = sb->st_dev; 499 ino_t inode_cache = sb->st_ino; 500 time_t mtime_cache = sb->st_mtime; 501 502 sqlstr = "INSERT INTO metadb.file_cache VALUES (:device, :inode," 503 " :mtime, :parent, :file)"; 504 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); 505 if (rc != SQLITE_OK) { 506 warnx("%s", sqlite3_errmsg(db)); 507 return; 508 } 509 510 idx = sqlite3_bind_parameter_index(stmt, ":device"); 511 rc = sqlite3_bind_int64(stmt, idx, device_cache); 512 if (rc != SQLITE_OK) { 513 warnx("%s", sqlite3_errmsg(db)); 514 sqlite3_finalize(stmt); 515 return; 516 } 517 518 idx = sqlite3_bind_parameter_index(stmt, ":inode"); 519 rc = sqlite3_bind_int64(stmt, idx, inode_cache); 520 if (rc != SQLITE_OK) { 521 warnx("%s", sqlite3_errmsg(db)); 522 sqlite3_finalize(stmt); 523 return; 524 } 525 526 idx = sqlite3_bind_parameter_index(stmt, ":mtime"); 527 rc = sqlite3_bind_int64(stmt, idx, mtime_cache); 528 if (rc != SQLITE_OK) { 529 warnx("%s", sqlite3_errmsg(db)); 530 sqlite3_finalize(stmt); 531 return; 532 } 533 534 idx = sqlite3_bind_parameter_index(stmt, ":parent"); 535 rc = sqlite3_bind_text(stmt, idx, parent, -1, NULL); 536 if (rc != SQLITE_OK) { 537 warnx("%s", sqlite3_errmsg(db)); 538 sqlite3_finalize(stmt); 539 return; 540 } 541 542 idx = sqlite3_bind_parameter_index(stmt, ":file"); 543 rc = sqlite3_bind_text(stmt, idx, file, -1, NULL); 544 if (rc != SQLITE_OK) { 545 warnx("%s", sqlite3_errmsg(db)); 546 sqlite3_finalize(stmt); 547 return; 548 } 549 550 sqlite3_step(stmt); 551 sqlite3_finalize(stmt); 552 } 553 554 static void 555 update_existing_entry(sqlite3 *db, const char *file, const char *hash, 556 mandb_rec *rec, int *new_count, int *link_count, int *err_count) 557 { 558 int update_count, rc, idx; 559 const char *inner_sqlstr; 560 sqlite3_stmt *inner_stmt; 561 562 update_count = sqlite3_total_changes(db); 563 inner_sqlstr = "UPDATE mandb_meta SET device = :device," 564 " inode = :inode, mtime = :mtime WHERE" 565 " md5_hash = :md5 AND file = :file AND" 566 " (device <> :device2 OR inode <> " 567 " :inode2 OR mtime <> :mtime2)"; 568 rc = sqlite3_prepare_v2(db, inner_sqlstr, -1, &inner_stmt, NULL); 569 if (rc != SQLITE_OK) { 570 warnx("%s", sqlite3_errmsg(db)); 571 return; 572 } 573 idx = sqlite3_bind_parameter_index(inner_stmt, ":device"); 574 sqlite3_bind_int64(inner_stmt, idx, rec->device); 575 idx = sqlite3_bind_parameter_index(inner_stmt, ":inode"); 576 sqlite3_bind_int64(inner_stmt, idx, rec->inode); 577 idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime"); 578 sqlite3_bind_int64(inner_stmt, idx, rec->mtime); 579 idx = sqlite3_bind_parameter_index(inner_stmt, ":md5"); 580 sqlite3_bind_text(inner_stmt, idx, hash, -1, NULL); 581 idx = sqlite3_bind_parameter_index(inner_stmt, ":file"); 582 sqlite3_bind_text(inner_stmt, idx, file, -1, NULL); 583 idx = sqlite3_bind_parameter_index(inner_stmt, ":device2"); 584 sqlite3_bind_int64(inner_stmt, idx, rec->device); 585 idx = sqlite3_bind_parameter_index(inner_stmt, ":inode2"); 586 sqlite3_bind_int64(inner_stmt, idx, rec->inode); 587 idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime2"); 588 sqlite3_bind_int64(inner_stmt, idx, rec->mtime); 589 590 rc = sqlite3_step(inner_stmt); 591 if (rc == SQLITE_DONE) { 592 /* Check if an update has been performed. */ 593 if (update_count != sqlite3_total_changes(db)) { 594 if (mflags.verbosity) 595 printf("Updated %s\n", file); 596 (*new_count)++; 597 } else { 598 /* Otherwise it was a hardlink. */ 599 (*link_count)++; 600 } 601 } else { 602 warnx("Could not update the meta data for %s", file); 603 (*err_count)++; 604 } 605 sqlite3_finalize(inner_stmt); 606 } 607 608 /* read_and_decompress -- 609 * Reads the given file into memory. If it is compressed, decompres 610 * it before returning to the caller. 611 */ 612 static int 613 read_and_decompress(const char *file, void **buf, size_t *len) 614 { 615 size_t off; 616 ssize_t r; 617 struct archive *a; 618 struct archive_entry *ae; 619 620 if ((a = archive_read_new()) == NULL) 621 errx(EXIT_FAILURE, "memory allocation failed"); 622 623 if (archive_read_support_compression_all(a) != ARCHIVE_OK || 624 archive_read_support_format_raw(a) != ARCHIVE_OK || 625 archive_read_open_filename(a, file, 65536) != ARCHIVE_OK || 626 archive_read_next_header(a, &ae) != ARCHIVE_OK) 627 goto archive_error; 628 *len = 65536; 629 *buf = emalloc(*len); 630 off = 0; 631 for (;;) { 632 r = archive_read_data(a, (char *)*buf + off, *len - off); 633 if (r == ARCHIVE_OK) { 634 archive_read_close(a); 635 *len = off; 636 return 0; 637 } 638 if (r <= 0) { 639 free(*buf); 640 break; 641 } 642 off += r; 643 if (off == *len) { 644 *len *= 2; 645 if (*len < off) { 646 warnx("File too large: %s", file); 647 free(*buf); 648 archive_read_close(a); 649 return -1; 650 } 651 *buf = erealloc(*buf, *len); 652 } 653 } 654 655 archive_error: 656 warnx("Error while reading `%s': %s", file, archive_error_string(a)); 657 archive_read_close(a); 658 return -1; 659 } 660 661 /* update_db -- 662 * Does an incremental updation of the database by checking the file_cache. 663 * It parses and adds the pages which are present in file_cache, 664 * but not in the database. 665 * It also removes the pages which are present in the databse, 666 * but not in the file_cache. 667 */ 668 static void 669 update_db(sqlite3 *db, struct mparse *mp, mandb_rec *rec) 670 { 671 const char *sqlstr; 672 sqlite3_stmt *stmt = NULL; 673 const char *file; 674 const char *parent; 675 char *errmsg = NULL; 676 char *md5sum; 677 void *buf; 678 size_t buflen; 679 int new_count = 0; /* Counter for newly indexed/updated pages */ 680 int total_count = 0; /* Counter for total number of pages */ 681 int err_count = 0; /* Counter for number of failed pages */ 682 int link_count = 0; /* Counter for number of hard/sym links */ 683 int md5_status; 684 int rc; 685 686 sqlstr = "SELECT device, inode, mtime, parent, file" 687 " FROM metadb.file_cache fc" 688 " WHERE NOT EXISTS(SELECT 1 FROM mandb_meta WHERE" 689 " device = fc.device AND inode = fc.inode AND " 690 " mtime = fc.mtime AND file = fc.file)"; 691 692 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); 693 if (rc != SQLITE_OK) { 694 warnx("%s", sqlite3_errmsg(db)); 695 close_db(db); 696 errx(EXIT_FAILURE, "Could not query file cache"); 697 } 698 699 buf = NULL; 700 while (sqlite3_step(stmt) == SQLITE_ROW) { 701 free(buf); 702 total_count++; 703 rec->device = sqlite3_column_int64(stmt, 0); 704 rec->inode = sqlite3_column_int64(stmt, 1); 705 rec->mtime = sqlite3_column_int64(stmt, 2); 706 parent = (const char *) sqlite3_column_text(stmt, 3); 707 file = (const char *) sqlite3_column_text(stmt, 4); 708 if (read_and_decompress(file, &buf, &buflen)) { 709 err_count++; 710 buf = NULL; 711 continue; 712 } 713 md5_status = check_md5(file, db, "mandb_meta", &md5sum, buf, buflen); 714 assert(md5sum != NULL); 715 if (md5_status == -1) { 716 warnx("An error occurred in checking md5 value" 717 " for file %s", file); 718 err_count++; 719 continue; 720 } 721 722 if (md5_status == 0) { 723 /* 724 * The MD5 hash is already present in the database, 725 * so simply update the metadata, ignoring symlinks. 726 */ 727 struct stat sb; 728 stat(file, &sb); 729 if (S_ISLNK(sb.st_mode)) { 730 free(md5sum); 731 link_count++; 732 continue; 733 } 734 update_existing_entry(db, file, md5sum, rec, 735 &new_count, &link_count, &err_count); 736 free(md5sum); 737 continue; 738 } 739 740 if (md5_status == 1) { 741 /* 742 * The MD5 hash was not present in the database. 743 * This means is either a new file or an updated file. 744 * We should go ahead with parsing. 745 */ 746 if (mflags.verbosity > 1) 747 printf("Parsing: %s\n", file); 748 rec->md5_hash = md5sum; 749 rec->file_path = estrdup(file); 750 // file_path is freed by insert_into_db itself. 751 chdir(parent); 752 begin_parse(file, mp, rec, buf, buflen); 753 if (insert_into_db(db, rec) < 0) { 754 warnx("Error in indexing %s", file); 755 err_count++; 756 } else { 757 new_count++; 758 } 759 } 760 } 761 free(buf); 762 763 sqlite3_finalize(stmt); 764 765 if (mflags.verbosity) { 766 printf("Total Number of new or updated pages enountered = %d\n" 767 "Total number of pages that were successfully" 768 " indexed/updated = %d\n" 769 "Total number of (hard or symbolic) links found = %d\n" 770 "Total number of pages that could not be indexed" 771 " due to errors = %d\n", 772 total_count, new_count, link_count, err_count); 773 } 774 775 if (mflags.recreate) 776 return; 777 778 if (mflags.verbosity) 779 printf("Deleting stale index entries\n"); 780 781 sqlstr = "DELETE FROM mandb_meta WHERE file NOT IN" 782 " (SELECT file FROM metadb.file_cache);" 783 "DELETE FROM mandb_links WHERE md5_hash NOT IN" 784 " (SELECT md5_hash from mandb_meta);" 785 "DROP TABLE metadb.file_cache;" 786 "DELETE FROM mandb WHERE rowid NOT IN" 787 " (SELECT id FROM mandb_meta);"; 788 789 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg); 790 if (errmsg != NULL) { 791 warnx("Removing old entries failed: %s", errmsg); 792 warnx("Please rebuild database from scratch with -f."); 793 free(errmsg); 794 return; 795 } 796 } 797 798 /* 799 * begin_parse -- 800 * parses the man page using libmandoc 801 */ 802 static void 803 begin_parse(const char *file, struct mparse *mp, mandb_rec *rec, 804 const void *buf, size_t len) 805 { 806 struct mdoc *mdoc; 807 struct man *man; 808 mparse_reset(mp); 809 810 rec->xr_found = 0; 811 812 if (mparse_readmem(mp, buf, len, file) >= MANDOCLEVEL_FATAL) { 813 warnx("%s: Parse failure", file); 814 return; 815 } 816 817 mparse_result(mp, &mdoc, &man); 818 if (mdoc == NULL && man == NULL) { 819 warnx("Not a man(7) or mdoc(7) page"); 820 return; 821 } 822 823 set_machine(mdoc, rec); 824 set_section(mdoc, man, rec); 825 if (mdoc) { 826 rec->page_type = MDOC; 827 pmdoc_node(mdoc_node(mdoc), rec); 828 } else { 829 rec->page_type = MAN; 830 pman_node(man_node(man), rec); 831 } 832 } 833 834 /* 835 * set_section -- 836 * Extracts the section number and normalizes it to only the numeric part 837 * (Which should be the first character of the string). 838 */ 839 static void 840 set_section(const struct mdoc *md, const struct man *m, mandb_rec *rec) 841 { 842 if (md) { 843 const struct mdoc_meta *md_meta = mdoc_meta(md); 844 rec->section[0] = md_meta->msec[0]; 845 } else if (m) { 846 const struct man_meta *m_meta = man_meta(m); 847 rec->section[0] = m_meta->msec[0]; 848 } 849 } 850 851 /* 852 * get_machine -- 853 * Extracts the machine architecture information if available. 854 */ 855 static void 856 set_machine(const struct mdoc *md, mandb_rec *rec) 857 { 858 if (md == NULL) 859 return; 860 const struct mdoc_meta *md_meta = mdoc_meta(md); 861 if (md_meta->arch) 862 rec->machine = estrdup(md_meta->arch); 863 } 864 865 static void 866 pmdoc_node(const struct mdoc_node *n, mandb_rec *rec) 867 { 868 869 if (n == NULL) 870 return; 871 872 switch (n->type) { 873 case (MDOC_BODY): 874 /* FALLTHROUGH */ 875 case (MDOC_TAIL): 876 /* FALLTHROUGH */ 877 case (MDOC_ELEM): 878 if (mdocs[n->tok] == NULL) 879 break; 880 (*mdocs[n->tok])(n, rec); 881 break; 882 default: 883 break; 884 } 885 886 pmdoc_node(n->child, rec); 887 pmdoc_node(n->next, rec); 888 } 889 890 /* 891 * pmdoc_Nm -- 892 * Extracts the Name of the manual page from the .Nm macro 893 */ 894 static void 895 pmdoc_Nm(const struct mdoc_node *n, mandb_rec *rec) 896 { 897 if (n->sec != SEC_NAME) 898 return; 899 900 for (n = n->child; n; n = n->next) { 901 if (n->type == MDOC_TEXT) { 902 concat(&rec->name, n->string); 903 } 904 } 905 } 906 907 /* 908 * pmdoc_Nd -- 909 * Extracts the one line description of the man page from the .Nd macro 910 */ 911 static void 912 pmdoc_Nd(const struct mdoc_node *n, mandb_rec *rec) 913 { 914 /* 915 * A static variable for keeping track of whether a Xr macro was seen 916 * previously. 917 */ 918 char *buf = NULL; 919 char *temp; 920 921 if (n == NULL) 922 return; 923 924 if (n->type == MDOC_TEXT) { 925 if (rec->xr_found && n->next) { 926 /* 927 * An Xr macro was seen previously, so parse this 928 * and the next node. 929 */ 930 temp = estrdup(n->string); 931 n = n->next; 932 easprintf(&buf, "%s(%s)", temp, n->string); 933 concat(&rec->name_desc, buf); 934 free(buf); 935 free(temp); 936 } else { 937 concat(&rec->name_desc, n->string); 938 } 939 rec->xr_found = 0; 940 } else if (mdocs[n->tok] == pmdoc_Xr) { 941 /* Remember that we have encountered an Xr macro */ 942 rec->xr_found = 1; 943 } 944 945 if (n->child) 946 pmdoc_Nd(n->child, rec); 947 948 if(n->next) 949 pmdoc_Nd(n->next, rec); 950 } 951 952 /* 953 * pmdoc_macro_handler-- 954 * This function is a single point of handling all the special macros that we 955 * want to handle especially. For example the .Xr macro for properly parsing 956 * the referenced page name along with the section number, or the .Pp macro 957 * for adding a new line whenever we encounter it. 958 */ 959 static void 960 pmdoc_macro_handler(const struct mdoc_node *n, mandb_rec *rec, enum mdoct doct) 961 { 962 const struct mdoc_node *sn; 963 assert(n); 964 965 switch (doct) { 966 /* Parse the man page references. 967 * Basically the .Xr macros are used like: 968 * .Xr ls 1 969 * and formatted like this: 970 * ls(1) 971 * Prepare a buffer to format the data like the above example and call 972 * pmdoc_parse_section to append it. 973 */ 974 case MDOC_Xr: 975 n = n->child; 976 while (n->type != MDOC_TEXT && n->next) 977 n = n->next; 978 979 if (n && n->type != MDOC_TEXT) 980 return; 981 sn = n; 982 if (n->next) 983 n = n->next; 984 985 while (n->type != MDOC_TEXT && n->next) 986 n = n->next; 987 988 if (n && n->type == MDOC_TEXT) { 989 size_t len = strlen(sn->string); 990 char *buf = emalloc(len + 4); 991 memcpy(buf, sn->string, len); 992 buf[len] = '('; 993 buf[len + 1] = n->string[0]; 994 buf[len + 2] = ')'; 995 buf[len + 3] = 0; 996 mdoc_parse_section(n->sec, buf, rec); 997 free(buf); 998 } 999 1000 break; 1001 1002 /* Parse the .Pp macro to add a new line */ 1003 case MDOC_Pp: 1004 if (n->type == MDOC_TEXT) 1005 mdoc_parse_section(n->sec, "\n", rec); 1006 break; 1007 default: 1008 break; 1009 } 1010 1011 } 1012 1013 /* 1014 * pmdoc_Xr, pmdoc_Pp-- 1015 * Empty stubs. 1016 * The parser calls these functions each time it encounters 1017 * a .Xr or .Pp macro. We are parsing all the data from 1018 * the pmdoc_Sh function, so don't do anything here. 1019 * (See if else blocks in pmdoc_Sh.) 1020 */ 1021 static void 1022 pmdoc_Xr(const struct mdoc_node *n, mandb_rec *rec) 1023 { 1024 } 1025 1026 static void 1027 pmdoc_Pp(const struct mdoc_node *n, mandb_rec *rec) 1028 { 1029 } 1030 1031 /* 1032 * pmdoc_Sh -- 1033 * Called when a .Sh macro is encountered and loops through its body, calling 1034 * mdoc_parse_section to append the data to the section specific buffer. 1035 * Two special macros which may occur inside the body of Sh are .Nm and .Xr and 1036 * they need special handling, thus the separate if branches for them. 1037 */ 1038 static void 1039 pmdoc_Sh(const struct mdoc_node *n, mandb_rec *rec) 1040 { 1041 if (n == NULL) 1042 return; 1043 int xr_found = 0; 1044 1045 if (n->type == MDOC_TEXT) { 1046 mdoc_parse_section(n->sec, n->string, rec); 1047 } else if (mdocs[n->tok] == pmdoc_Nm && rec->name != NULL) { 1048 /* 1049 * When encountering a .Nm macro, substitute it 1050 * with its previously cached value of the argument. 1051 */ 1052 mdoc_parse_section(n->sec, rec->name, rec); 1053 } else if (mdocs[n->tok] == pmdoc_Xr) { 1054 /* 1055 * When encountering other inline macros, 1056 * call pmdoc_macro_handler. 1057 */ 1058 pmdoc_macro_handler(n, rec, MDOC_Xr); 1059 xr_found = 1; 1060 } else if (mdocs[n->tok] == pmdoc_Pp) { 1061 pmdoc_macro_handler(n, rec, MDOC_Pp); 1062 } 1063 1064 /* 1065 * If an Xr macro was encountered then the child node has 1066 * already been explored by pmdoc_macro_handler. 1067 */ 1068 if (xr_found == 0) 1069 pmdoc_Sh(n->child, rec); 1070 pmdoc_Sh(n->next, rec); 1071 } 1072 1073 /* 1074 * mdoc_parse_section-- 1075 * Utility function for parsing sections of the mdoc type pages. 1076 * Takes two params: 1077 * 1. sec is an enum which indicates the section in which we are present 1078 * 2. string is the string which we need to append to the secbuff for this 1079 * particular section. 1080 * The function appends string to the global section buffer and returns. 1081 */ 1082 static void 1083 mdoc_parse_section(enum mdoc_sec sec, const char *string, mandb_rec *rec) 1084 { 1085 /* 1086 * If the user specified the 'l' flag, then parse and store only the 1087 * NAME section. Ignore the rest. 1088 */ 1089 if (mflags.limit) 1090 return; 1091 1092 switch (sec) { 1093 case SEC_LIBRARY: 1094 append(&rec->lib, string); 1095 break; 1096 case SEC_RETURN_VALUES: 1097 append(&rec->return_vals, string); 1098 break; 1099 case SEC_ENVIRONMENT: 1100 append(&rec->env, string); 1101 break; 1102 case SEC_FILES: 1103 append(&rec->files, string); 1104 break; 1105 case SEC_EXIT_STATUS: 1106 append(&rec->exit_status, string); 1107 break; 1108 case SEC_DIAGNOSTICS: 1109 append(&rec->diagnostics, string); 1110 break; 1111 case SEC_ERRORS: 1112 append(&rec->errors, string); 1113 break; 1114 case SEC_NAME: 1115 case SEC_SYNOPSIS: 1116 case SEC_EXAMPLES: 1117 case SEC_STANDARDS: 1118 case SEC_HISTORY: 1119 case SEC_AUTHORS: 1120 case SEC_BUGS: 1121 break; 1122 default: 1123 append(&rec->desc, string); 1124 break; 1125 } 1126 } 1127 1128 static void 1129 pman_node(const struct man_node *n, mandb_rec *rec) 1130 { 1131 if (n == NULL) 1132 return; 1133 1134 switch (n->type) { 1135 case (MAN_BODY): 1136 /* FALLTHROUGH */ 1137 case (MAN_TAIL): 1138 /* FALLTHROUGH */ 1139 case (MAN_BLOCK): 1140 /* FALLTHROUGH */ 1141 case (MAN_ELEM): 1142 if (mans[n->tok] != NULL) 1143 (*mans[n->tok])(n, rec); 1144 break; 1145 default: 1146 break; 1147 } 1148 1149 pman_node(n->child, rec); 1150 pman_node(n->next, rec); 1151 } 1152 1153 /* 1154 * pman_parse_name -- 1155 * Parses the NAME section and puts the complete content in the name_desc 1156 * variable. 1157 */ 1158 static void 1159 pman_parse_name(const struct man_node *n, mandb_rec *rec) 1160 { 1161 if (n == NULL) 1162 return; 1163 1164 if (n->type == MAN_TEXT) { 1165 char *tmp = parse_escape(n->string); 1166 concat(&rec->name_desc, tmp); 1167 free(tmp); 1168 } 1169 1170 if (n->child) 1171 pman_parse_name(n->child, rec); 1172 1173 if(n->next) 1174 pman_parse_name(n->next, rec); 1175 } 1176 1177 /* 1178 * A stub function to be able to parse the macros like .B embedded inside 1179 * a section. 1180 */ 1181 static void 1182 pman_block(const struct man_node *n, mandb_rec *rec) 1183 { 1184 } 1185 1186 /* 1187 * pman_sh -- 1188 * This function does one of the two things: 1189 * 1. If the present section is NAME, then it will: 1190 * (a) Extract the name of the page (in case of multiple comma separated 1191 * names, it will pick up the first one). 1192 * (b) Build a space spearated list of all the symlinks/hardlinks to 1193 * this page and store in the buffer 'links'. These are extracted from 1194 * the comma separated list of names in the NAME section as well. 1195 * (c) Move on to the one line description section, which is after the list 1196 * of names in the NAME section. 1197 * 2. Otherwise, it will check the section name and call the man_parse_section 1198 * function, passing the enum corresponding that section. 1199 */ 1200 static void 1201 pman_sh(const struct man_node *n, mandb_rec *rec) 1202 { 1203 static const struct { 1204 enum man_sec section; 1205 const char *header; 1206 } mapping[] = { 1207 { MANSEC_DESCRIPTION, "DESCRIPTION" }, 1208 { MANSEC_SYNOPSIS, "SYNOPSIS" }, 1209 { MANSEC_LIBRARY, "LIBRARY" }, 1210 { MANSEC_ERRORS, "ERRORS" }, 1211 { MANSEC_FILES, "FILES" }, 1212 { MANSEC_RETURN_VALUES, "RETURN VALUE" }, 1213 { MANSEC_RETURN_VALUES, "RETURN VALUES" }, 1214 { MANSEC_EXIT_STATUS, "EXIT STATUS" }, 1215 { MANSEC_EXAMPLES, "EXAMPLES" }, 1216 { MANSEC_EXAMPLES, "EXAMPLE" }, 1217 { MANSEC_STANDARDS, "STANDARDS" }, 1218 { MANSEC_HISTORY, "HISTORY" }, 1219 { MANSEC_BUGS, "BUGS" }, 1220 { MANSEC_AUTHORS, "AUTHORS" }, 1221 { MANSEC_COPYRIGHT, "COPYRIGHT" }, 1222 }; 1223 const struct man_node *head; 1224 char *name_desc; 1225 int sz; 1226 size_t i; 1227 1228 if ((head = n->parent->head) == NULL || (head = head->child) == NULL || 1229 head->type != MAN_TEXT) 1230 return; 1231 1232 /* 1233 * Check if this section should be extracted and 1234 * where it should be stored. Handled the trival cases first. 1235 */ 1236 for (i = 0; i < sizeof(mapping) / sizeof(mapping[0]); ++i) { 1237 if (strcmp(head->string, mapping[i].header) == 0) { 1238 man_parse_section(mapping[i].section, n, rec); 1239 return; 1240 } 1241 } 1242 1243 if (strcmp(head->string, "NAME") == 0) { 1244 /* 1245 * We are in the NAME section. 1246 * pman_parse_name will put the complete content in name_desc. 1247 */ 1248 pman_parse_name(n, rec); 1249 1250 name_desc = rec->name_desc; 1251 1252 /* Remove any leading spaces. */ 1253 while (name_desc[0] == ' ') 1254 name_desc++; 1255 1256 /* If the line begins with a "\&", avoid those */ 1257 if (name_desc[0] == '\\' && name_desc[1] == '&') 1258 name_desc += 2; 1259 1260 /* Now name_desc should be left with a comma-space 1261 * separated list of names and the one line description 1262 * of the page: 1263 * "a, b, c \- sample description" 1264 * Take out the first name, before the first comma 1265 * (or space) and store it in rec->name. 1266 * If the page has aliases then they should be 1267 * in the form of a comma separated list. 1268 * Keep looping while there is a comma in name_desc, 1269 * extract the alias name and store in rec->links. 1270 * When there are no more commas left, break out. 1271 */ 1272 int has_alias = 0; // Any more aliases left? 1273 while (*name_desc) { 1274 /* Remove any leading spaces or hyphens. */ 1275 if (name_desc[0] == ' ' || name_desc[0] =='-') { 1276 name_desc++; 1277 continue; 1278 } 1279 sz = strcspn(name_desc, ", "); 1280 1281 /* Extract the first term and store it in rec->name. */ 1282 if (rec->name == NULL) { 1283 if (name_desc[sz] == ',') 1284 has_alias = 1; 1285 name_desc[sz] = 0; 1286 rec->name = emalloc(sz + 1); 1287 memcpy(rec->name, name_desc, sz + 1); 1288 name_desc += sz + 1; 1289 continue; 1290 } 1291 1292 /* 1293 * Once rec->name is set, rest of the names 1294 * are to be treated as links or aliases. 1295 */ 1296 if (rec->name && has_alias) { 1297 if (name_desc[sz] != ',') { 1298 /* No more commas left --> 1299 * no more aliases to take out 1300 */ 1301 has_alias = 0; 1302 } 1303 name_desc[sz] = 0; 1304 concat2(&rec->links, name_desc, sz); 1305 name_desc += sz + 1; 1306 continue; 1307 } 1308 break; 1309 } 1310 1311 /* Parse any escape sequences that might be there */ 1312 char *temp = parse_escape(name_desc); 1313 free(rec->name_desc); 1314 rec->name_desc = temp; 1315 temp = parse_escape(rec->name); 1316 free(rec->name); 1317 rec->name = temp; 1318 return; 1319 } 1320 1321 /* The RETURN VALUE section might be specified in multiple ways */ 1322 if (strcmp(head->string, "RETURN") == 0 && 1323 head->next != NULL && head->next->type == MAN_TEXT && 1324 (strcmp(head->next->string, "VALUE") == 0 || 1325 strcmp(head->next->string, "VALUES") == 0)) { 1326 man_parse_section(MANSEC_RETURN_VALUES, n, rec); 1327 return; 1328 } 1329 1330 /* 1331 * EXIT STATUS section can also be specified all on one line or on two 1332 * separate lines. 1333 */ 1334 if (strcmp(head->string, "EXIT") == 0 && 1335 head->next != NULL && head->next->type == MAN_TEXT && 1336 strcmp(head->next->string, "STATUS") == 0) { 1337 man_parse_section(MANSEC_EXIT_STATUS, n, rec); 1338 return; 1339 } 1340 1341 /* Store the rest of the content in desc. */ 1342 man_parse_section(MANSEC_NONE, n, rec); 1343 } 1344 1345 /* 1346 * pman_parse_node -- 1347 * Generic function to iterate through a node. Usually called from 1348 * man_parse_section to parse a particular section of the man page. 1349 */ 1350 static void 1351 pman_parse_node(const struct man_node *n, secbuff *s) 1352 { 1353 if (n == NULL) 1354 return; 1355 1356 if (n->type == MAN_TEXT) 1357 append(s, n->string); 1358 1359 pman_parse_node(n->child, s); 1360 pman_parse_node(n->next, s); 1361 } 1362 1363 /* 1364 * man_parse_section -- 1365 * Takes two parameters: 1366 * sec: Tells which section we are present in 1367 * n: Is the present node of the AST. 1368 * Depending on the section, we call pman_parse_node to parse that section and 1369 * concatenate the content from that section into the buffer for that section. 1370 */ 1371 static void 1372 man_parse_section(enum man_sec sec, const struct man_node *n, mandb_rec *rec) 1373 { 1374 /* 1375 * If the user sepecified the 'l' flag then just parse 1376 * the NAME section, ignore the rest. 1377 */ 1378 if (mflags.limit) 1379 return; 1380 1381 switch (sec) { 1382 case MANSEC_LIBRARY: 1383 pman_parse_node(n, &rec->lib); 1384 break; 1385 case MANSEC_RETURN_VALUES: 1386 pman_parse_node(n, &rec->return_vals); 1387 break; 1388 case MANSEC_ENVIRONMENT: 1389 pman_parse_node(n, &rec->env); 1390 break; 1391 case MANSEC_FILES: 1392 pman_parse_node(n, &rec->files); 1393 break; 1394 case MANSEC_EXIT_STATUS: 1395 pman_parse_node(n, &rec->exit_status); 1396 break; 1397 case MANSEC_DIAGNOSTICS: 1398 pman_parse_node(n, &rec->diagnostics); 1399 break; 1400 case MANSEC_ERRORS: 1401 pman_parse_node(n, &rec->errors); 1402 break; 1403 case MANSEC_NAME: 1404 case MANSEC_SYNOPSIS: 1405 case MANSEC_EXAMPLES: 1406 case MANSEC_STANDARDS: 1407 case MANSEC_HISTORY: 1408 case MANSEC_BUGS: 1409 case MANSEC_AUTHORS: 1410 case MANSEC_COPYRIGHT: 1411 break; 1412 default: 1413 pman_parse_node(n, &rec->desc); 1414 break; 1415 } 1416 1417 } 1418 1419 /* 1420 * insert_into_db -- 1421 * Inserts the parsed data of the man page in the Sqlite databse. 1422 * If any of the values is NULL, then we cleanup and return -1 indicating 1423 * an error. 1424 * Otherwise, store the data in the database and return 0. 1425 */ 1426 static int 1427 insert_into_db(sqlite3 *db, mandb_rec *rec) 1428 { 1429 int rc = 0; 1430 int idx = -1; 1431 const char *sqlstr = NULL; 1432 sqlite3_stmt *stmt = NULL; 1433 char *ln = NULL; 1434 char *errmsg = NULL; 1435 long int mandb_rowid; 1436 1437 /* 1438 * At the very minimum we want to make sure that we store 1439 * the following data: 1440 * Name, one line description, and the MD5 hash 1441 */ 1442 if (rec->name == NULL || rec->name_desc == NULL || 1443 rec->md5_hash == NULL) { 1444 cleanup(rec); 1445 return -1; 1446 } 1447 1448 /* Write null byte at the end of all the sec_buffs */ 1449 rec->desc.data[rec->desc.offset] = 0; 1450 rec->lib.data[rec->lib.offset] = 0; 1451 rec->env.data[rec->env.offset] = 0; 1452 rec->return_vals.data[rec->return_vals.offset] = 0; 1453 rec->exit_status.data[rec->exit_status.offset] = 0; 1454 rec->files.data[rec->files.offset] = 0; 1455 rec->diagnostics.data[rec->diagnostics.offset] = 0; 1456 rec->errors.data[rec->errors.offset] = 0; 1457 1458 /* 1459 * In case of a mdoc page: (sorry, no better place to put this code) 1460 * parse the comma separated list of names of man pages, 1461 * the first name will be stored in the mandb table, rest will be 1462 * treated as links and put in the mandb_links table. 1463 */ 1464 if (rec->page_type == MDOC) { 1465 char *tmp; 1466 rec->links = estrdup(rec->name); 1467 free(rec->name); 1468 int sz = strcspn(rec->links, " \0"); 1469 rec->name = emalloc(sz + 1); 1470 memcpy(rec->name, rec->links, sz); 1471 if(rec->name[sz - 1] == ',') 1472 rec->name[sz - 1] = 0; 1473 else 1474 rec->name[sz] = 0; 1475 while (rec->links[sz] == ' ') 1476 ++sz; 1477 tmp = estrdup(rec->links + sz); 1478 free(rec->links); 1479 rec->links = tmp; 1480 } 1481 1482 /*------------------------ Populate the mandb table---------------------------*/ 1483 sqlstr = "INSERT INTO mandb VALUES (:section, :name, :name_desc, :desc," 1484 " :lib, :return_vals, :env, :files, :exit_status," 1485 " :diagnostics, :errors, :md5_hash, :machine)"; 1486 1487 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); 1488 if (rc != SQLITE_OK) 1489 goto Out; 1490 1491 idx = sqlite3_bind_parameter_index(stmt, ":name"); 1492 rc = sqlite3_bind_text(stmt, idx, rec->name, -1, NULL); 1493 if (rc != SQLITE_OK) { 1494 sqlite3_finalize(stmt); 1495 goto Out; 1496 } 1497 1498 idx = sqlite3_bind_parameter_index(stmt, ":section"); 1499 rc = sqlite3_bind_text(stmt, idx, rec->section, -1, NULL); 1500 if (rc != SQLITE_OK) { 1501 sqlite3_finalize(stmt); 1502 goto Out; 1503 } 1504 1505 idx = sqlite3_bind_parameter_index(stmt, ":name_desc"); 1506 rc = sqlite3_bind_text(stmt, idx, rec->name_desc, -1, NULL); 1507 if (rc != SQLITE_OK) { 1508 sqlite3_finalize(stmt); 1509 goto Out; 1510 } 1511 1512 idx = sqlite3_bind_parameter_index(stmt, ":desc"); 1513 rc = sqlite3_bind_text(stmt, idx, rec->desc.data, 1514 rec->desc.offset + 1, NULL); 1515 if (rc != SQLITE_OK) { 1516 sqlite3_finalize(stmt); 1517 goto Out; 1518 } 1519 1520 idx = sqlite3_bind_parameter_index(stmt, ":lib"); 1521 rc = sqlite3_bind_text(stmt, idx, rec->lib.data, rec->lib.offset + 1, NULL); 1522 if (rc != SQLITE_OK) { 1523 sqlite3_finalize(stmt); 1524 goto Out; 1525 } 1526 1527 idx = sqlite3_bind_parameter_index(stmt, ":return_vals"); 1528 rc = sqlite3_bind_text(stmt, idx, rec->return_vals.data, 1529 rec->return_vals.offset + 1, NULL); 1530 if (rc != SQLITE_OK) { 1531 sqlite3_finalize(stmt); 1532 goto Out; 1533 } 1534 1535 idx = sqlite3_bind_parameter_index(stmt, ":env"); 1536 rc = sqlite3_bind_text(stmt, idx, rec->env.data, rec->env.offset + 1, NULL); 1537 if (rc != SQLITE_OK) { 1538 sqlite3_finalize(stmt); 1539 goto Out; 1540 } 1541 1542 idx = sqlite3_bind_parameter_index(stmt, ":files"); 1543 rc = sqlite3_bind_text(stmt, idx, rec->files.data, 1544 rec->files.offset + 1, NULL); 1545 if (rc != SQLITE_OK) { 1546 sqlite3_finalize(stmt); 1547 goto Out; 1548 } 1549 1550 idx = sqlite3_bind_parameter_index(stmt, ":exit_status"); 1551 rc = sqlite3_bind_text(stmt, idx, rec->exit_status.data, 1552 rec->exit_status.offset + 1, NULL); 1553 if (rc != SQLITE_OK) { 1554 sqlite3_finalize(stmt); 1555 goto Out; 1556 } 1557 1558 idx = sqlite3_bind_parameter_index(stmt, ":diagnostics"); 1559 rc = sqlite3_bind_text(stmt, idx, rec->diagnostics.data, 1560 rec->diagnostics.offset + 1, NULL); 1561 if (rc != SQLITE_OK) { 1562 sqlite3_finalize(stmt); 1563 goto Out; 1564 } 1565 1566 idx = sqlite3_bind_parameter_index(stmt, ":errors"); 1567 rc = sqlite3_bind_text(stmt, idx, rec->errors.data, 1568 rec->errors.offset + 1, NULL); 1569 if (rc != SQLITE_OK) { 1570 sqlite3_finalize(stmt); 1571 goto Out; 1572 } 1573 1574 idx = sqlite3_bind_parameter_index(stmt, ":md5_hash"); 1575 rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL); 1576 if (rc != SQLITE_OK) { 1577 sqlite3_finalize(stmt); 1578 goto Out; 1579 } 1580 1581 idx = sqlite3_bind_parameter_index(stmt, ":machine"); 1582 if (rec->machine) 1583 rc = sqlite3_bind_text(stmt, idx, rec->machine, -1, NULL); 1584 else 1585 rc = sqlite3_bind_null(stmt, idx); 1586 if (rc != SQLITE_OK) { 1587 sqlite3_finalize(stmt); 1588 goto Out; 1589 } 1590 1591 rc = sqlite3_step(stmt); 1592 if (rc != SQLITE_DONE) { 1593 sqlite3_finalize(stmt); 1594 goto Out; 1595 } 1596 1597 sqlite3_finalize(stmt); 1598 1599 /* Get the row id of the last inserted row */ 1600 mandb_rowid = sqlite3_last_insert_rowid(db); 1601 1602 /*------------------------Populate the mandb_meta table-----------------------*/ 1603 sqlstr = "INSERT INTO mandb_meta VALUES (:device, :inode, :mtime," 1604 " :file, :md5_hash, :id)"; 1605 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); 1606 if (rc != SQLITE_OK) 1607 goto Out; 1608 1609 idx = sqlite3_bind_parameter_index(stmt, ":device"); 1610 rc = sqlite3_bind_int64(stmt, idx, rec->device); 1611 if (rc != SQLITE_OK) { 1612 sqlite3_finalize(stmt); 1613 goto Out; 1614 } 1615 1616 idx = sqlite3_bind_parameter_index(stmt, ":inode"); 1617 rc = sqlite3_bind_int64(stmt, idx, rec->inode); 1618 if (rc != SQLITE_OK) { 1619 sqlite3_finalize(stmt); 1620 goto Out; 1621 } 1622 1623 idx = sqlite3_bind_parameter_index(stmt, ":mtime"); 1624 rc = sqlite3_bind_int64(stmt, idx, rec->mtime); 1625 if (rc != SQLITE_OK) { 1626 sqlite3_finalize(stmt); 1627 goto Out; 1628 } 1629 1630 idx = sqlite3_bind_parameter_index(stmt, ":file"); 1631 rc = sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL); 1632 if (rc != SQLITE_OK) { 1633 sqlite3_finalize(stmt); 1634 goto Out; 1635 } 1636 1637 idx = sqlite3_bind_parameter_index(stmt, ":md5_hash"); 1638 rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL); 1639 if (rc != SQLITE_OK) { 1640 sqlite3_finalize(stmt); 1641 goto Out; 1642 } 1643 1644 idx = sqlite3_bind_parameter_index(stmt, ":id"); 1645 rc = sqlite3_bind_int64(stmt, idx, mandb_rowid); 1646 if (rc != SQLITE_OK) { 1647 sqlite3_finalize(stmt); 1648 goto Out; 1649 } 1650 1651 rc = sqlite3_step(stmt); 1652 sqlite3_finalize(stmt); 1653 if (rc == SQLITE_CONSTRAINT) { 1654 /* The *most* probable reason for reaching here is that 1655 * the UNIQUE contraint on the file column of the mandb_meta 1656 * table was violated. 1657 * This can happen when a file was updated/modified. 1658 * To fix this we need to do two things: 1659 * 1. Delete the row for the older version of this file 1660 * from mandb table. 1661 * 2. Run an UPDATE query to update the row for this file 1662 * in the mandb_meta table. 1663 */ 1664 warnx("Trying to update index for %s", rec->file_path); 1665 char *sql = sqlite3_mprintf("DELETE FROM mandb " 1666 "WHERE rowid = (SELECT id" 1667 " FROM mandb_meta" 1668 " WHERE file = %Q)", 1669 rec->file_path); 1670 sqlite3_exec(db, sql, NULL, NULL, &errmsg); 1671 sqlite3_free(sql); 1672 if (errmsg != NULL) { 1673 warnx("%s", errmsg); 1674 free(errmsg); 1675 } 1676 sqlstr = "UPDATE mandb_meta SET device = :device," 1677 " inode = :inode, mtime = :mtime, id = :id," 1678 " md5_hash = :md5 WHERE file = :file"; 1679 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); 1680 if (rc != SQLITE_OK) { 1681 warnx("Update failed with error: %s", 1682 sqlite3_errmsg(db)); 1683 close_db(db); 1684 cleanup(rec); 1685 errx(EXIT_FAILURE, 1686 "Consider running makemandb with -f option"); 1687 } 1688 1689 idx = sqlite3_bind_parameter_index(stmt, ":device"); 1690 sqlite3_bind_int64(stmt, idx, rec->device); 1691 idx = sqlite3_bind_parameter_index(stmt, ":inode"); 1692 sqlite3_bind_int64(stmt, idx, rec->inode); 1693 idx = sqlite3_bind_parameter_index(stmt, ":mtime"); 1694 sqlite3_bind_int64(stmt, idx, rec->mtime); 1695 idx = sqlite3_bind_parameter_index(stmt, ":id"); 1696 sqlite3_bind_int64(stmt, idx, mandb_rowid); 1697 idx = sqlite3_bind_parameter_index(stmt, ":md5"); 1698 sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL); 1699 idx = sqlite3_bind_parameter_index(stmt, ":file"); 1700 sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL); 1701 rc = sqlite3_step(stmt); 1702 sqlite3_finalize(stmt); 1703 1704 if (rc != SQLITE_DONE) { 1705 warnx("%s", sqlite3_errmsg(db)); 1706 close_db(db); 1707 cleanup(rec); 1708 errx(EXIT_FAILURE, 1709 "Consider running makemandb with -f option"); 1710 } 1711 } else if (rc != SQLITE_DONE) { 1712 /* Otherwise make this error fatal */ 1713 warnx("Failed at %s\n%s", rec->file_path, sqlite3_errmsg(db)); 1714 cleanup(rec); 1715 close_db(db); 1716 exit(EXIT_FAILURE); 1717 } 1718 1719 /*------------------------ Populate the mandb_links table---------------------*/ 1720 char *str = NULL; 1721 char *links; 1722 if (rec->links && strlen(rec->links)) { 1723 links = rec->links; 1724 for(ln = strtok(links, " "); ln; ln = strtok(NULL, " ")) { 1725 if (ln[0] == ',') 1726 ln++; 1727 if(ln[strlen(ln) - 1] == ',') 1728 ln[strlen(ln) - 1] = 0; 1729 1730 str = sqlite3_mprintf("INSERT INTO mandb_links" 1731 " VALUES (%Q, %Q, %Q, %Q, %Q)", 1732 ln, rec->name, rec->section, 1733 rec->machine, rec->md5_hash); 1734 sqlite3_exec(db, str, NULL, NULL, &errmsg); 1735 sqlite3_free(str); 1736 if (errmsg != NULL) { 1737 warnx("%s", errmsg); 1738 cleanup(rec); 1739 free(errmsg); 1740 return -1; 1741 } 1742 } 1743 } 1744 1745 cleanup(rec); 1746 return 0; 1747 1748 Out: 1749 warnx("%s", sqlite3_errmsg(db)); 1750 cleanup(rec); 1751 return -1; 1752 } 1753 1754 /* 1755 * check_md5-- 1756 * Generates the md5 hash of the file and checks if it already doesn't exist 1757 * in the table (passed as the 3rd parameter). 1758 * This function is being used to avoid hardlinks. 1759 * On successful completion it will also set the value of the fourth parameter 1760 * to the md5 hash of the file (computed previously). It is the responsibility 1761 * of the caller to free this buffer. 1762 * Return values: 1763 * -1: If an error occurs somewhere and sets the md5 return buffer to NULL. 1764 * 0: If the md5 hash does not exist in the table. 1765 * 1: If the hash exists in the database. 1766 */ 1767 static int 1768 check_md5(const char *file, sqlite3 *db, const char *table, char **md5sum, 1769 void *buf, size_t buflen) 1770 { 1771 int rc = 0; 1772 int idx = -1; 1773 char *sqlstr = NULL; 1774 sqlite3_stmt *stmt = NULL; 1775 1776 assert(file != NULL); 1777 *md5sum = MD5Data(buf, buflen, NULL); 1778 if (*md5sum == NULL) { 1779 warn("md5 failed: %s", file); 1780 return -1; 1781 } 1782 1783 easprintf(&sqlstr, "SELECT * FROM %s WHERE md5_hash = :md5_hash", 1784 table); 1785 rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL); 1786 if (rc != SQLITE_OK) { 1787 free(sqlstr); 1788 free(*md5sum); 1789 *md5sum = NULL; 1790 return -1; 1791 } 1792 1793 idx = sqlite3_bind_parameter_index(stmt, ":md5_hash"); 1794 rc = sqlite3_bind_text(stmt, idx, *md5sum, -1, NULL); 1795 if (rc != SQLITE_OK) { 1796 warnx("%s", sqlite3_errmsg(db)); 1797 sqlite3_finalize(stmt); 1798 free(sqlstr); 1799 free(*md5sum); 1800 *md5sum = NULL; 1801 return -1; 1802 } 1803 1804 if (sqlite3_step(stmt) == SQLITE_ROW) { 1805 sqlite3_finalize(stmt); 1806 free(sqlstr); 1807 return 0; 1808 } 1809 1810 sqlite3_finalize(stmt); 1811 free(sqlstr); 1812 return 1; 1813 } 1814 1815 /* Optimize the index for faster search */ 1816 static void 1817 optimize(sqlite3 *db) 1818 { 1819 const char *sqlstr; 1820 char *errmsg = NULL; 1821 1822 if (mflags.verbosity) 1823 printf("Optimizing the database index\n"); 1824 sqlstr = "INSERT INTO mandb(mandb) VALUES (\'optimize\');" 1825 "VACUUM"; 1826 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg); 1827 if (errmsg != NULL) { 1828 warnx("%s", errmsg); 1829 free(errmsg); 1830 return; 1831 } 1832 } 1833 1834 /* 1835 * cleanup -- 1836 * cleans up the global buffers 1837 */ 1838 static void 1839 cleanup(mandb_rec *rec) 1840 { 1841 rec->desc.offset = 0; 1842 rec->lib.offset = 0; 1843 rec->return_vals.offset = 0; 1844 rec->env.offset = 0; 1845 rec->exit_status.offset = 0; 1846 rec->diagnostics.offset = 0; 1847 rec->errors.offset = 0; 1848 rec->files.offset = 0; 1849 1850 free(rec->machine); 1851 rec->machine = NULL; 1852 1853 free(rec->links); 1854 rec->links = NULL; 1855 1856 free(rec->file_path); 1857 rec->file_path = NULL; 1858 1859 free(rec->name); 1860 rec->name = NULL; 1861 1862 free(rec->name_desc); 1863 rec->name_desc = NULL; 1864 1865 free(rec->md5_hash); 1866 rec->md5_hash = NULL; 1867 } 1868 1869 /* 1870 * init_secbuffs-- 1871 * Sets the value of buflen for all the sec_buff field of rec. And then 1872 * allocate memory to each sec_buff member of rec. 1873 */ 1874 static void 1875 init_secbuffs(mandb_rec *rec) 1876 { 1877 /* 1878 * Some sec_buff might need more memory, for example desc, 1879 * which stores the data of the DESCRIPTION section, 1880 * while some might need very small amount of memory. 1881 * Therefore explicitly setting the value of buflen field for 1882 * each sec_buff. 1883 */ 1884 rec->desc.buflen = 10 * BUFLEN; 1885 rec->desc.data = emalloc(rec->desc.buflen); 1886 rec->desc.offset = 0; 1887 1888 rec->lib.buflen = BUFLEN / 2; 1889 rec->lib.data = emalloc(rec->lib.buflen); 1890 rec->lib.offset = 0; 1891 1892 rec->return_vals.buflen = BUFLEN; 1893 rec->return_vals.data = emalloc(rec->return_vals.buflen); 1894 rec->return_vals.offset = 0; 1895 1896 rec->exit_status.buflen = BUFLEN; 1897 rec->exit_status.data = emalloc(rec->exit_status.buflen); 1898 rec->exit_status.offset = 0; 1899 1900 rec->env.buflen = BUFLEN; 1901 rec->env.data = emalloc(rec->env.buflen); 1902 rec->env.offset = 0; 1903 1904 rec->files.buflen = BUFLEN; 1905 rec->files.data = emalloc(rec->files.buflen); 1906 rec->files.offset = 0; 1907 1908 rec->diagnostics.buflen = BUFLEN; 1909 rec->diagnostics.data = emalloc(rec->diagnostics.buflen); 1910 rec->diagnostics.offset = 0; 1911 1912 rec->errors.buflen = BUFLEN; 1913 rec->errors.data = emalloc(rec->errors.buflen); 1914 rec->errors.offset = 0; 1915 } 1916 1917 /* 1918 * free_secbuffs-- 1919 * This function should be called at the end, when all the pages have been 1920 * parsed. 1921 * It frees the memory allocated to sec_buffs by init_secbuffs in the starting. 1922 */ 1923 static void 1924 free_secbuffs(mandb_rec *rec) 1925 { 1926 free(rec->desc.data); 1927 free(rec->lib.data); 1928 free(rec->return_vals.data); 1929 free(rec->exit_status.data); 1930 free(rec->env.data); 1931 free(rec->files.data); 1932 free(rec->diagnostics.data); 1933 free(rec->errors.data); 1934 } 1935 1936 static void 1937 replace_hyph(char *str) 1938 { 1939 char *iter = str; 1940 while ((iter = strchr(iter, ASCII_HYPH)) != NULL) 1941 *iter = '-'; 1942 } 1943 1944 static char * 1945 parse_escape(const char *str) 1946 { 1947 const char *backslash, *last_backslash; 1948 char *result, *iter; 1949 size_t len; 1950 1951 assert(str); 1952 1953 last_backslash = str; 1954 backslash = strchr(str, '\\'); 1955 if (backslash == NULL) { 1956 result = estrdup(str); 1957 replace_hyph(result); 1958 return result; 1959 } 1960 1961 result = emalloc(strlen(str) + 1); 1962 iter = result; 1963 1964 do { 1965 len = backslash - last_backslash; 1966 memcpy(iter, last_backslash, len); 1967 iter += len; 1968 if (backslash[1] == '-' || backslash[1] == ' ') { 1969 *iter++ = backslash[1]; 1970 last_backslash = backslash + 2; 1971 backslash = strchr(backslash + 2, '\\'); 1972 } else { 1973 ++backslash; 1974 mandoc_escape(&backslash, NULL, NULL); 1975 last_backslash = backslash; 1976 if (backslash == NULL) 1977 break; 1978 backslash = strchr(last_backslash, '\\'); 1979 } 1980 } while (backslash != NULL); 1981 if (last_backslash != NULL) 1982 strcpy(iter, last_backslash); 1983 1984 replace_hyph(result); 1985 return result; 1986 } 1987 1988 /* 1989 * append-- 1990 * Concatenates a space and src at the end of sbuff->data (much like concat in 1991 * apropos-utils.c). 1992 * Rather than reallocating space for writing data, it uses the value of the 1993 * offset field of sec_buff to write new data at the free space left in the 1994 * buffer. 1995 * In case the size of the data to be appended exceeds the number of bytes left 1996 * in the buffer, it reallocates buflen number of bytes and then continues. 1997 * Value of offset field should be adjusted as new data is written. 1998 * 1999 * NOTE: This function does not write the null byte at the end of the buffers, 2000 * write a null byte at the position pointed to by offset before inserting data 2001 * in the db. 2002 */ 2003 static void 2004 append(secbuff *sbuff, const char *src) 2005 { 2006 short flag = 0; 2007 size_t srclen, newlen; 2008 char *temp; 2009 2010 assert(src != NULL); 2011 temp = parse_escape(src); 2012 srclen = strlen(temp); 2013 2014 if (sbuff->data == NULL) { 2015 sbuff->data = emalloc(sbuff->buflen); 2016 sbuff->offset = 0; 2017 } 2018 2019 newlen = sbuff->offset + srclen + 2; 2020 if (newlen >= sbuff->buflen) { 2021 while (sbuff->buflen < newlen) 2022 sbuff->buflen += sbuff->buflen; 2023 sbuff->data = erealloc(sbuff->data, sbuff->buflen); 2024 flag = 1; 2025 } 2026 2027 /* Append a space at the end of the buffer. */ 2028 if (sbuff->offset || flag) 2029 sbuff->data[sbuff->offset++] = ' '; 2030 /* Now, copy src at the end of the buffer. */ 2031 memcpy(sbuff->data + sbuff->offset, temp, srclen); 2032 sbuff->offset += srclen; 2033 free(temp); 2034 } 2035 2036 static void 2037 usage(void) 2038 { 2039 fprintf(stderr, "Usage: %s [-flo]\n", getprogname()); 2040 exit(1); 2041 } 2042
Indexes created Mon Sep 29 03:10:08 GMT 2025