1 /* $NetBSD: dict_cache.c,v 1.5 2026/05/09 18:49:22 christos Exp $ */ 2 3 /*++ 4 /* NAME 5 /* dict_cache 3 6 /* SUMMARY 7 /* External cache manager 8 /* SYNOPSIS 9 /* #include <dict_cache.h> 10 /* 11 /* DICT_CACHE *dict_cache_open(dbname, open_flags, dict_flags) 12 /* const char *dbname; 13 /* int open_flags; 14 /* int dict_flags; 15 /* 16 /* void dict_cache_close(cache) 17 /* DICT_CACHE *cache; 18 /* 19 /* const char *dict_cache_lookup(cache, cache_key) 20 /* DICT_CACHE *cache; 21 /* const char *cache_key; 22 /* 23 /* int dict_cache_update(cache, cache_key, cache_val) 24 /* DICT_CACHE *cache; 25 /* const char *cache_key; 26 /* const char *cache_val; 27 /* 28 /* int dict_cache_delete(cache, cache_key) 29 /* DICT_CACHE *cache; 30 /* const char *cache_key; 31 /* 32 /* int dict_cache_sequence(cache, first_next, cache_key, cache_val) 33 /* DICT_CACHE *cache; 34 /* int first_next; 35 /* const char **cache_key; 36 /* const char **cache_val; 37 /* 38 /* int dict_cache_error(cache) 39 /* DICT_CACHE *cache; 40 /* AUXILIARY FUNCTIONS 41 /* void dict_cache_control(cache, name, value, ...) 42 /* DICT_CACHE *cache; 43 /* int name; 44 /* 45 /* typedef int (*DICT_CACHE_VALIDATOR_FN) (const char *cache_key, 46 /* const char *cache_val, void *context); 47 /* 48 /* const char *dict_cache_name(cache) 49 /* DICT_CACHE *cache; 50 /* DESCRIPTION 51 /* This module maintains external cache files with support 52 /* for expiration. The underlying table must implement the 53 /* "lookup", "update", "delete" and "sequence" operations. 54 /* 55 /* Although this API is similar to the one documented in 56 /* dict_open(3), there are subtle differences in the interaction 57 /* between the iterators that access all cache elements, and 58 /* other operations that access individual cache elements. 59 /* 60 /* In particular, when a "sequence" or "cleanup" operation is 61 /* in progress the cache intercepts requests to delete the 62 /* "current" entry, as this would cause some databases to 63 /* mis-behave. Instead, the cache implements a "delete behind" 64 /* strategy, and deletes such an entry after the "sequence" 65 /* or "cleanup" operation moves on to the next cache element. 66 /* The "delete behind" strategy also affects the cache lookup 67 /* and update operations as detailed below. 68 /* 69 /* dict_cache_open() is a wrapper around the dict_open() 70 /* function. It opens the specified cache and returns a handle 71 /* that must be used for subsequent access. This function does 72 /* not return in case of error. 73 /* 74 /* dict_cache_close() closes the specified cache and releases 75 /* memory that was allocated by dict_cache_open(), and terminates 76 /* any thread that was started with dict_cache_control(). 77 /* 78 /* dict_cache_lookup() looks up the specified cache entry. 79 /* The result value is a null pointer when the cache entry was 80 /* not found, or when the entry is scheduled for "delete 81 /* behind". 82 /* 83 /* dict_cache_update() updates the specified cache entry. If 84 /* the entry is scheduled for "delete behind", the delete 85 /* operation is canceled (because of this, the cache must be 86 /* opened with DICT_FLAG_DUP_REPLACE). This function does not 87 /* return in case of error. 88 /* 89 /* dict_cache_delete() removes the specified cache entry. If 90 /* this is the "current" entry of a "sequence" operation, the 91 /* entry is scheduled for "delete behind". The result value 92 /* is zero when the entry was found. 93 /* 94 /* dict_cache_sequence() iterates over the specified cache and 95 /* returns each entry in an implementation-defined order. The 96 /* result value is zero when a cache entry was found. 97 /* 98 /* Important: programs must not use both dict_cache_sequence() 99 /* and the built-in cache cleanup feature. 100 /* 101 /* dict_cache_control() provides control over the built-in 102 /* cache cleanup feature and logging. The arguments are a list 103 /* of macros with zero or more arguments, terminated with 104 /* CA_DICT_CACHE_CTL_END which has none. The following lists 105 /* the macros and corresponding argument types. 106 /* .IP "CA_DICT_CACHE_CTL_FLAGS(int flags)" 107 /* The arguments to this command are the bit-wise OR of zero 108 /* or more of the following: 109 /* .RS 110 /* .IP CA_DICT_CACHE_CTL_FLAG_VERBOSE 111 /* Enable verbose logging of cache activity. 112 /* .IP CA_DICT_CACHE_CTL_FLAG_EXP_SUMMARY 113 /* Log cache statistics after each cache cleanup run. 114 /* .RE 115 /* .IP "CA_DICT_CACHE_CTL_INTERVAL(int interval)" 116 /* The interval between cache cleanup runs. Specify a null 117 /* validator or interval to stop cache cleanup and log cache 118 /* statistics if a cleanup run was in progress. 119 /* .IP "CA_DICT_CACHE_CTL_VALIDATOR(DICT_CACHE_VALIDATOR_FN validator)" 120 /* An application call-back routine that returns non-zero when 121 /* a cache entry should be kept. The call-back function should 122 /* not make changes to the cache. Specify a null validator or 123 /* interval to stop cache cleanup. 124 /* .IP "CA_DICT_CACHE_CTL_CONTEXT(void *context)" 125 /* Application context that is passed to the validator function. 126 /* .RE 127 /* .PP 128 /* dict_cache_name() returns the name of the specified cache. 129 /* 130 /* dict_cache_error() returns the error status for the underlying 131 /* dictionary. 132 /* 133 /* Arguments: 134 /* .IP "dbname, open_flags, dict_flags" 135 /* These are passed unchanged to dict_open(). The cache must 136 /* be opened with DICT_FLAG_DUP_REPLACE. 137 /* .IP cache 138 /* Cache handle created with dict_cache_open(). 139 /* .IP cache_key 140 /* Cache lookup key. 141 /* .IP cache_val 142 /* Information that is stored under a cache lookup key. 143 /* .IP first_next 144 /* One of DICT_SEQ_FUN_FIRST (first cache element) or 145 /* DICT_SEQ_FUN_NEXT (next cache element). 146 /* .sp 147 /* Note: there is no "stop" request. To ensure that the "delete 148 /* behind" strategy does not interfere with database access, 149 /* allow dict_cache_sequence() to run to completion. 150 /* .IP table 151 /* A bare dictionary handle. 152 /* DIAGNOSTICS 153 /* When a request is satisfied, the lookup routine returns 154 /* non-null, and the update, delete and sequence routines 155 /* return zero. The cache->error value is zero when a request 156 /* could not be satisfied because an item did not exist (delete, 157 /* sequence) or if it could not be updated. The cache->error 158 /* value is non-zero only when a request could not be satisfied, 159 /* and the cause was a database error. 160 /* 161 /* Cache access errors are logged with a warning message. To 162 /* avoid spamming the log, each type of operation logs no more 163 /* than one cache access error per second, per cache. Specify 164 /* the DICT_CACHE_FLAG_VERBOSE flag (see above) to log all 165 /* warnings. 166 /* BUGS 167 /* There should be a way to suspend automatic program suicide 168 /* until a cache cleanup run is completed. Some entries may 169 /* never be removed when the process max_idle time is less 170 /* than the time needed to make a full pass over the cache. 171 /* 172 /* The delete-behind strategy assumes that all updates are 173 /* made by a single process. Otherwise, delete-behind may 174 /* remove an entry that was updated after it was scheduled for 175 /* deletion. 176 /* LICENSE 177 /* .ad 178 /* .fi 179 /* The Secure Mailer license must be distributed with this software. 180 /* HISTORY 181 /* .ad 182 /* .fi 183 /* A predecessor of this code was written first for the Postfix 184 /* tlsmgr(8) daemon. 185 /* AUTHOR(S) 186 /* Wietse Venema 187 /* IBM T.J. Watson Research 188 /* P.O. Box 704 189 /* Yorktown Heights, NY 10598, USA 190 /*--*/ 191 192 /* System library. */ 193 194 #include <sys_defs.h> 195 #include <string.h> 196 #include <stdlib.h> 197 198 /* Utility library. */ 199 200 #include <msg.h> 201 #include <dict.h> 202 #include <mymalloc.h> 203 #include <events.h> 204 #include <dict_cache.h> 205 206 /* Application-specific. */ 207 208 /* 209 * XXX Deleting entries while enumerating a map can he tricky. Some map 210 * types have a concept of cursor and support a "delete the current element" 211 * operation. Some map types without cursors don't behave well when the 212 * current first/next entry is deleted (example: with Berkeley DB < 2, the 213 * "next" operation produces garbage). To avoid trouble, we delete an entry 214 * after advancing the current first/next position beyond it; we use the 215 * same strategy with application requests to delete the current entry. 216 */ 217 218 /* 219 * Opaque data structure. Use dict_cache_name() to access the name of the 220 * underlying database. 221 */ 222 struct DICT_CACHE { 223 char *name; /* full name including proxy: */ 224 int cache_flags; /* see below */ 225 int user_flags; /* logging */ 226 DICT *db; /* database handle */ 227 int error; /* last operation only */ 228 229 /* Delete-behind support. */ 230 char *saved_curr_key; /* "current" cache lookup key */ 231 char *saved_curr_val; /* "current" cache lookup result */ 232 233 /* Cleanup support. */ 234 int exp_interval; /* time between cleanup runs */ 235 DICT_CACHE_VALIDATOR_FN exp_validator; /* expiration call-back */ 236 void *exp_context; /* call-back context */ 237 int retained; /* entries retained in cleanup run */ 238 int dropped; /* entries removed in cleanup run */ 239 240 /* Rate-limited logging support. */ 241 int log_delay; 242 time_t upd_log_stamp; /* last update warning */ 243 time_t get_log_stamp; /* last lookup warning */ 244 time_t del_log_stamp; /* last delete warning */ 245 time_t seq_log_stamp; /* last sequence warning */ 246 }; 247 248 #define DC_FLAG_DEL_SAVED_CURRENT_KEY (1<<0) /* delete-behind is scheduled */ 249 250 /* 251 * Don't log cache access errors more than once per second. 252 */ 253 #define DC_DEF_LOG_DELAY 1 254 255 /* 256 * Macros to make obscure code more readable. 257 */ 258 #define DC_SCHEDULE_FOR_DELETE_BEHIND(cp) \ 259 ((cp)->cache_flags |= DC_FLAG_DEL_SAVED_CURRENT_KEY) 260 261 #define DC_MATCH_SAVED_CURRENT_KEY(cp, cache_key) \ 262 ((cp)->saved_curr_key && strcmp((cp)->saved_curr_key, (cache_key)) == 0) 263 264 #define DC_IS_SCHEDULED_FOR_DELETE_BEHIND(cp) \ 265 (/* NOT: (cp)->saved_curr_key && */ \ 266 ((cp)->cache_flags & DC_FLAG_DEL_SAVED_CURRENT_KEY) != 0) 267 268 #define DC_CANCEL_DELETE_BEHIND(cp) \ 269 ((cp)->cache_flags &= ~DC_FLAG_DEL_SAVED_CURRENT_KEY) 270 271 /* 272 * Special key to store the time of the last cache cleanup run completion. 273 */ 274 #define DC_LAST_CACHE_CLEANUP_COMPLETED "_LAST_CACHE_CLEANUP_COMPLETED_" 275 276 /* dict_cache_lookup - load entry from cache */ 277 278 const char *dict_cache_lookup(DICT_CACHE *cp, const char *cache_key) 279 { 280 const char *myname = "dict_cache_lookup"; 281 const char *cache_val; 282 DICT *db = cp->db; 283 284 /* 285 * Search for the cache entry. Don't return an entry that is scheduled 286 * for delete-behind. 287 */ 288 if (DC_IS_SCHEDULED_FOR_DELETE_BEHIND(cp) 289 && DC_MATCH_SAVED_CURRENT_KEY(cp, cache_key)) { 290 if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) 291 msg_info("%s: key=%s (pretend not found - scheduled for deletion)", 292 myname, cache_key); 293 DICT_ERR_VAL_RETURN(cp, DICT_ERR_NONE, (char *) 0); 294 } else { 295 cache_val = dict_get(db, cache_key); 296 if (cache_val == 0 && db->error != 0) 297 msg_rate_delay(&cp->get_log_stamp, cp->log_delay, msg_warn, 298 "%s: cache lookup for '%s' failed due to error", 299 cp->name, cache_key); 300 if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) 301 msg_info("%s: key=%s value=%s", myname, cache_key, 302 cache_val ? cache_val : db->error ? 303 "error" : "(not found)"); 304 DICT_ERR_VAL_RETURN(cp, db->error, cache_val); 305 } 306 } 307 308 /* dict_cache_update - save entry to cache */ 309 310 int dict_cache_update(DICT_CACHE *cp, const char *cache_key, 311 const char *cache_val) 312 { 313 const char *myname = "dict_cache_update"; 314 DICT *db = cp->db; 315 int put_res; 316 317 /* 318 * Store the cache entry and cancel the delete-behind operation. 319 */ 320 if (DC_IS_SCHEDULED_FOR_DELETE_BEHIND(cp) 321 && DC_MATCH_SAVED_CURRENT_KEY(cp, cache_key)) { 322 if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) 323 msg_info("%s: cancel delete-behind for key=%s", myname, cache_key); 324 DC_CANCEL_DELETE_BEHIND(cp); 325 } 326 if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) 327 msg_info("%s: key=%s value=%s", myname, cache_key, cache_val); 328 put_res = dict_put(db, cache_key, cache_val); 329 if (put_res != 0) 330 msg_rate_delay(&cp->upd_log_stamp, cp->log_delay, msg_warn, 331 "%s: could not update entry for %s", cp->name, cache_key); 332 DICT_ERR_VAL_RETURN(cp, db->error, put_res); 333 } 334 335 /* dict_cache_delete - delete entry from cache */ 336 337 int dict_cache_delete(DICT_CACHE *cp, const char *cache_key) 338 { 339 const char *myname = "dict_cache_delete"; 340 int del_res; 341 DICT *db = cp->db; 342 343 /* 344 * Delete the entry, unless we would delete the current first/next entry. 345 * In that case, schedule the "current" entry for delete-behind to avoid 346 * mis-behavior by some databases. 347 */ 348 if (DC_MATCH_SAVED_CURRENT_KEY(cp, cache_key)) { 349 DC_SCHEDULE_FOR_DELETE_BEHIND(cp); 350 if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) 351 msg_info("%s: key=%s (current entry - schedule for delete-behind)", 352 myname, cache_key); 353 DICT_ERR_VAL_RETURN(cp, DICT_ERR_NONE, DICT_STAT_SUCCESS); 354 } else { 355 del_res = dict_del(db, cache_key); 356 if (del_res != 0) 357 msg_rate_delay(&cp->del_log_stamp, cp->log_delay, msg_warn, 358 "%s: could not delete entry for %s", cp->name, cache_key); 359 if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) 360 msg_info("%s: key=%s (%s)", myname, cache_key, 361 del_res == 0 ? "found" : 362 db->error ? "error" : "not found"); 363 DICT_ERR_VAL_RETURN(cp, db->error, del_res); 364 } 365 } 366 367 /* dict_cache_sequence - look up the first/next cache entry */ 368 369 int dict_cache_sequence(DICT_CACHE *cp, int first_next, 370 const char **cache_key, 371 const char **cache_val) 372 { 373 const char *myname = "dict_cache_sequence"; 374 int seq_res; 375 const char *raw_cache_key; 376 const char *raw_cache_val; 377 char *previous_curr_key; 378 char *previous_curr_val; 379 DICT *db = cp->db; 380 381 /* 382 * Find the first or next database entry. Hide the record with the cache 383 * cleanup completion time stamp. 384 */ 385 seq_res = dict_seq(db, first_next, &raw_cache_key, &raw_cache_val); 386 if (seq_res == 0 387 && strcmp(raw_cache_key, DC_LAST_CACHE_CLEANUP_COMPLETED) == 0) 388 seq_res = 389 dict_seq(db, DICT_SEQ_FUN_NEXT, &raw_cache_key, &raw_cache_val); 390 if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) 391 msg_info("%s: key=%s value=%s", myname, 392 seq_res == 0 ? raw_cache_key : db->error ? 393 "(error)" : "(not found)", 394 seq_res == 0 ? raw_cache_val : db->error ? 395 "(error)" : "(not found)"); 396 if (db->error) 397 msg_rate_delay(&cp->seq_log_stamp, cp->log_delay, msg_warn, 398 "%s: sequence error", cp->name); 399 400 /* 401 * Save the current cache_key and cache_val before they are clobbered by 402 * our own delete operation below. This also prevents surprises when the 403 * application accesses the database after this function returns. 404 * 405 * We also use the saved cache_key to protect the current entry against 406 * application delete requests. 407 */ 408 previous_curr_key = cp->saved_curr_key; 409 previous_curr_val = cp->saved_curr_val; 410 if (seq_res == 0) { 411 cp->saved_curr_key = mystrdup(raw_cache_key); 412 cp->saved_curr_val = mystrdup(raw_cache_val); 413 } else { 414 cp->saved_curr_key = 0; 415 cp->saved_curr_val = 0; 416 } 417 418 /* 419 * Delete behind. 420 */ 421 if (db->error == 0 && DC_IS_SCHEDULED_FOR_DELETE_BEHIND(cp)) { 422 DC_CANCEL_DELETE_BEHIND(cp); 423 if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) 424 msg_info("%s: delete-behind key=%s value=%s", 425 myname, previous_curr_key, previous_curr_val); 426 if (dict_del(db, previous_curr_key) != 0) 427 msg_rate_delay(&cp->del_log_stamp, cp->log_delay, msg_warn, 428 "%s: could not delete entry for %s", 429 cp->name, previous_curr_key); 430 } 431 432 /* 433 * Clean up previous iteration key and value. 434 */ 435 if (previous_curr_key) 436 myfree(previous_curr_key); 437 if (previous_curr_val) 438 myfree(previous_curr_val); 439 440 /* 441 * Return the result. 442 */ 443 *cache_key = (cp)->saved_curr_key; 444 *cache_val = (cp)->saved_curr_val; 445 DICT_ERR_VAL_RETURN(cp, db->error, seq_res); 446 } 447 448 /* dict_cache_delete_behind_reset - reset "delete behind" state */ 449 450 static void dict_cache_delete_behind_reset(DICT_CACHE *cp) 451 { 452 #define FREE_AND_WIPE(s) do { if (s) { myfree(s); (s) = 0; } } while (0) 453 454 DC_CANCEL_DELETE_BEHIND(cp); 455 FREE_AND_WIPE(cp->saved_curr_key); 456 FREE_AND_WIPE(cp->saved_curr_val); 457 } 458 459 /* dict_cache_clean_stat_log_reset - log and reset cache cleanup statistics */ 460 461 static void dict_cache_clean_stat_log_reset(DICT_CACHE *cp, 462 const char *full_partial) 463 { 464 if (cp->user_flags & DICT_CACHE_FLAG_STATISTICS) 465 msg_info("cache %s %s cleanup: retained=%d dropped=%d entries", 466 cp->name, full_partial, cp->retained, cp->dropped); 467 cp->retained = cp->dropped = 0; 468 } 469 470 /* dict_cache_clean_event - examine one cache entry */ 471 472 static void dict_cache_clean_event(int unused_event, void *cache_context) 473 { 474 const char *myname = "dict_cache_clean_event"; 475 DICT_CACHE *cp = (DICT_CACHE *) cache_context; 476 const char *cache_key; 477 const char *cache_val; 478 int next_interval; 479 VSTRING *stamp_buf; 480 int first_next; 481 482 /* 483 * We interleave cache cleanup with other processing, so that the 484 * application's service remains available, with perhaps increased 485 * latency. 486 */ 487 488 /* 489 * Start a new cache cleanup run. 490 */ 491 if (cp->saved_curr_key == 0) { 492 cp->retained = cp->dropped = 0; 493 first_next = DICT_SEQ_FUN_FIRST; 494 if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) 495 msg_info("%s: start %s cache cleanup", myname, cp->name); 496 } 497 498 /* 499 * Continue a cache cleanup run in progress. 500 */ 501 else { 502 first_next = DICT_SEQ_FUN_NEXT; 503 } 504 505 /* 506 * Examine one cache entry. 507 */ 508 if (dict_cache_sequence(cp, first_next, &cache_key, &cache_val) == 0) { 509 if (cp->exp_validator(cache_key, cache_val, cp->exp_context) == 0) { 510 DC_SCHEDULE_FOR_DELETE_BEHIND(cp); 511 cp->dropped++; 512 if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) 513 msg_info("%s: drop %s cache entry for %s", 514 myname, cp->name, cache_key); 515 } else { 516 cp->retained++; 517 if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) 518 msg_info("%s: keep %s cache entry for %s", 519 myname, cp->name, cache_key); 520 } 521 next_interval = 0; 522 } 523 524 /* 525 * Cache cleanup completed. Report vital statistics. 526 */ 527 else if (cp->error != 0) { 528 msg_warn("%s: cache cleanup scan terminated due to error", cp->name); 529 dict_cache_clean_stat_log_reset(cp, "partial"); 530 next_interval = cp->exp_interval; 531 } else { 532 if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) 533 msg_info("%s: done %s cache cleanup scan", myname, cp->name); 534 dict_cache_clean_stat_log_reset(cp, "full"); 535 stamp_buf = vstring_alloc(100); 536 vstring_sprintf(stamp_buf, "%ld", (long) event_time()); 537 dict_put(cp->db, DC_LAST_CACHE_CLEANUP_COMPLETED, 538 vstring_str(stamp_buf)); 539 vstring_free(stamp_buf); 540 next_interval = cp->exp_interval; 541 } 542 event_request_timer(dict_cache_clean_event, cache_context, next_interval); 543 } 544 545 /* dict_cache_control - schedule or stop the cache cleanup thread */ 546 547 void dict_cache_control(DICT_CACHE *cp,...) 548 { 549 const char *myname = "dict_cache_control"; 550 const char *last_done; 551 time_t next_interval; 552 int cache_cleanup_is_active = (cp->exp_validator && cp->exp_interval); 553 va_list ap; 554 int name; 555 556 /* 557 * Update the control settings. 558 */ 559 va_start(ap, cp); 560 while ((name = va_arg(ap, int)) > 0) { 561 switch (name) { 562 case DICT_CACHE_CTL_END: 563 break; 564 case DICT_CACHE_CTL_FLAGS: 565 cp->user_flags = va_arg(ap, int); 566 cp->log_delay = (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) ? 567 0 : DC_DEF_LOG_DELAY; 568 break; 569 case DICT_CACHE_CTL_INTERVAL: 570 cp->exp_interval = va_arg(ap, int); 571 if (cp->exp_interval < 0) 572 msg_panic("%s: bad %s cache cleanup interval %d", 573 myname, cp->name, cp->exp_interval); 574 break; 575 case DICT_CACHE_CTL_VALIDATOR: 576 cp->exp_validator = va_arg(ap, DICT_CACHE_VALIDATOR_FN); 577 break; 578 case DICT_CACHE_CTL_CONTEXT: 579 cp->exp_context = va_arg(ap, void *); 580 break; 581 default: 582 msg_panic("%s: bad command: %d", myname, name); 583 } 584 } 585 va_end(ap); 586 587 /* 588 * Schedule the cache cleanup thread. 589 */ 590 if (cp->exp_interval && cp->exp_validator) { 591 592 /* 593 * Sanity checks. 594 */ 595 if (cache_cleanup_is_active) 596 msg_panic("%s: %s cache cleanup is already scheduled", 597 myname, cp->name); 598 599 /* 600 * The next start time depends on the last completion time. 601 */ 602 #define NEXT_START(last, delta) ((delta) + (unsigned long) atol(last)) 603 #define NOW (time((time_t *) 0)) /* NOT: event_time() */ 604 605 if ((last_done = dict_get(cp->db, DC_LAST_CACHE_CLEANUP_COMPLETED)) == 0 606 || (next_interval = (NEXT_START(last_done, cp->exp_interval) - NOW)) < 0) 607 next_interval = 0; 608 if (next_interval > cp->exp_interval) 609 next_interval = cp->exp_interval; 610 if ((cp->user_flags & DICT_CACHE_FLAG_VERBOSE) && next_interval > 0) 611 msg_info("%s cache cleanup will start after %ds", 612 cp->name, (int) next_interval); 613 event_request_timer(dict_cache_clean_event, (void *) cp, 614 (int) next_interval); 615 } 616 617 /* 618 * Cancel the cache cleanup thread. 619 */ 620 else if (cache_cleanup_is_active) { 621 if (cp->retained || cp->dropped) 622 dict_cache_clean_stat_log_reset(cp, "partial"); 623 dict_cache_delete_behind_reset(cp); 624 event_cancel_timer(dict_cache_clean_event, (void *) cp); 625 } 626 } 627 628 /* dict_cache_open - open cache file */ 629 630 DICT_CACHE *dict_cache_open(const char *dbname, int open_flags, int dict_flags) 631 { 632 DICT_CACHE *cp; 633 DICT *dict; 634 635 /* 636 * Open the database as requested. Don't attempt to second-guess the 637 * application. 638 */ 639 dict = dict_open(dbname, open_flags, dict_flags); 640 641 /* 642 * Create the DICT_CACHE object. 643 */ 644 cp = (DICT_CACHE *) mymalloc(sizeof(*cp)); 645 cp->name = mystrdup(dbname); 646 cp->cache_flags = 0; 647 cp->user_flags = 0; 648 cp->db = dict; 649 cp->saved_curr_key = 0; 650 cp->saved_curr_val = 0; 651 cp->exp_interval = 0; 652 cp->exp_validator = 0; 653 cp->exp_context = 0; 654 cp->retained = 0; 655 cp->dropped = 0; 656 cp->log_delay = DC_DEF_LOG_DELAY; 657 cp->upd_log_stamp = cp->get_log_stamp = 658 cp->del_log_stamp = cp->seq_log_stamp = 0; 659 660 return (cp); 661 } 662 663 /* dict_cache_close - close cache file */ 664 665 void dict_cache_close(DICT_CACHE *cp) 666 { 667 668 /* 669 * Cancel the cache cleanup thread. This also logs (and resets) 670 * statistics for a scan that is in progress. 671 */ 672 dict_cache_control(cp, DICT_CACHE_CTL_INTERVAL, 0, DICT_CACHE_CTL_END); 673 674 /* 675 * Destroy the DICT_CACHE object. 676 */ 677 myfree(cp->name); 678 dict_close(cp->db); 679 if (cp->saved_curr_key) 680 myfree(cp->saved_curr_key); 681 if (cp->saved_curr_val) 682 myfree(cp->saved_curr_val); 683 myfree((void *) cp); 684 } 685 686 /* dict_cache_name - get the cache name */ 687 688 const char *dict_cache_name(DICT_CACHE *cp) 689 { 690 691 /* 692 * This is used for verbose logging or warning messages, so the cost of a 693 * call is only made where needed (well sort of - code that does not 694 * execute still presents overhead for the processor pipeline, processor 695 * cache, etc). 696 */ 697 return (cp->name); 698 } 699 700 /* dict_cache_error - get the dictionary error status */ 701 702 int dict_cache_error(DICT_CACHE *cp) 703 { 704 return (cp->error); 705 } 706 707 /* 708 * Test driver with support for interleaved access. First, enter a number of 709 * requests to look up, update or delete a sequence of cache entries, then 710 * interleave those sequences with the "run" command. 711 */ 712 #ifdef TEST 713 #include <msg_vstream.h> 714 #include <vstring_vstream.h> 715 #include <argv.h> 716 #include <stringops.h> 717 718 #define DELIMS " " 719 #define USAGE "\n\tTo manage settings:" \ 720 "\n\tverbose <level> (verbosity level)" \ 721 "\n\telapsed <level> (0=don't show elapsed time)" \ 722 "\n\tlmdb_map_size <limit> (initial LMDB size limit)" \ 723 "\n\tcache <type>:<name> (switch to named database)" \ 724 "\n\tstatus (show map size, cache, pending requests)" \ 725 "\n\n\tTo manage pending requests:" \ 726 "\n\treset (discard pending requests)" \ 727 "\n\trun (execute pending requests in interleaved order)" \ 728 "\n\n\tTo add a pending request:" \ 729 "\n\tquery <key-suffix> <count> (negative to reverse order)" \ 730 "\n\tupdate <key-suffix> <count> (negative to reverse order)" \ 731 "\n\tdelete <key-suffix> <count> (negative to reverse order)" \ 732 "\n\tpurge <key-suffix>" \ 733 "\n\tcount <key-suffix>" 734 735 /* 736 * For realism, open the cache with the same flags as postscreen(8) and 737 * verify(8). 738 */ 739 #define DICT_CACHE_OPEN_FLAGS (DICT_FLAG_DUP_REPLACE | DICT_FLAG_SYNC_UPDATE | \ 740 DICT_FLAG_OPEN_LOCK) 741 742 /* 743 * Storage for one request to access a sequence of cache entries. 744 */ 745 typedef struct DICT_CACHE_SREQ { 746 int flags; /* per-request: reverse, purge */ 747 char *cmd; /* command for status report */ 748 void (*action) (struct DICT_CACHE_SREQ *, DICT_CACHE *, VSTRING *); 749 char *suffix; /* key suffix */ 750 int done; /* progress indicator */ 751 int todo; /* number of entries to process */ 752 int first_next; /* first/next */ 753 } DICT_CACHE_SREQ; 754 755 #define DICT_CACHE_SREQ_FLAG_PURGE (1<<1) /* purge instead of count */ 756 #define DICT_CACHE_SREQ_FLAG_REVERSE (1<<2) /* reverse instead of forward */ 757 758 #define DICT_CACHE_SREQ_LIMIT 10 759 760 /* 761 * All test requests combined. 762 */ 763 typedef struct DICT_CACHE_TEST { 764 int flags; /* exclusion flags */ 765 int size; /* allocated slots */ 766 int used; /* used slots */ 767 DICT_CACHE_SREQ job_list[1]; /* actually, a bunch */ 768 } DICT_CACHE_TEST; 769 770 #define DICT_CACHE_TEST_FLAG_ITER (1<<0) /* count or purge */ 771 772 #define STR(x) vstring_str(x) 773 774 int show_elapsed = 1; /* show elapsed time */ 775 776 #ifdef HAS_LMDB 777 extern size_t dict_lmdb_map_size; /* LMDB-specific */ 778 779 #endif 780 781 /* usage - command-line usage message */ 782 783 static NORETURN usage(const char *progname) 784 { 785 msg_fatal("usage: %s (no argument)", progname); 786 } 787 788 /* make_tagged_key - make tagged search key */ 789 790 static void make_tagged_key(VSTRING *bp, DICT_CACHE_SREQ *cp) 791 { 792 if (cp->done < 0) 793 msg_panic("make_tagged_key: bad done count: %d", cp->done); 794 if (cp->todo < 1) 795 msg_panic("make_tagged_key: bad todo count: %d", cp->todo); 796 vstring_sprintf(bp, "%d-%s", 797 (cp->flags & DICT_CACHE_SREQ_FLAG_REVERSE) ? 798 cp->todo - cp->done - 1 : cp->done, cp->suffix); 799 } 800 801 /* create_requests - create request list */ 802 803 static DICT_CACHE_TEST *create_requests(int count) 804 { 805 DICT_CACHE_TEST *tp; 806 DICT_CACHE_SREQ *cp; 807 808 tp = (DICT_CACHE_TEST *) mymalloc(sizeof(DICT_CACHE_TEST) + 809 (count - 1) *sizeof(DICT_CACHE_SREQ)); 810 tp->flags = 0; 811 tp->size = count; 812 tp->used = 0; 813 for (cp = tp->job_list; cp < tp->job_list + count; cp++) { 814 cp->flags = 0; 815 cp->cmd = 0; 816 cp->action = 0; 817 cp->suffix = 0; 818 cp->todo = 0; 819 cp->first_next = DICT_SEQ_FUN_FIRST; 820 } 821 return (tp); 822 } 823 824 /* reset_requests - reset request list */ 825 826 static void reset_requests(DICT_CACHE_TEST *tp) 827 { 828 DICT_CACHE_SREQ *cp; 829 830 tp->flags = 0; 831 tp->used = 0; 832 for (cp = tp->job_list; cp < tp->job_list + tp->size; cp++) { 833 cp->flags = 0; 834 if (cp->cmd) { 835 myfree(cp->cmd); 836 cp->cmd = 0; 837 } 838 cp->action = 0; 839 if (cp->suffix) { 840 myfree(cp->suffix); 841 cp->suffix = 0; 842 } 843 cp->todo = 0; 844 cp->first_next = DICT_SEQ_FUN_FIRST; 845 } 846 } 847 848 /* free_requests - destroy request list */ 849 850 static void free_requests(DICT_CACHE_TEST *tp) 851 { 852 reset_requests(tp); 853 myfree((void *) tp); 854 } 855 856 /* run_requests - execute pending requests in interleaved order */ 857 858 static void run_requests(DICT_CACHE_TEST *tp, DICT_CACHE *dp, VSTRING *bp) 859 { 860 DICT_CACHE_SREQ *cp; 861 int todo; 862 struct timeval start; 863 struct timeval finish; 864 struct timeval elapsed; 865 866 if (dp == 0) { 867 msg_warn("no cache"); 868 return; 869 } 870 GETTIMEOFDAY(&start); 871 do { 872 todo = 0; 873 for (cp = tp->job_list; cp < tp->job_list + tp->used; cp++) { 874 if (cp->done < cp->todo) { 875 todo = 1; 876 cp->action(cp, dp, bp); 877 } 878 } 879 } while (todo); 880 GETTIMEOFDAY(&finish); 881 timersub(&finish, &start, &elapsed); 882 if (show_elapsed) 883 vstream_printf("Elapsed: %g\n", 884 elapsed.tv_sec + elapsed.tv_usec / 1000000.0); 885 886 reset_requests(tp); 887 } 888 889 /* show_status - show settings and pending requests */ 890 891 static void show_status(DICT_CACHE_TEST *tp, DICT_CACHE *dp) 892 { 893 DICT_CACHE_SREQ *cp; 894 895 #ifdef HAS_LMDB 896 vstream_printf("lmdb_map_size\t%ld\n", (long) dict_lmdb_map_size); 897 #endif 898 vstream_printf("cache\t%s\n", dp ? dp->name : "(none)"); 899 900 if (tp->used == 0) 901 vstream_printf("No pending requests\n"); 902 else 903 vstream_printf("%s\t%s\t%s\t%s\t%s\t%s\n", 904 "cmd", "dir", "suffix", "count", "done", "first/next"); 905 906 for (cp = tp->job_list; cp < tp->job_list + tp->used; cp++) 907 if (cp->todo > 0) 908 vstream_printf("%s\t%s\t%s\t%d\t%d\t%d\n", 909 cp->cmd, 910 (cp->flags & DICT_CACHE_SREQ_FLAG_REVERSE) ? 911 "reverse" : "forward", 912 cp->suffix ? cp->suffix : "(null)", cp->todo, 913 cp->done, cp->first_next); 914 } 915 916 /* query_action - lookup cache entry */ 917 918 static void query_action(DICT_CACHE_SREQ *cp, DICT_CACHE *dp, VSTRING *bp) 919 { 920 const char *lookup; 921 922 make_tagged_key(bp, cp); 923 if ((lookup = dict_cache_lookup(dp, STR(bp))) == 0) { 924 if (dp->error) 925 msg_warn("query_action: query failed: %s: %m", STR(bp)); 926 else 927 msg_warn("query_action: query failed: %s", STR(bp)); 928 } else if (strcmp(STR(bp), lookup) != 0) { 929 msg_warn("lookup result \"%s\" differs from key \"%s\"", 930 lookup, STR(bp)); 931 } 932 cp->done += 1; 933 } 934 935 /* update_action - update cache entry */ 936 937 static void update_action(DICT_CACHE_SREQ *cp, DICT_CACHE *dp, VSTRING *bp) 938 { 939 make_tagged_key(bp, cp); 940 if (dict_cache_update(dp, STR(bp), STR(bp)) != 0) { 941 if (dp->error) 942 msg_warn("update_action: update failed: %s: %m", STR(bp)); 943 else 944 msg_warn("update_action: update failed: %s", STR(bp)); 945 } 946 cp->done += 1; 947 } 948 949 /* delete_action - delete cache entry */ 950 951 static void delete_action(DICT_CACHE_SREQ *cp, DICT_CACHE *dp, VSTRING *bp) 952 { 953 make_tagged_key(bp, cp); 954 if (dict_cache_delete(dp, STR(bp)) != 0) { 955 if (dp->error) 956 msg_warn("delete_action: delete failed: %s: %m", STR(bp)); 957 else 958 msg_warn("delete_action: delete failed: %s", STR(bp)); 959 } 960 cp->done += 1; 961 } 962 963 /* iter_action - iterate over cache and act on entries with given suffix */ 964 965 static void iter_action(DICT_CACHE_SREQ *cp, DICT_CACHE *dp, VSTRING *bp) 966 { 967 const char *cache_key; 968 const char *cache_val; 969 const char *what; 970 const char *suffix; 971 972 if (dict_cache_sequence(dp, cp->first_next, &cache_key, &cache_val) == 0) { 973 if (strcmp(cache_key, cache_val) != 0) 974 msg_warn("value \"%s\" differs from key \"%s\"", 975 cache_val, cache_key); 976 suffix = cache_key + strspn(cache_key, "0123456789"); 977 if (suffix[0] == '-' && strcmp(suffix + 1, cp->suffix) == 0) { 978 cp->done += 1; 979 cp->todo = cp->done + 1; /* XXX */ 980 if ((cp->flags & DICT_CACHE_SREQ_FLAG_PURGE) 981 && dict_cache_delete(dp, cache_key) != 0) { 982 if (dp->error) 983 msg_warn("purge_action: delete failed: %s: %m", STR(bp)); 984 else 985 msg_warn("purge_action: delete failed: %s", STR(bp)); 986 } 987 } 988 cp->first_next = DICT_SEQ_FUN_NEXT; 989 } else { 990 what = (cp->flags & DICT_CACHE_SREQ_FLAG_PURGE) ? "purge" : "count"; 991 if (dp->error) 992 msg_warn("%s error after %d: %m", what, cp->done); 993 else 994 vstream_printf("suffix=%s %s=%d\n", cp->suffix, what, cp->done); 995 cp->todo = 0; 996 } 997 } 998 999 /* 1000 * Table-driven support. 1001 */ 1002 typedef struct DICT_CACHE_SREQ_INFO { 1003 const char *name; 1004 int argc; 1005 void (*action) (DICT_CACHE_SREQ *, DICT_CACHE *, VSTRING *); 1006 int test_flags; 1007 int req_flags; 1008 } DICT_CACHE_SREQ_INFO; 1009 1010 static DICT_CACHE_SREQ_INFO req_info[] = { 1011 {"query", 3, query_action}, 1012 {"update", 3, update_action}, 1013 {"delete", 3, delete_action}, 1014 {"count", 2, iter_action, DICT_CACHE_TEST_FLAG_ITER}, 1015 {"purge", 2, iter_action, DICT_CACHE_TEST_FLAG_ITER, DICT_CACHE_SREQ_FLAG_PURGE}, 1016 0, 1017 }; 1018 1019 /* add_request - add a request to the list */ 1020 1021 static void add_request(DICT_CACHE_TEST *tp, ARGV *argv) 1022 { 1023 DICT_CACHE_SREQ_INFO *rp; 1024 DICT_CACHE_SREQ *cp; 1025 int req_flags; 1026 int count; 1027 char *cmd = argv->argv[0]; 1028 char *suffix = (argv->argc > 1 ? argv->argv[1] : 0); 1029 char *todo = (argv->argc > 2 ? argv->argv[2] : "1"); /* XXX */ 1030 1031 if (tp->used >= tp->size) { 1032 msg_warn("%s: request list is full", cmd); 1033 return; 1034 } 1035 for (rp = req_info; /* See below */ ; rp++) { 1036 if (rp->name == 0) { 1037 vstream_printf("usage: %s\n", USAGE); 1038 return; 1039 } 1040 if (strcmp(rp->name, argv->argv[0]) == 0 1041 && rp->argc == argv->argc) 1042 break; 1043 } 1044 req_flags = rp->req_flags; 1045 if (todo[0] == '-') { 1046 req_flags |= DICT_CACHE_SREQ_FLAG_REVERSE; 1047 todo += 1; 1048 } 1049 if (!alldig(todo) || (count = atoi(todo)) == 0) { 1050 msg_warn("%s: bad count: %s", cmd, todo); 1051 return; 1052 } 1053 if (tp->flags & rp->test_flags) { 1054 msg_warn("%s: command conflicts with other command", cmd); 1055 return; 1056 } 1057 tp->flags |= rp->test_flags; 1058 cp = tp->job_list + tp->used; 1059 cp->cmd = mystrdup(cmd); 1060 cp->action = rp->action; 1061 if (suffix) 1062 cp->suffix = mystrdup(suffix); 1063 cp->done = 0; 1064 cp->flags = req_flags; 1065 cp->todo = count; 1066 tp->used += 1; 1067 } 1068 1069 /* main - main program */ 1070 1071 int main(int argc, char **argv) 1072 { 1073 DICT_CACHE_TEST *test_job; 1074 VSTRING *inbuf = vstring_alloc(100); 1075 char *bufp; 1076 ARGV *args; 1077 DICT_CACHE *cache = 0; 1078 int stdin_is_tty; 1079 1080 msg_vstream_init(argv[0], VSTREAM_ERR); 1081 if (argc != 1) 1082 usage(argv[0]); 1083 1084 1085 test_job = create_requests(DICT_CACHE_SREQ_LIMIT); 1086 1087 stdin_is_tty = isatty(0); 1088 1089 for (;;) { 1090 if (stdin_is_tty) { 1091 vstream_printf("> "); 1092 vstream_fflush(VSTREAM_OUT); 1093 } 1094 if (vstring_fgets_nonl(inbuf, VSTREAM_IN) == 0) 1095 break; 1096 bufp = vstring_str(inbuf); 1097 if (!stdin_is_tty) { 1098 vstream_printf("> %s\n", bufp); 1099 vstream_fflush(VSTREAM_OUT); 1100 } 1101 if (*bufp == '#') 1102 continue; 1103 args = argv_split(bufp, DELIMS); 1104 if (argc == 0) { 1105 vstream_printf("usage: %s\n", USAGE); 1106 vstream_fflush(VSTREAM_OUT); 1107 continue; 1108 } 1109 if (strcmp(args->argv[0], "verbose") == 0 && args->argc == 2) { 1110 msg_verbose = atoi(args->argv[1]); 1111 } else if (strcmp(args->argv[0], "elapsed") == 0 && args->argc == 2) { 1112 show_elapsed = atoi(args->argv[1]); 1113 #ifdef HAS_LMDB 1114 } else if (strcmp(args->argv[0], "lmdb_map_size") == 0 && args->argc == 2) { 1115 dict_lmdb_map_size = atol(args->argv[1]); 1116 #endif 1117 } else if (strcmp(args->argv[0], "cache") == 0 && args->argc == 2) { 1118 if (cache) 1119 dict_cache_close(cache); 1120 cache = dict_cache_open(args->argv[1], O_CREAT | O_RDWR, 1121 DICT_CACHE_OPEN_FLAGS); 1122 } else if (strcmp(args->argv[0], "reset") == 0 && args->argc == 1) { 1123 reset_requests(test_job); 1124 } else if (strcmp(args->argv[0], "run") == 0 && args->argc == 1) { 1125 run_requests(test_job, cache, inbuf); 1126 } else if (strcmp(args->argv[0], "status") == 0 && args->argc == 1) { 1127 show_status(test_job, cache); 1128 } else { 1129 add_request(test_job, args); 1130 } 1131 vstream_fflush(VSTREAM_OUT); 1132 argv_free(args); 1133 } 1134 1135 vstring_free(inbuf); 1136 free_requests(test_job); 1137 if (cache) 1138 dict_cache_close(cache); 1139 return (0); 1140 } 1141 1142 #endif 1143
Indexes created Wed Jun 24 00:24:50 UTC 2026