Home | History | Annotate | Line # | Download | only in lloadd
      1 /*	$NetBSD: backend.c,v 1.3 2025/09/05 21:16:24 christos Exp $	*/
      2 
      3 /* $OpenLDAP$ */
      4 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
      5  *
      6  * Copyright 1998-2024 The OpenLDAP Foundation.
      7  * All rights reserved.
      8  *
      9  * Redistribution and use in source and binary forms, with or without
     10  * modification, are permitted only as authorized by the OpenLDAP
     11  * Public License.
     12  *
     13  * A copy of this license is available in the file LICENSE in the
     14  * top-level directory of the distribution or, alternatively, at
     15  * <http://www.OpenLDAP.org/license.html>.
     16  */
     17 
     18 #include <sys/cdefs.h>
     19 __RCSID("$NetBSD: backend.c,v 1.3 2025/09/05 21:16:24 christos Exp $");
     20 
     21 #include "portable.h"
     22 
     23 #include <ac/socket.h>
     24 #include <ac/errno.h>
     25 #include <ac/string.h>
     26 #include <ac/time.h>
     27 #include <ac/unistd.h>
     28 
     29 #include <event2/event.h>
     30 #include <event2/dns.h>
     31 
     32 #include "lutil.h"
     33 #include "lload.h"
     34 
     35 static void
     36 upstream_connect_cb( evutil_socket_t s, short what, void *arg )
     37 {
     38     LloadPendingConnection *conn = arg;
     39     LloadBackend *b = conn->backend;
     40     int error = 0, rc = -1;
     41     epoch_t epoch;
     42 
     43     checked_lock( &b->b_mutex );
     44     Debug( LDAP_DEBUG_CONNS, "upstream_connect_cb: "
     45             "fd=%d connection callback for backend uri='%s'\n",
     46             s, b->b_uri.bv_val );
     47 
     48     if ( s != conn->fd ) {
     49         /* backend_reset has been here first */
     50         goto preempted;
     51     }
     52 
     53     epoch = epoch_join();
     54 
     55     if ( what == EV_WRITE ) {
     56         socklen_t optlen = sizeof(error);
     57 
     58         if ( getsockopt( conn->fd, SOL_SOCKET, SO_ERROR, (void *)&error,
     59                      &optlen ) < 0 ) {
     60             goto done;
     61         }
     62         if ( error == EINTR || error == EINPROGRESS || error == EWOULDBLOCK ) {
     63             checked_unlock( &b->b_mutex );
     64             epoch_leave( epoch );
     65             return;
     66         } else if ( error ) {
     67             goto done;
     68         } else if ( upstream_init( s, conn->backend ) == NULL ) {
     69             goto done;
     70         }
     71         rc = LDAP_SUCCESS;
     72     }
     73 
     74 done:
     75     epoch_leave( epoch );
     76 
     77     LDAP_LIST_REMOVE( conn, next );
     78     if ( rc ) {
     79         evutil_closesocket( conn->fd );
     80         b->b_opening--;
     81         b->b_failed++;
     82         if ( what & EV_TIMEOUT ) {
     83             Debug( LDAP_DEBUG_ANY, "upstream_connect_cb: "
     84                     "fd=%d connection timed out\n",
     85                     s );
     86         } else {
     87             char ebuf[128];
     88             Debug( LDAP_DEBUG_ANY, "upstream_connect_cb: "
     89                     "fd=%d connection set up failed%s%s\n",
     90                     s, error ? ": " : "",
     91                     error ? sock_errstr( error, ebuf, sizeof(ebuf) ) : "" );
     92         }
     93         backend_retry( b );
     94     }
     95 preempted:
     96     checked_unlock( &b->b_mutex );
     97 
     98     event_free( conn->event );
     99     ch_free( conn );
    100 }
    101 
    102 static void
    103 upstream_name_cb( int result, struct evutil_addrinfo *res, void *arg )
    104 {
    105     LloadBackend *b = arg;
    106     ber_socket_t s = AC_SOCKET_INVALID;
    107     epoch_t epoch;
    108     int rc;
    109 
    110     if ( result == EVUTIL_EAI_CANCEL ) {
    111         Debug( LDAP_DEBUG_ANY, "upstream_name_cb: "
    112                 "cancelled\n" );
    113         return;
    114     }
    115 
    116     checked_lock( &b->b_mutex );
    117     /* We were already running when backend_reset tried to cancel us, but were
    118      * already stuck waiting for the mutex, nothing to do and b_opening has
    119      * been decremented as well */
    120     if ( b->b_dns_req == NULL ) {
    121         checked_unlock( &b->b_mutex );
    122         return;
    123     }
    124     b->b_dns_req = NULL;
    125 
    126     epoch = epoch_join();
    127     if ( result || !res ) {
    128         Debug( LDAP_DEBUG_ANY, "upstream_name_cb: "
    129                 "name resolution failed for backend '%s': %s\n",
    130                 b->b_uri.bv_val, evutil_gai_strerror( result ) );
    131         goto fail;
    132     }
    133 
    134     /* TODO: if we get failures, try the other addrinfos */
    135     if ( (s = socket( res->ai_family, SOCK_STREAM, 0 )) ==
    136             AC_SOCKET_INVALID ) {
    137         goto fail;
    138     }
    139 
    140     if ( ber_pvt_socket_set_nonblock( s, 1 ) ) {
    141         goto fail;
    142     }
    143 
    144 #if defined(SO_KEEPALIVE) || defined(TCP_NODELAY)
    145     if ( b->b_proto == LDAP_PROTO_TCP ) {
    146         int dummy = 1;
    147 #ifdef SO_KEEPALIVE
    148         if ( setsockopt( s, SOL_SOCKET, SO_KEEPALIVE, (char *)&dummy,
    149                      sizeof(dummy) ) == AC_SOCKET_ERROR ) {
    150             Debug( LDAP_DEBUG_TRACE, "upstream_name_cb: "
    151                     "setsockopt(%d, SO_KEEPALIVE) failed (ignored).\n",
    152                     s );
    153         }
    154         if ( bindconf.sb_keepalive.sk_idle > 0 ) {
    155 #ifdef TCP_KEEPIDLE
    156             if ( setsockopt( s, IPPROTO_TCP, TCP_KEEPIDLE,
    157                          (void *)&bindconf.sb_keepalive.sk_idle,
    158                          sizeof(bindconf.sb_keepalive.sk_idle) ) ==
    159                     AC_SOCKET_ERROR ) {
    160                 Debug( LDAP_DEBUG_TRACE, "upstream_name_cb: "
    161                         "setsockopt(%d, TCP_KEEPIDLE) failed (ignored).\n",
    162                         s );
    163             }
    164 #else
    165             Debug( LDAP_DEBUG_TRACE, "upstream_name_cb: "
    166                     "sockopt TCP_KEEPIDLE not supported on this system.\n" );
    167 #endif /* TCP_KEEPIDLE */
    168         }
    169         if ( bindconf.sb_keepalive.sk_probes > 0 ) {
    170 #ifdef TCP_KEEPCNT
    171             if ( setsockopt( s, IPPROTO_TCP, TCP_KEEPCNT,
    172                          (void *)&bindconf.sb_keepalive.sk_probes,
    173                          sizeof(bindconf.sb_keepalive.sk_probes) ) ==
    174                     AC_SOCKET_ERROR ) {
    175                 Debug( LDAP_DEBUG_TRACE, "upstream_name_cb: "
    176                         "setsockopt(%d, TCP_KEEPCNT) failed (ignored).\n",
    177                         s );
    178             }
    179 #else
    180             Debug( LDAP_DEBUG_TRACE, "upstream_name_cb: "
    181                     "sockopt TCP_KEEPCNT not supported on this system.\n" );
    182 #endif /* TCP_KEEPCNT */
    183         }
    184         if ( bindconf.sb_keepalive.sk_interval > 0 ) {
    185 #ifdef TCP_KEEPINTVL
    186             if ( setsockopt( s, IPPROTO_TCP, TCP_KEEPINTVL,
    187                          (void *)&bindconf.sb_keepalive.sk_interval,
    188                          sizeof(bindconf.sb_keepalive.sk_interval) ) ==
    189                     AC_SOCKET_ERROR ) {
    190                 Debug( LDAP_DEBUG_TRACE, "upstream_name_cb: "
    191                         "setsockopt(%d, TCP_KEEPINTVL) failed (ignored).\n",
    192                         s );
    193             }
    194 #else
    195             Debug( LDAP_DEBUG_TRACE, "upstream_name_cb: "
    196                     "sockopt TCP_KEEPINTVL not supported on this system.\n" );
    197 #endif /* TCP_KEEPINTVL */
    198         }
    199 #endif /* SO_KEEPALIVE */
    200         if ( bindconf.sb_tcp_user_timeout > 0 ) {
    201 #ifdef TCP_USER_TIMEOUT
    202             if ( setsockopt( s, IPPROTO_TCP, TCP_USER_TIMEOUT,
    203                          (void *)&bindconf.sb_tcp_user_timeout,
    204                          sizeof(bindconf.sb_tcp_user_timeout) ) ==
    205                     AC_SOCKET_ERROR ) {
    206                 Debug( LDAP_DEBUG_TRACE, "upstream_name_cb: "
    207                         "setsockopt(%d, TCP_USER_TIMEOUT) failed (ignored).\n",
    208                         s );
    209             }
    210 #else
    211             Debug( LDAP_DEBUG_TRACE, "upstream_name_cb: "
    212                     "sockopt TCP_USER_TIMEOUT not supported on this "
    213                     "system.\n" );
    214 #endif /* TCP_USER_TIMEOUT */
    215         }
    216 #ifdef TCP_NODELAY
    217         if ( setsockopt( s, IPPROTO_TCP, TCP_NODELAY, (char *)&dummy,
    218                      sizeof(dummy) ) == AC_SOCKET_ERROR ) {
    219             Debug( LDAP_DEBUG_TRACE, "upstream_name_cb: "
    220                     "setsockopt(%d, TCP_NODELAY) failed (ignored).\n",
    221                     s );
    222         }
    223 #endif /* TCP_NODELAY */
    224     }
    225 #endif /* SO_KEEPALIVE || TCP_NODELAY */
    226 
    227     if ( res->ai_family == PF_INET ) {
    228         struct sockaddr_in *ai = (struct sockaddr_in *)res->ai_addr;
    229         ai->sin_port = htons( b->b_port );
    230         rc = connect( s, (struct sockaddr *)ai, res->ai_addrlen );
    231     } else {
    232         struct sockaddr_in6 *ai = (struct sockaddr_in6 *)res->ai_addr;
    233         ai->sin6_port = htons( b->b_port );
    234         rc = connect( s, (struct sockaddr *)ai, res->ai_addrlen );
    235     }
    236     /* Asynchronous connect */
    237     if ( rc ) {
    238         LloadPendingConnection *conn;
    239 
    240         if ( errno != EINPROGRESS && errno != EWOULDBLOCK ) {
    241             Debug( LDAP_DEBUG_ANY, "upstream_name_cb: "
    242                     "failed to connect to server '%s'\n",
    243                     b->b_uri.bv_val );
    244             evutil_closesocket( s );
    245             goto fail;
    246         }
    247 
    248         conn = ch_calloc( 1, sizeof(LloadPendingConnection) );
    249         LDAP_LIST_ENTRY_INIT( conn, next );
    250         conn->backend = b;
    251         conn->fd = s;
    252 
    253         conn->event = event_new( lload_get_base( s ), s, EV_WRITE|EV_PERSIST,
    254                 upstream_connect_cb, conn );
    255         if ( !conn->event ) {
    256             Debug( LDAP_DEBUG_ANY, "upstream_name_cb: "
    257                     "failed to acquire an event to finish upstream "
    258                     "connection setup.\n" );
    259             ch_free( conn );
    260             evutil_closesocket( s );
    261             goto fail;
    262         }
    263 
    264         event_add( conn->event, lload_timeout_net );
    265         LDAP_LIST_INSERT_HEAD( &b->b_connecting, conn, next );
    266         Debug( LDAP_DEBUG_CONNS, "upstream_name_cb: "
    267                 "connection to backend uri=%s in progress\n",
    268                 b->b_uri.bv_val );
    269     } else if ( upstream_init( s, b ) == NULL ) {
    270         goto fail;
    271     }
    272 
    273     checked_unlock( &b->b_mutex );
    274     evutil_freeaddrinfo( res );
    275     epoch_leave( epoch );
    276     return;
    277 
    278 fail:
    279     if ( s != AC_SOCKET_INVALID ) {
    280         evutil_closesocket( s );
    281     }
    282     b->b_opening--;
    283     b->b_failed++;
    284     backend_retry( b );
    285     checked_unlock( &b->b_mutex );
    286     if ( res ) {
    287         evutil_freeaddrinfo( res );
    288     }
    289     epoch_leave( epoch );
    290 }
    291 
    292 int
    293 try_upstream(
    294     LloadBackend *b,
    295     lload_c_head *head,
    296     LloadOperation *op,
    297     LloadConnection *c,
    298     int *res,
    299     char **message )
    300 {
    301     assert_locked( &b->b_mutex );
    302 
    303     checked_lock( &c->c_io_mutex );
    304     CONNECTION_LOCK(c);
    305     if ( c->c_state == LLOAD_C_READY && !c->c_pendingber &&
    306             ( b->b_max_conn_pending == 0 ||
    307                     c->c_n_ops_executing < b->b_max_conn_pending ) ) {
    308         Debug( LDAP_DEBUG_CONNS, "try_upstream: "
    309                 "selected connection connid=%lu for client "
    310                 "connid=%lu msgid=%d\n",
    311                 c->c_connid, op->o_client_connid, op->o_client_msgid );
    312 
    313         /* c_state is DYING if we're about to be unlinked */
    314         assert( IS_ALIVE( c, c_live ) );
    315 
    316         if ( head ) {
    317             /*
    318              * Round-robin step:
    319              * Rotate the queue to put this connection at the end.
    320              */
    321             LDAP_CIRCLEQ_MAKE_TAIL( head, c, c_next );
    322         }
    323 
    324         b->b_n_ops_executing++;
    325         if ( op->o_tag == LDAP_REQ_BIND ) {
    326             b->b_counters[LLOAD_STATS_OPS_BIND].lc_ops_received++;
    327         } else {
    328             b->b_counters[LLOAD_STATS_OPS_OTHER].lc_ops_received++;
    329         }
    330         c->c_n_ops_executing++;
    331         c->c_counters.lc_ops_received++;
    332 
    333         *res = LDAP_SUCCESS;
    334         CONNECTION_ASSERT_LOCKED(c);
    335         assert_locked( &c->c_io_mutex );
    336         return 1;
    337     }
    338     CONNECTION_UNLOCK(c);
    339     checked_unlock( &c->c_io_mutex );
    340     return 0;
    341 }
    342 
    343 int
    344 backend_select(
    345         LloadBackend *b,
    346         LloadOperation *op,
    347         LloadConnection **cp,
    348         int *res,
    349         char **message )
    350 {
    351     lload_c_head *head;
    352     LloadConnection *c;
    353 
    354     assert_locked( &b->b_mutex );
    355     if ( b->b_max_pending && b->b_n_ops_executing >= b->b_max_pending ) {
    356         Debug( LDAP_DEBUG_CONNS, "backend_select: "
    357                 "backend %s too busy\n",
    358                 b->b_uri.bv_val );
    359         *res = LDAP_BUSY;
    360         *message = "server busy";
    361         return 1;
    362     }
    363 
    364     if ( op->o_tag == LDAP_REQ_BIND
    365 #ifdef LDAP_API_FEATURE_VERIFY_CREDENTIALS
    366             && !(lload_features & LLOAD_FEATURE_VC)
    367 #endif /* LDAP_API_FEATURE_VERIFY_CREDENTIALS */
    368             ) {
    369         head = &b->b_bindconns;
    370     } else {
    371         head = &b->b_conns;
    372     }
    373 
    374     if ( LDAP_CIRCLEQ_EMPTY( head ) ) {
    375         return 0;
    376     }
    377 
    378     *res = LDAP_BUSY;
    379     *message = "server busy";
    380 
    381     LDAP_CIRCLEQ_FOREACH( c, head, c_next ) {
    382         if ( try_upstream( b, head, op, c, res, message ) ) {
    383             *cp = c;
    384             CONNECTION_ASSERT_LOCKED(c);
    385             assert_locked( &c->c_io_mutex );
    386             return 1;
    387         }
    388     }
    389 
    390     return 1;
    391 }
    392 
    393 int
    394 upstream_select(
    395         LloadOperation *op,
    396         LloadConnection **cp,
    397         int *res,
    398         char **message )
    399 {
    400     LloadTier *tier;
    401     int finished = 0;
    402 
    403     LDAP_STAILQ_FOREACH( tier, &tiers, t_next ) {
    404         if ( (finished = tier->t_type.tier_select(
    405                        tier, op, cp, res, message )) ) {
    406             break;
    407         }
    408     }
    409 
    410     return finished;
    411 }
    412 
    413 /*
    414  * Will schedule a connection attempt if there is a need for it. Need exclusive
    415  * access to backend, its b_mutex is not touched here, though.
    416  */
    417 void
    418 backend_retry( LloadBackend *b )
    419 {
    420     int requested;
    421 
    422     if ( slapd_shutdown ) {
    423         Debug( LDAP_DEBUG_CONNS, "backend_retry: "
    424                 "shutting down\n" );
    425         return;
    426     }
    427     assert_locked( &b->b_mutex );
    428 
    429     requested = b->b_numconns;
    430 #ifdef LDAP_API_FEATURE_VERIFY_CREDENTIALS
    431     if ( !(lload_features & LLOAD_FEATURE_VC) )
    432 #endif /* LDAP_API_FEATURE_VERIFY_CREDENTIALS */
    433     {
    434         requested += b->b_numbindconns;
    435     }
    436 
    437     if ( b->b_active + b->b_bindavail + b->b_opening >= requested ) {
    438         Debug( LDAP_DEBUG_CONNS, "backend_retry: "
    439                 "no more connections needed for this backend\n" );
    440         assert_locked( &b->b_mutex );
    441         return;
    442     }
    443 
    444     if ( b->b_opening > 0 ) {
    445         Debug( LDAP_DEBUG_CONNS, "backend_retry: "
    446                 "retry in progress already\n" );
    447         assert( b->b_opening == 1 );
    448         assert_locked( &b->b_mutex );
    449         return;
    450     }
    451 
    452     /* We incremented b_opening when we activated the event, so it can't be
    453      * pending */
    454     assert( !event_pending( b->b_retry_event, EV_TIMEOUT, NULL ) );
    455     b->b_opening++;
    456 
    457     if ( b->b_failed > 0 ) {
    458         Debug( LDAP_DEBUG_CONNS, "backend_retry: "
    459                 "scheduling a retry in %d ms\n",
    460                 b->b_retry_timeout );
    461         event_add( b->b_retry_event, &b->b_retry_tv );
    462         assert_locked( &b->b_mutex );
    463         return;
    464     }
    465 
    466     Debug( LDAP_DEBUG_CONNS, "backend_retry: "
    467             "scheduling re-connection straight away\n" );
    468 
    469     if ( ldap_pvt_thread_pool_submit2(
    470                  &connection_pool, backend_connect_task, b, &b->b_cookie ) ) {
    471         Debug( LDAP_DEBUG_ANY, "backend_retry: "
    472                 "failed to submit retry task, scheduling a retry instead\n" );
    473         /* The current implementation of ldap_pvt_thread_pool_submit2 can fail
    474          * and still set (an invalid) cookie */
    475         b->b_cookie = NULL;
    476         b->b_failed++;
    477         event_add( b->b_retry_event, &b->b_retry_tv );
    478     }
    479     assert_locked( &b->b_mutex );
    480 }
    481 
    482 void
    483 backend_connect( evutil_socket_t s, short what, void *arg )
    484 {
    485     struct evutil_addrinfo hints = {};
    486     LloadBackend *b = arg;
    487     struct evdns_getaddrinfo_request *request, *placeholder;
    488     char *hostname;
    489     epoch_t epoch;
    490 
    491     checked_lock( &b->b_mutex );
    492     assert( b->b_dns_req == NULL );
    493 
    494     if ( b->b_cookie ) {
    495         b->b_cookie = NULL;
    496     }
    497 
    498     if ( slapd_shutdown ) {
    499         Debug( LDAP_DEBUG_CONNS, "backend_connect: "
    500                 "doing nothing, shutdown in progress\n" );
    501         b->b_opening--;
    502         checked_unlock( &b->b_mutex );
    503         return;
    504     }
    505 
    506     epoch = epoch_join();
    507 
    508     Debug( LDAP_DEBUG_CONNS, "backend_connect: "
    509             "%sattempting connection to %s\n",
    510             (what & EV_TIMEOUT) ? "retry timeout finished, " : "",
    511             b->b_host );
    512 
    513 #ifdef LDAP_PF_LOCAL
    514     if ( b->b_proto == LDAP_PROTO_IPC ) {
    515         struct sockaddr_un addr;
    516         ber_socket_t s = socket( PF_LOCAL, SOCK_STREAM, 0 );
    517         int rc;
    518 
    519         if ( s == AC_SOCKET_INVALID ) {
    520             goto fail;
    521         }
    522 
    523         rc = ber_pvt_socket_set_nonblock( s, 1 );
    524         if ( rc ) {
    525             evutil_closesocket( s );
    526             goto fail;
    527         }
    528 
    529         if ( strlen( b->b_host ) > ( sizeof(addr.sun_path) - 1 ) ) {
    530             evutil_closesocket( s );
    531             goto fail;
    532         }
    533         memset( &addr, '\0', sizeof(addr) );
    534         addr.sun_family = AF_LOCAL;
    535         strcpy( addr.sun_path, b->b_host );
    536 
    537         rc = connect(
    538                 s, (struct sockaddr *)&addr, sizeof(struct sockaddr_un) );
    539         /* Asynchronous connect */
    540         if ( rc ) {
    541             LloadPendingConnection *conn;
    542 
    543             if ( errno != EINPROGRESS && errno != EWOULDBLOCK ) {
    544                 evutil_closesocket( s );
    545                 goto fail;
    546             }
    547 
    548             conn = ch_calloc( 1, sizeof(LloadPendingConnection) );
    549             LDAP_LIST_ENTRY_INIT( conn, next );
    550             conn->backend = b;
    551             conn->fd = s;
    552 
    553             conn->event = event_new( lload_get_base( s ), s,
    554                     EV_WRITE|EV_PERSIST, upstream_connect_cb, conn );
    555             if ( !conn->event ) {
    556                 Debug( LDAP_DEBUG_ANY, "backend_connect: "
    557                         "failed to acquire an event to finish upstream "
    558                         "connection setup.\n" );
    559                 ch_free( conn );
    560                 evutil_closesocket( s );
    561                 goto fail;
    562             }
    563 
    564             event_add( conn->event, lload_timeout_net );
    565             LDAP_LIST_INSERT_HEAD( &b->b_connecting, conn, next );
    566             Debug( LDAP_DEBUG_CONNS, "backend_connect: "
    567                     "connection to backend uri=%s in progress\n",
    568                     b->b_uri.bv_val );
    569         } else if ( upstream_init( s, b ) == NULL ) {
    570             goto fail;
    571         }
    572 
    573         checked_unlock( &b->b_mutex );
    574         epoch_leave( epoch );
    575         return;
    576     }
    577 #endif /* LDAP_PF_LOCAL */
    578 
    579     hints.ai_family = AF_UNSPEC;
    580     hints.ai_flags = EVUTIL_AI_CANONNAME;
    581     hints.ai_socktype = SOCK_STREAM;
    582     hints.ai_protocol = IPPROTO_TCP;
    583 
    584     hostname = b->b_host;
    585 
    586     /*
    587      * Picking any value on the stack. This is unique to our thread without
    588      * having to call ldap_pvt_thread_self.
    589      * We might have to revert to using ldap_pvt_thread_self eventually since
    590      * this betrays where exactly our stack lies - potentially weakening some
    591      * protections like ASLR.
    592      */
    593     placeholder = (struct evdns_getaddrinfo_request *)&request;
    594     b->b_dns_req = placeholder;
    595     checked_unlock( &b->b_mutex );
    596 
    597     request = evdns_getaddrinfo(
    598             dnsbase, hostname, NULL, &hints, upstream_name_cb, b );
    599 
    600     checked_lock( &b->b_mutex );
    601     assert( request || b->b_dns_req != placeholder );
    602 
    603     /* Record the request, unless upstream_name_cb or another thread
    604      * cleared it. Another thread is usually backend_reset or backend_connect
    605      * if upstream_name_cb finished and scheduled another one */
    606     if ( b->b_dns_req == placeholder ) {
    607         b->b_dns_req = request;
    608     }
    609     checked_unlock( &b->b_mutex );
    610     epoch_leave( epoch );
    611     return;
    612 
    613 fail:
    614     b->b_opening--;
    615     b->b_failed++;
    616     backend_retry( b );
    617     checked_unlock( &b->b_mutex );
    618     epoch_leave( epoch );
    619 }
    620 
    621 void *
    622 backend_connect_task( void *ctx, void *arg )
    623 {
    624     backend_connect( -1, 0, arg );
    625     return NULL;
    626 }
    627 
    628 /*
    629  * Needs exclusive access to the backend and no other thread is allowed to call
    630  * backend_retry while we're handling this.
    631  *
    632  * If gentle == 0, a full pause must be in effect, else we risk deadlocking on
    633  * event_free().
    634  */
    635 void
    636 backend_reset( LloadBackend *b, int gentle )
    637 {
    638     assert_locked( &b->b_mutex );
    639     if ( b->b_cookie ) {
    640         if ( ldap_pvt_thread_pool_retract( b->b_cookie ) ) {
    641             b->b_cookie = NULL;
    642             b->b_opening--;
    643         } else {
    644             /*
    645              * The task might not be cancelable because it just started
    646              * executing.
    647              *
    648              * Shutdown should be the only time when the thread pool is
    649              * in that state. Keep the cookie in to keep an eye on whether
    650              * it's finished yet.
    651              */
    652             assert( slapd_shutdown );
    653         }
    654     }
    655     /* Not safe to hold our mutex and call event_del/free if the event's
    656      * callback is running, relinquish the mutex while we do so. */
    657     if ( b->b_retry_event &&
    658             event_pending( b->b_retry_event, EV_TIMEOUT, NULL ) ) {
    659         assert( b->b_failed );
    660         checked_unlock( &b->b_mutex );
    661         event_del( b->b_retry_event );
    662         checked_lock( &b->b_mutex );
    663         b->b_opening--;
    664     }
    665     if ( b->b_dns_req ) {
    666         evdns_getaddrinfo_cancel( b->b_dns_req );
    667         b->b_dns_req = NULL;
    668         b->b_opening--;
    669     }
    670     while ( !LDAP_LIST_EMPTY( &b->b_connecting ) ) {
    671         LloadPendingConnection *pending = LDAP_LIST_FIRST( &b->b_connecting );
    672 
    673         Debug( LDAP_DEBUG_CONNS, "backend_reset: "
    674                 "destroying socket pending connect() fd=%d\n",
    675                 pending->fd );
    676 
    677         event_active( pending->event, EV_WRITE, 0 );
    678         evutil_closesocket( pending->fd );
    679         pending->fd = -1;
    680         LDAP_LIST_REMOVE( pending, next );
    681 
    682         if ( !gentle ) {
    683             /* None of the event bases are running, we're safe to free the
    684              * event right now and potentially free the backend itself */
    685             event_free( pending->event );
    686             ch_free( pending );
    687         }
    688         /* else, just let the event dispose of the resources on its own later */
    689         b->b_opening--;
    690     }
    691     connections_walk(
    692             &b->b_mutex, &b->b_preparing, lload_connection_close, &gentle );
    693     assert( LDAP_CIRCLEQ_EMPTY( &b->b_preparing ) );
    694     assert( b->b_opening == ( b->b_cookie ? 1 : 0 ) );
    695     b->b_failed = 0;
    696 
    697     connections_walk_last( &b->b_mutex, &b->b_bindconns, b->b_last_bindconn,
    698             lload_connection_close, &gentle );
    699     assert( gentle || b->b_bindavail == 0 );
    700 
    701     connections_walk_last( &b->b_mutex, &b->b_conns, b->b_last_conn,
    702             lload_connection_close, &gentle );
    703     assert( gentle || b->b_active == 0 );
    704     assert_locked( &b->b_mutex );
    705 }
    706 
    707 LloadBackend *
    708 lload_backend_new( void )
    709 {
    710     LloadBackend *b;
    711 
    712     b = ch_calloc( 1, sizeof(LloadBackend) );
    713 
    714     LDAP_CIRCLEQ_INIT( &b->b_conns );
    715     LDAP_CIRCLEQ_INIT( &b->b_bindconns );
    716     LDAP_CIRCLEQ_INIT( &b->b_preparing );
    717     LDAP_CIRCLEQ_ENTRY_INIT( b, b_next );
    718 
    719     b->b_numconns = 1;
    720     b->b_numbindconns = 1;
    721     b->b_weight = 1;
    722 
    723     b->b_retry_timeout = 5000;
    724 
    725     ldap_pvt_thread_mutex_init( &b->b_mutex );
    726 
    727     return b;
    728 }
    729 
    730 void
    731 lload_backend_destroy( LloadBackend *b )
    732 {
    733     Debug( LDAP_DEBUG_CONNS, "lload_backend_destroy: "
    734             "destroying backend uri='%s', numconns=%d, numbindconns=%d\n",
    735             b->b_uri.bv_val, b->b_numconns, b->b_numbindconns );
    736 
    737     checked_lock( &b->b_mutex );
    738     b->b_tier->t_type.tier_remove_backend( b->b_tier, b );
    739     b->b_numconns = b->b_numbindconns = 0;
    740     backend_reset( b, 0 );
    741 
    742 #ifdef BALANCER_MODULE
    743     if ( b->b_monitor ) {
    744         BackendDB *be;
    745         struct berval monitordn = BER_BVC("cn=monitor");
    746         int rc;
    747 
    748         be = select_backend( &monitordn, 0 );
    749 
    750         /* FIXME: implement proper subsys shutdown in back-monitor or make
    751          * backend just an entry, not a subsys */
    752         rc = b->b_monitor->mss_destroy( be, b->b_monitor );
    753         assert( rc == LDAP_SUCCESS );
    754     }
    755 #endif /* BALANCER_MODULE */
    756 
    757     checked_unlock( &b->b_mutex );
    758     ldap_pvt_thread_mutex_destroy( &b->b_mutex );
    759 
    760     if ( b->b_retry_event ) {
    761         event_del( b->b_retry_event );
    762         event_free( b->b_retry_event );
    763         b->b_retry_event = NULL;
    764     }
    765 
    766     ch_free( b->b_host );
    767     ch_free( b->b_uri.bv_val );
    768     ch_free( b->b_name.bv_val );
    769     ch_free( b );
    770 }
    771