/* * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * For more ,please contact QQ/wechat:4108863/mail:4108863@qq.com * http://www.hihttps.com */ #include "config.h" #include #include #include #include #include /* WAIT_PID */ #ifdef __linux__ #include #endif #ifdef __sun #include #endif #include #include #include /* TCP_NODELAY */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "configuration.h" #include "aihttps.h" #include "hssl_locks.h" #include "logging.h" #include "ocsp.h" #include "shctx.h" #include "foreign/vpf.h" #include "foreign/uthash.h" #include "foreign/vsa.h" #include "waf/rules.h" #include "waf/cc_ddos.h" #include "waf/atklog.h" #ifndef MSG_NOSIGNAL # define MSG_NOSIGNAL 0 #endif #ifndef AI_ADDRCONFIG # define AI_ADDRCONFIG 0 #endif /* For Mac OS X */ #ifndef TCP_KEEPIDLE # ifdef TCP_KEEPALIVE # define TCP_KEEPIDLE TCP_KEEPALIVE # endif #endif #ifndef SOL_TCP # define SOL_TCP IPPROTO_TCP #endif /* Do we have SNI support? */ #ifndef OPENSSL_NO_TLSEXT #ifndef SSL_CTRL_SET_TLSEXT_HOSTNAME #define OPENSSL_NO_TLSEXT #endif #endif /* logging.c */ extern FILE *logfile; extern struct stat logf_st; extern time_t logf_check_t; /* Globals */ struct ev_loop *loop; aihttps_config *CONFIG; /* Worker proc's read side of mgt->worker pipe(2) */ static ev_io mgt_rd; struct backend { unsigned magic; #define BACKEND_MAGIC 0x41c09397 struct suckaddr *backaddr; int ref; }; static struct backend *backaddr; static pid_t master_pid; static pid_t ocsp_proc_pid; static int core_id; static SSL_SESSION *client_session; /* The current number of active client connections. */ static uint64_t n_conns; /* Current generation of worker processes. after a sighup prior * to launching new children. */ static unsigned worker_gen; static volatile unsigned n_sighup; static volatile unsigned n_sigchld; enum worker_state_e { WORKER_ACTIVE, WORKER_EXITING }; static enum worker_state_e worker_state; struct worker_proc { unsigned magic; #define WORKER_PROC_MAGIC 0xbc7fe9e6 /* Writer end of pipe(2) for mgt -> worker ipc */ int pfd; pid_t pid; unsigned gen; int core_id; VTAILQ_ENTRY(worker_proc) list; }; VTAILQ_HEAD(worker_proc_head, worker_proc); static struct worker_proc_head worker_procs; struct sslctx_s; struct sni_name_s; struct listen_sock { unsigned magic; #define LISTEN_SOCK_MAGIC 0xda96b2f6 int sock; char *name; ev_io listener; struct sockaddr_storage addr; VTAILQ_ENTRY(listen_sock) list; }; VTAILQ_HEAD(listen_sock_head, listen_sock); struct frontend { unsigned magic; #define FRONTEND_MAGIC 0x5b04e577 int match_global_certs; int sni_nomatch_abort; struct sni_name_s *sni_names; struct sslctx_s *ssl_ctxs; struct sslctx_s *default_ctx; char *pspec; struct listen_sock_head socks; VTAILQ_ENTRY(frontend) list; }; VTAILQ_HEAD(frontend_head, frontend); static struct frontend_head frontends; #ifdef USE_SHARED_CACHE static ev_io shcupd_listener; static int shcupd_socket; struct addrinfo *shcupd_peers[MAX_SHCUPD_PEERS+1]; static unsigned char shared_secret[SHA_DIGEST_LENGTH]; #endif /*USE_SHARED_CACHE*/ int create_workers; static struct vpf_fh *pfh = NULL; static char tcp_proxy_line[128] = ""; /* What agent/state requests the shutdown--for proper half-closed * handling */ typedef enum _SHUTDOWN_REQUESTOR { SHUTDOWN_HARD, SHUTDOWN_CLEAR, SHUTDOWN_SSL } SHUTDOWN_REQUESTOR; static const char *SHUTDOWN_STR[] = { [SHUTDOWN_HARD] = "SHUTDOWN_HARD", [SHUTDOWN_CLEAR] = "SHUTDOWN_CLEAR", [SHUTDOWN_SSL] = "SHUTDOWN_SSL", }; #ifndef OPENSSL_NO_TLSEXT sni_name *sni_names; static sslctx *ssl_ctxs; static sslctx *default_ctx; static void insert_sni_names(sslctx *sc, sni_name **sn_tab); static int load_cert_ctx(sslctx *so); #endif /* OPENSSL_NO_TLSEXT */ union ha_proxy_v2_addr { struct { /* for TCP/UDP over IPv4, len = 12 */ uint32_t src_addr; uint32_t dst_addr; uint16_t src_port; uint16_t dst_port; } ipv4; struct { /* for TCP/UDP over IPv6, len = 36 */ uint8_t src_addr[16]; uint8_t dst_addr[16]; uint16_t src_port; uint16_t dst_port; } ipv6; struct { /* for AF_UNIX sockets, len = 216 */ uint8_t src_addr[108]; uint8_t dst_addr[108]; } local; }; struct ha_proxy_v2_hdr { uint8_t sig[12]; uint8_t ver_cmd; uint8_t fam; uint16_t len; /* number of following bytes * part of the header */ union ha_proxy_v2_addr addr; }; enum worker_update_type { WORKER_GEN, BACKEND_REFRESH }; union worker_update_payload { unsigned gen; struct sockaddr_storage addr; }; struct worker_update { enum worker_update_type type; union worker_update_payload payload; }; /* set a file descriptor (socket) to non-blocking mode */ static int setnonblocking(int fd) { int flag = 1; if (ioctl(fd, FIONBIO, &flag) < 0) { assert (errno == ECONNRESET || errno == ENOTCONN); return (-1); } return (0); } /* set a tcp socket to use TCP Keepalive */ static void settcpkeepalive(int fd) { int optval = 1; socklen_t optlen = sizeof(optval); if(setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &optval, optlen) < 0) { SOCKERR("Error activating SO_KEEPALIVE on client socket"); } #ifdef TCP_KEEPIDLE optval = CONFIG->TCP_KEEPALIVE_TIME; optlen = sizeof(optval); if (setsockopt(fd, SOL_TCP, TCP_KEEPIDLE, &optval, optlen) < 0) { SOCKERR("Error setting TCP_KEEPIDLE on client socket"); } #endif } #ifndef OPENSSL_NO_DH static int init_dh(SSL_CTX *ctx, const char *cert) { DH *dh; BIO *bio; AN(cert); bio = BIO_new_file(cert, "r"); if (!bio) { log_ssl_error(NULL, "{core} BIO_new_file"); return (-1); } dh = PEM_read_bio_DHparams(bio, NULL, NULL, NULL); BIO_free(bio); if (!dh) { LOG("{core} Note: no DH parameters found in %s\n", cert); return (-1); } LOG("{core} Using DH parameters from %s\n", cert); if (!SSL_CTX_set_tmp_dh(ctx, dh)) { log_ssl_error(NULL, "{core} Error setting temp DH params"); return (-1); } LOG("{core} DH initialized with %d bit key\n", 8*DH_size(dh)); DH_free(dh); return (0); } static int init_ecdh(SSL_CTX *ctx) { #ifndef OPENSSL_NO_EC #ifdef NID_X9_62_prime256v1 EC_KEY *ecdh = NULL; ecdh = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1); SSL_CTX_set_tmp_ecdh(ctx, ecdh); EC_KEY_free(ecdh); LOG("{core} ECDH Initialized with NIST P-256\n"); #endif /* NID_X9_62_prime256v1 */ #endif /* OPENSSL_NO_EC */ return 0; } #endif /* OPENSSL_NO_DH */ /* This callback function is executed while OpenSSL processes the SSL * handshake and does SSL record layer stuff. It's used to trap * client-initiated renegotiations. */ static void info_callback(const SSL *ssl, int where, int ret) { proxystate *ps; (void)ret; if (where & SSL_CB_HANDSHAKE_START) { CAST_OBJ_NOTNULL(ps, SSL_get_app_data(ssl), PROXYSTATE_MAGIC); if (ps->handshaked) { ps->renegotiation = 1; LOG("{core} SSL renegotiation asked by client\n"); } } } #ifdef OPENSSL_WITH_NPN static int npn_select_cb(SSL *ssl, const unsigned char **out, unsigned *outlen, void *arg) { proxystate *ps; (void)arg; CAST_OBJ_NOTNULL(ps, SSL_get_app_data(ssl), PROXYSTATE_MAGIC); ps->npn_alpn_tried = 1; LOG("{npn} Got NPN callback\n"); *out = (unsigned char *) CONFIG->ALPN_PROTOS_LV; *outlen = CONFIG->ALPN_PROTOS_LV_LEN; return SSL_TLSEXT_ERR_OK; } #endif #ifdef OPENSSL_WITH_ALPN static int alpn_select_cb(SSL *ssl, const unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned int inlen, void *arg) { int selected; proxystate *ps; (void)arg; CAST_OBJ_NOTNULL(ps, SSL_get_app_data(ssl), PROXYSTATE_MAGIC); ps->npn_alpn_tried = 1; selected = SSL_select_next_proto((unsigned char **)out, outlen, CONFIG->ALPN_PROTOS_LV, CONFIG->ALPN_PROTOS_LV_LEN, in, inlen); if (selected == OPENSSL_NPN_NEGOTIATED) return SSL_TLSEXT_ERR_OK; else { assert(selected == OPENSSL_NPN_NO_OVERLAP); LOGPROXY(ps, "ALPN: no overlap in protocols.\n"); /* Here it is possible to add logging of which protocols the client wanted */ return SSL_TLSEXT_ERR_NOACK; } return SSL_TLSEXT_ERR_NOACK; } #endif #ifdef USE_SHARED_CACHE /* Handle incoming message updates */ static void handle_shcupd(struct ev_loop *loop, ev_io *w, int revents) { (void)revents; unsigned char msg[SHSESS_MAX_ENCODED_LEN], hash[EVP_MAX_MD_SIZE]; ssize_t r; unsigned int hash_len; uint32_t encdate; long now = (time_t)ev_now(loop); while ((r = recv(w->fd, msg, sizeof(msg), 0)) > 0) { /* msg len must be greater than 1 Byte of data + sig length */ if (r < (int)(1+sizeof(shared_secret))) continue; /* compute sig */ r -= sizeof(shared_secret); HMAC(EVP_sha1(), shared_secret, sizeof(shared_secret), msg, r, hash, &hash_len); if (hash_len != sizeof(shared_secret)) /* should never happen */ continue; /* check sign */ if (memcmp(msg+r, hash, hash_len)) continue; /* msg len must be greater than 1 Byte of data + * encdate length */ if (r < (int)(1+sizeof(uint32_t))) continue; /* drop too unsync updates */ r -= sizeof(uint32_t); encdate = *((uint32_t *)&msg[r]); if (!(abs((int)(int32_t)now - ntohl(encdate)) < SSL_CTX_get_timeout(default_ctx->ctx))) continue; shctx_sess_add(msg, r, now); } } /* Send remote updates messages callback */ void shcupd_session_new(unsigned char *msg, unsigned int len, long cdate) { unsigned int hash_len; struct addrinfo **pai = shcupd_peers; uint32_t ncdate; /* add session creation encoded date to footer */ ncdate = htonl((uint32_t)cdate); memcpy(msg+len, &ncdate, sizeof(ncdate)); len += sizeof(ncdate); /* add msg sign */ HMAC(EVP_sha1(), shared_secret, sizeof(shared_secret), msg, len, msg+len, &hash_len); len += hash_len; /* send msg to peers */ while (*pai) { sendto(shcupd_socket, msg, len, 0, (*pai)->ai_addr, (*pai)->ai_addrlen); pai++; } } /* Compute a sha1 secret from an ASN1 rsa private key */ static int compute_secret(RSA *rsa, unsigned char *secret) { unsigned char *buf, *p; unsigned int length; length = i2d_RSAPrivateKey(rsa, NULL); if (length <= 0) return -1; p = buf = (unsigned char *)malloc(length*sizeof(unsigned char)); if (!buf) return -1; i2d_RSAPrivateKey(rsa,&p); SHA1(buf, length, secret); free(buf); return 0; } /* Create udp socket to receive and send updates */ static int create_shcupd_socket() { struct addrinfo *ai, hints; struct addrinfo **pai = shcupd_peers; memset(&hints, 0, sizeof hints); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_DGRAM; hints.ai_flags = AI_PASSIVE | AI_ADDRCONFIG; const int gai_err = getaddrinfo(CONFIG->SHCUPD_IP, CONFIG->SHCUPD_PORT, &hints, &ai); if (gai_err != 0) { ERR("{getaddrinfo}: %s\n", gai_strerror(gai_err)); exit(1); } /* check if peers inet family addresses match */ while (*pai) { if ((*pai)->ai_family != ai->ai_family) { ERR("Share host and peers inet family differs\n"); exit(1); } pai++; } int s = socket(ai->ai_family, SOCK_DGRAM, IPPROTO_UDP); if (s == -1) fail("{socket: shared cache updates}"); int t = 1; (void)setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &t, sizeof(int)); #ifdef SO_REUSEPORT_WORKS (void)setsockopt(s, SOL_SOCKET, SO_REUSEPORT, &t, sizeof(int)); #endif if (setnonblocking(s) < 0) fail("{shared cache: setnonblocking}"); if (ai->ai_addr->sa_family == AF_INET) { struct ip_mreqn mreqn; memset(&mreqn, 0, sizeof(mreqn)); mreqn.imr_multiaddr.s_addr = ((struct sockaddr_in *)ai->ai_addr)->sin_addr.s_addr; if (CONFIG->SHCUPD_MCASTIF) { if (isalpha(*CONFIG->SHCUPD_MCASTIF)) { /* appears to be an iface name */ struct ifreq ifr; memset(&ifr, 0, sizeof(ifr)); if (strlen(CONFIG->SHCUPD_MCASTIF) > IFNAMSIZ) { ERR("Error iface name is too " "long [%s]\n", CONFIG->SHCUPD_MCASTIF); exit(1); } memcpy(ifr.ifr_name, CONFIG->SHCUPD_MCASTIF, strlen(CONFIG->SHCUPD_MCASTIF)); if (ioctl(s, SIOCGIFINDEX, &ifr)) { fail("{ioctl: SIOCGIFINDEX}"); } mreqn.imr_ifindex = ifr.ifr_ifindex; } else if (strchr(CONFIG->SHCUPD_MCASTIF,'.')) { /* appears to be an ipv4 address */ mreqn.imr_address.s_addr = inet_addr(CONFIG->SHCUPD_MCASTIF); } else { /* appears to be an iface index */ mreqn.imr_ifindex = atoi(CONFIG->SHCUPD_MCASTIF); } } if (setsockopt(s, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreqn, sizeof(mreqn)) < 0) { if (errno != EINVAL) { /* EINVAL if it is not a multicast address, * not an error we consider unicast */ fail("{setsockopt: IP_ADD_MEMBERSIP}"); } } else { /* this is a multicast address */ unsigned char loop = 0; if (setsockopt(s, IPPROTO_IP, IP_MULTICAST_LOOP, &loop, sizeof(loop)) < 0) { fail("{setsockopt: IP_MULTICAST_LOOP}"); } } /* optional set sockopts for sending to multicast msg */ if (CONFIG->SHCUPD_MCASTIF && setsockopt(s, IPPROTO_IP, IP_MULTICAST_IF, &mreqn, sizeof(mreqn)) < 0) { fail("{setsockopt: IP_MULTICAST_IF}"); } if (CONFIG->SHCUPD_MCASTTTL) { unsigned char ttl; ttl = (unsigned char)atoi(CONFIG->SHCUPD_MCASTTTL); if (setsockopt(s, IPPROTO_IP, IP_MULTICAST_TTL, &ttl, sizeof(ttl)) < 0) { fail("{setsockopt: IP_MULTICAST_TTL}"); } } } #ifdef IPV6_ADD_MEMBERSHIP else if (ai->ai_addr->sa_family == AF_INET6) { struct ipv6_mreq mreq; memset(&mreq, 0, sizeof(mreq)); memcpy(&mreq.ipv6mr_multiaddr, &((struct sockaddr_in6 *)ai->ai_addr)->sin6_addr, sizeof(mreq.ipv6mr_multiaddr)); if (CONFIG->SHCUPD_MCASTIF) { if (isalpha(*CONFIG->SHCUPD_MCASTIF)) { /* appears to be an iface name */ struct ifreq ifr; memset(&ifr, 0, sizeof(ifr)); if (strlen(CONFIG->SHCUPD_MCASTIF) > IFNAMSIZ) { ERR("Error iface name is too " "long [%s]\n", CONFIG->SHCUPD_MCASTIF); exit(1); } memcpy(ifr.ifr_name, CONFIG->SHCUPD_MCASTIF, strlen(CONFIG->SHCUPD_MCASTIF)); if (ioctl(s, SIOCGIFINDEX, &ifr)) fail("{ioctl: SIOCGIFINDEX}"); mreq.ipv6mr_interface = ifr.ifr_ifindex; } else { /* option appears to be an iface index */ mreq.ipv6mr_interface = atoi(CONFIG->SHCUPD_MCASTIF); } } if (setsockopt(s, IPPROTO_IPV6, IPV6_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) < 0) { if (errno != EINVAL) { /* EINVAL if it is not a multicast address, * not an error we consider unicast */ fail("{setsockopt: IPV6_ADD_MEMBERSIP}"); } } else { /* this is a multicast address */ unsigned int loop = 0; if (setsockopt(s, IPPROTO_IPV6, IPV6_MULTICAST_LOOP, &loop, sizeof(loop)) < 0) { fail("{setsockopt: IPV6_MULTICAST_LOOP}"); } } /* optional set sockopts for sending to multicast msg */ if (setsockopt(s, IPPROTO_IPV6, IPV6_MULTICAST_IF, &mreq.ipv6mr_interface, sizeof(mreq.ipv6mr_interface)) < 0) { fail("{setsockopt: IPV6_MULTICAST_IF}"); } if (CONFIG->SHCUPD_MCASTTTL) { int hops; hops = atoi(CONFIG->SHCUPD_MCASTTTL); if (setsockopt(s, IPPROTO_IPV6, IPV6_MULTICAST_HOPS, &hops, sizeof(hops)) < 0) { fail("{setsockopt: IPV6_MULTICAST_HOPS}"); } } } #endif /* IPV6_ADD_MEMBERSHIP */ if (bind(s, ai->ai_addr, ai->ai_addrlen)) { fail("{bind-socket}"); } freeaddrinfo(ai); return s; } #endif /*USE_SHARED_CACHE */ EVP_PKEY * load_privatekey(SSL_CTX *ctx, const char *file) { BIO *bio; EVP_PKEY *pkey; bio = BIO_new_file(file, "r"); if (!bio) { log_ssl_error(NULL, "{core} BIO_new_file"); return NULL; } #if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER) #define SSL_CTX_get_default_passwd_cb(ctx) (ctx->default_passwd_callback) #define SSL_CTX_get_default_passwd_cb_userdata(ctx) (ctx->default_passwd_callback_userdata) #endif pkey = PEM_read_bio_PrivateKey(bio, NULL, SSL_CTX_get_default_passwd_cb(ctx), SSL_CTX_get_default_passwd_cb_userdata(ctx)); BIO_free(bio); if (!pkey) { log_ssl_error(NULL, "{core} PEM_read_bio_PrivateKey"); return NULL; } return (pkey); } #ifndef OPENSSL_NO_TLSEXT static int sni_match(const sni_name *sn, const char *srvname) { if (!sn->is_wildcard) return (strcasecmp(srvname, sn->servername) == 0); else { char *s = strchr(srvname, '.'); if (s == NULL) return (0); return (strcasecmp(s, sn->servername + 1) == 0); } } static sslctx * sni_lookup(const char *servername, const sni_name *sn_tab) { const sni_name *sn; HASH_FIND_STR(sn_tab, servername, sn); if (sn == NULL) { char *s; /* attempt another lookup for wildcard matches */ s = strchr(servername, '.'); if (s != NULL) { HASH_FIND_STR(sn_tab, s, sn); } } if (sn != NULL) { CHECK_OBJ_NOTNULL(sn, SNI_NAME_MAGIC); if (sni_match(sn, servername)) return (sn->sctx); } return (NULL); } /* * Switch the context of the current SSL object to the most appropriate one * based on the SNI header */ static int sni_switch_ctx(SSL *ssl, int *al, void *data) { const char *servername; sslctx *sc; const struct frontend *fr = NULL; int lookup_global = 1; int sni_nomatch_abort = CONFIG->SNI_NOMATCH_ABORT; (void)al; if (data != NULL) CAST_OBJ_NOTNULL(fr, data, FRONTEND_MAGIC); servername = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); if (!servername) return (SSL_TLSEXT_ERR_NOACK); #define TRY_SNI_MATCH(sn_tab) \ do { \ sc = sni_lookup(servername, (sn_tab)); \ if (sc != NULL) { \ CHECK_OBJ_NOTNULL(sc, SSLCTX_MAGIC); \ SSL_set_SSL_CTX(ssl, sc->ctx); \ return (SSL_TLSEXT_ERR_OK); \ } \ } while (0) if (fr != NULL) { TRY_SNI_MATCH(fr->sni_names); lookup_global = fr->match_global_certs; if (fr->sni_nomatch_abort != -1) sni_nomatch_abort = fr->sni_nomatch_abort; } if (lookup_global) TRY_SNI_MATCH(sni_names); /* No matching certs */ if (sni_nomatch_abort) return (SSL_TLSEXT_ERR_ALERT_FATAL); else return (SSL_TLSEXT_ERR_NOACK); } #endif /* OPENSSL_NO_TLSEXT */ static void sctx_free(sslctx *sc, sni_name **sn_tab) { sni_name *sn, *sntmp; if (sc == NULL) return; HOCSP_free(&sc->staple); if (sn_tab != NULL) CHECK_OBJ_NOTNULL(*sn_tab, SNI_NAME_MAGIC); CHECK_OBJ_NOTNULL(sc, SSLCTX_MAGIC); VTAILQ_FOREACH_SAFE(sn, &sc->sni_list, list, sntmp) { CHECK_OBJ_NOTNULL(sn, SNI_NAME_MAGIC); VTAILQ_REMOVE(&sc->sni_list, sn, list); if (sn_tab != NULL) HASH_DEL(*sn_tab, sn); free(sn->servername); FREE_OBJ(sn); } free(sc->filename); SSL_CTX_free(sc->ctx); FREE_OBJ(sc); } X509 * Find_issuer(X509 *subj, STACK_OF(X509) *chain) { int i; X509 *x; AN(subj); for (i = 0; i < sk_X509_num(chain); i++) { x = sk_X509_value(chain, i); if (X509_check_issued(x, subj) == X509_V_OK) return (x); } /* todo: look in cert store? */ return (NULL); } /* Initialize an SSL context */ static sslctx * make_ctx_fr(const struct cfg_cert_file *cf, const struct frontend *fr, const struct front_arg *fa) { SSL_CTX *ctx; sslctx *sc; EVP_PKEY *pkey; int selected_protos = CONFIG->SELECTED_TLS_PROTOS; char *ciphers = CONFIG->CIPHER_SUITE; int pref_srv_ciphers = CONFIG->PREFER_SERVER_CIPHERS; if (fa != NULL) { CHECK_OBJ_NOTNULL(fa, FRONT_ARG_MAGIC); if (fa->selected_protos != 0) selected_protos = fa->selected_protos; if (fa->ciphers != NULL) ciphers = fa->ciphers; if (fa->prefer_server_ciphers != -1) pref_srv_ciphers = fa->prefer_server_ciphers; } long ssloptions = SSL_OP_NO_SSLv2 | SSL_OP_ALL | SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION; #ifdef SSL_OP_NO_COMPRESSION ssloptions |= SSL_OP_NO_COMPRESSION; #endif #ifdef SSL_OP_SINGLE_DH_USE ssloptions |= SSL_OP_SINGLE_DH_USE; #endif #ifdef SSL_OP_SINGLE_ECDH_USE ssloptions |= SSL_OP_SINGLE_ECDH_USE; #endif if (!(selected_protos & SSLv3_PROTO)) ssloptions |= SSL_OP_NO_SSLv3; if (!(selected_protos & TLSv1_0_PROTO)) ssloptions |= SSL_OP_NO_TLSv1; if (!(selected_protos & TLSv1_1_PROTO)) ssloptions |= SSL_OP_NO_TLSv1_1; if (!(selected_protos & TLSv1_2_PROTO)) ssloptions |= SSL_OP_NO_TLSv1_2; #ifdef SSL_OP_NO_TLSv1_3 if (!(selected_protos & TLSv1_3_PROTO)) ssloptions |= SSL_OP_NO_TLSv1_3; #endif ctx = SSL_CTX_new((CONFIG->PMODE == SSL_CLIENT) ? SSLv23_client_method() : SSLv23_server_method()); SSL_CTX_set_options(ctx, ssloptions); SSL_CTX_set_info_callback(ctx, info_callback); #ifdef OPENSSL_WITH_ALPN if (CONFIG->ALPN_PROTOS != NULL) SSL_CTX_set_alpn_select_cb(ctx, alpn_select_cb, NULL); #endif #ifdef OPENSSL_WITH_NPN if (CONFIG->ALPN_PROTOS != NULL) SSL_CTX_set_next_protos_advertised_cb(ctx, npn_select_cb, NULL); #endif if (ciphers != NULL) { if (SSL_CTX_set_cipher_list(ctx, ciphers) != 1) { log_ssl_error(NULL, "{core} SSL_CTX_set_cipher_list"); return (NULL); } } if (pref_srv_ciphers) SSL_CTX_set_options(ctx, SSL_OP_CIPHER_SERVER_PREFERENCE); ALLOC_OBJ(sc, SSLCTX_MAGIC); AN(sc); sc->filename = strdup(cf->filename); sc->mtim = cf->mtim; sc->ctx = ctx; sc->staple_vfy = cf->ocsp_vfy; VTAILQ_INIT(&sc->sni_list); if (sc->staple_vfy > 0 || (sc-> staple_vfy < 0 && CONFIG->OCSP_VFY)) AN(SSL_CTX_set_default_verify_paths(ctx)); if (CONFIG->PMODE == SSL_CLIENT) return (sc); /* SSL_SERVER Mode stuff */ if (SSL_CTX_use_certificate_chain_file(ctx, cf->filename) <= 0) { log_ssl_error(NULL, "Error loading certificate file %s\n", cf->filename); sctx_free(sc, NULL); return (NULL); } pkey = 0; if (cf->priv_key_filename != 0) { pkey = load_privatekey(ctx, cf->priv_key_filename); if (!pkey) { ERR("Error loading private key (%s)\n", cf->priv_key_filename); sctx_free(sc, NULL); return (NULL); } } if (!pkey) { pkey = load_privatekey(ctx, cf->filename); if (!pkey) { ERR("Error loading private key (%s)\n", cf->filename); sctx_free(sc, NULL); return (NULL); } } if (SSL_CTX_use_PrivateKey(ctx, pkey) <= 0) { log_ssl_error(NULL, "SSL_CTX_use_PrivateKey: %s", cf->filename); EVP_PKEY_free(pkey); sctx_free(sc, NULL); return (NULL); } #ifndef OPENSSL_NO_DH init_dh(ctx, cf->filename); init_ecdh(ctx); #endif /* OPENSSL_NO_DH */ #ifndef OPENSSL_NO_TLSEXT if (!SSL_CTX_set_tlsext_servername_callback(ctx, sni_switch_ctx)) { ERR("Error setting up SNI support.\n"); } CHECK_OBJ_ORNULL(fr, FRONTEND_MAGIC); if (!SSL_CTX_set_tlsext_servername_arg(ctx, (void *)fr)) { ERR("Error setting SNI servername arg.\n"); } if (load_cert_ctx(sc) != 0) { EVP_PKEY_free(pkey); sctx_free(sc, NULL); return (NULL); } if (CONFIG->OCSP_DIR) { char *fn = HOCSP_fn(sc->filename); /* attempt loading of cached ocsp staple */ if (fn != NULL && HOCSP_init_file(fn, sc, 1) == 0) { LOG("{core} Loaded cached OCSP staple for cert '%s'\n", sc->filename); sc->staple_fn = fn; } } if (sc->staple == NULL && cf->ocspfn != NULL) { if (HOCSP_init_file(cf->ocspfn, sc, 0) != 0) { ERR("Error loading OCSP response %s for stapling.\n", cf->ocspfn); EVP_PKEY_free(pkey); sctx_free(sc, NULL); return (NULL); } else { LOG("{core} Loaded OCSP staple '%s'\n", cf->ocspfn); sc->staple_fn = strdup(cf->ocspfn); sc->staple->mtim = cf->ocsp_mtim; } } if (CONFIG->OCSP_DIR != NULL) HOCSP_ev_stat(sc); #endif /* OPENSSL_NO_TLSEXT */ #ifdef USE_SHARED_CACHE if (CONFIG->SHARED_CACHE) { if (shared_context_init(ctx, CONFIG->SHARED_CACHE) < 0) { ERR("Unable to alloc memory for shared cache.\n"); EVP_PKEY_free(pkey); sctx_free(sc, NULL); return (NULL); } if (CONFIG->SHCUPD_PORT) { RSA *rsa; rsa = EVP_PKEY_get1_RSA(pkey); if (rsa != NULL && compute_secret(rsa, shared_secret) < 0) { ERR("Unable to compute shared secret.\n"); EVP_PKEY_free(pkey); sctx_free(sc, NULL); return (NULL); } /* Force TLS tickets because keys differs. */ SSL_CTX_set_options(ctx, SSL_OP_NO_TICKET); if (*shcupd_peers) { shsess_set_new_cbk(shcupd_session_new); } } } #endif EVP_PKEY_free(pkey); return (sc); } static void insert_sni_names(sslctx *sc, sni_name **sn_tab) { sni_name *sn, *sn2; char *key; CHECK_OBJ_NOTNULL(sc, SSLCTX_MAGIC); VTAILQ_FOREACH(sn, &sc->sni_list, list) { CHECK_OBJ_NOTNULL(sn, SNI_NAME_MAGIC); key = sn->servername; if (sn->is_wildcard) key = sn->servername + 1; HASH_FIND_STR(*sn_tab, key, sn2); if (sn2 != NULL) { ERR("Warning: SNI name '%s' from '%s' overridden" " by '%s'\n", key, sn2->sctx->filename, sn->sctx->filename); } HASH_ADD_KEYPTR(hh, *sn_tab, key, strlen(key), sn); } } static sslctx * make_ctx(const struct cfg_cert_file *cf) { return (make_ctx_fr(cf, NULL, NULL)); } #ifndef OPENSSL_NO_TLSEXT static int load_cert_ctx(sslctx *so) { X509 *x509; X509_NAME *x509_name; X509_NAME_ENTRY *x509_entry; BIO *f; STACK_OF(GENERAL_NAME) *names = NULL; GENERAL_NAME *name; int i; #define PUSH_CTX(asn1_str, ctx) \ do { \ sni_name *sn; \ ALLOC_OBJ(sn, SNI_NAME_MAGIC); \ ASN1_STRING_to_UTF8( \ (unsigned char **)&sn->servername, asn1_str); \ sn->is_wildcard = \ (strstr(sn->servername, "*.") == sn->servername); \ sn->sctx = so; \ VTAILQ_INSERT_TAIL(&so->sni_list, sn, list); \ } while (0) f = BIO_new(BIO_s_file()); // TODO: error checking if (!BIO_read_filename(f, so->filename)) { ERR("Could not read certificate '%s'\n", so->filename); return (1); } x509 = PEM_read_bio_X509_AUX(f, NULL, NULL, NULL); BIO_free(f); so->x509 = x509; /* First, look for Subject Alternative Names. */ names = X509_get_ext_d2i(x509, NID_subject_alt_name, NULL, NULL); for (i = 0; i < sk_GENERAL_NAME_num(names); i++) { name = sk_GENERAL_NAME_value(names, i); if (name->type == GEN_DNS) { PUSH_CTX(name->d.dNSName, ctx); } } if (sk_GENERAL_NAME_num(names) > 0) { sk_GENERAL_NAME_pop_free(names, GENERAL_NAME_free); /* If we found some, don't bother looking any further. */ return (0); } else if (names != NULL) { sk_GENERAL_NAME_pop_free(names, GENERAL_NAME_free); } /* Now we're left looking at the CN on the cert. */ x509_name = X509_get_subject_name(x509); i = X509_NAME_get_index_by_NID(x509_name, NID_commonName, -1); if (i < 0) { ERR("Could not find Subject Alternative Names" " or a CN on cert %s\n", so->filename); return (1); } x509_entry = X509_NAME_get_entry(x509_name, i); #if OPENSSL_VERSION_NUMBER < 0x10100000L #define X509_NAME_ENTRY_get_data(e) (e->value) #endif AN(x509_entry); PUSH_CTX(X509_NAME_ENTRY_get_data(x509_entry), ctx); return (0); } #endif /* OPENSSL_NO_TLSEXT */ /* Check that we don't needlessly load a cert that's already loaded. */ static sslctx * find_ctx(const char *file) { sslctx *so; HASH_FIND_STR(ssl_ctxs, file, so); return (so); } /* Init library and load specified certificate. * Establishes a SSL_ctx, to act as a template for * each connection */ void init_openssl(void) { SSL_library_init(); SSL_load_error_strings(); OpenSSL_add_all_digests(); if (CONFIG->ENGINE) { ENGINE *e = NULL; ENGINE_load_builtin_engines(); HSSL_Locks_Init(); if (!strcmp(CONFIG->ENGINE, "auto")) ENGINE_register_all_complete(); else { if ((e = ENGINE_by_id(CONFIG->ENGINE)) == NULL || !ENGINE_init(e) || !ENGINE_set_default(e, ENGINE_METHOD_ALL)) { log_ssl_error(NULL, "{core} ENGINE initialization failed"); exit(1); } LOG("{core} will use OpenSSL engine %s.\n", ENGINE_get_id(e)); ENGINE_finish(e); ENGINE_free(e); } } } static void init_certs(void) { struct cfg_cert_file *cf, *cftmp; sslctx *so; if (CONFIG->CERT_DEFAULT != NULL) { default_ctx = make_ctx(CONFIG->CERT_DEFAULT); if (default_ctx == NULL) exit(1); #ifndef OPENSSL_NO_TLSEXT insert_sni_names(default_ctx, &sni_names); #endif } // Go through the list of PEMs and make some SSL contexts for // them. We also keep track of the names associated with each // cert so we can do SNI on them later HASH_ITER(hh, CONFIG->CERT_FILES, cf, cftmp) { if (find_ctx(cf->filename) == NULL) { so = make_ctx(cf); if (so == NULL) exit(1); HASH_ADD_KEYPTR(hh, ssl_ctxs, so->filename, strlen(so->filename), so); #ifndef OPENSSL_NO_TLSEXT insert_sni_names(so, &sni_names); #endif } } } static void destroy_lsock(struct listen_sock *ls) { CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); if (ls->sock > 0) (void) close(ls->sock); free(ls->name); FREE_OBJ(ls); } static void destroy_frontend(struct frontend *fr) { struct listen_sock *ls, *lstmp; sslctx *sc, *sctmp; CHECK_OBJ_NOTNULL(fr, FRONTEND_MAGIC); VTAILQ_FOREACH_SAFE(ls, &fr->socks, list, lstmp) { CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); VTAILQ_REMOVE(&fr->socks, ls, list); destroy_lsock(ls); } HASH_ITER(hh, fr->ssl_ctxs, sc, sctmp) { HASH_DEL(fr->ssl_ctxs, sc); sctx_free(sc, &fr->sni_names); } AZ(HASH_COUNT(fr->sni_names)); free(fr->pspec); FREE_OBJ(fr); } /* Create the bound socket in the parent process */ static int frontend_listen(const struct front_arg *fa, struct listen_sock_head *slist) { struct addrinfo *ai, hints, *it; struct listen_sock *ls, *lstmp; char buf[INET6_ADDRSTRLEN+20]; char abuf[INET6_ADDRSTRLEN]; char pbuf[8]; int r, count = 0; CHECK_OBJ_NOTNULL(fa, FRONT_ARG_MAGIC); memset(&hints, 0, sizeof hints); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_PASSIVE | AI_ADDRCONFIG; r = getaddrinfo(fa->ip, fa->port, &hints, &ai); if (r != 0) { ERR("{getaddrinfo-listen}: %s: %s\n", fa->pspec, gai_strerror(r)); return (-1); } for (it = ai; it != NULL; it = it->ai_next) { ALLOC_OBJ(ls, LISTEN_SOCK_MAGIC); VTAILQ_INSERT_TAIL(slist, ls, list); count++; ls->sock = socket(it->ai_family, SOCK_STREAM, IPPROTO_TCP); if (ls->sock == -1) { ERR("{socket: main}: %s: %s\n", strerror(errno), fa->pspec); goto creat_frontend_err; } int t = 1; if (setsockopt(ls->sock, SOL_SOCKET, SO_REUSEADDR, &t, sizeof(int)) < 0) { ERR("{setsockopt-reuseaddr}: %s: %s\n", strerror(errno), fa->pspec); goto creat_frontend_err; } #ifdef SO_REUSEPORT_WORKS if (setsockopt(ls->sock, SOL_SOCKET, SO_REUSEPORT, &t, sizeof(int)) < 0) { ERR("{setsockopt-reuseport}: %s: %s\n", strerror(errno), fa->pspec); goto creat_frontend_err; } #endif if(setnonblocking(ls->sock) < 0) { ERR("{listen sock: setnonblocking}: %s: %s\n", strerror(errno), fa->pspec); goto creat_frontend_err; } #ifdef IPV6_V6ONLY t = 1; if (it->ai_family == AF_INET6 && setsockopt(ls->sock, IPPROTO_IPV6, IPV6_V6ONLY, &t, sizeof (t)) != 0) { ERR("{setsockopt-ipv6only}: %s: %s\n", strerror(errno), fa->pspec); goto creat_frontend_err; } #endif if (CONFIG->RECV_BUFSIZE > 0) { r = setsockopt(ls->sock, SOL_SOCKET, SO_RCVBUF, &CONFIG->RECV_BUFSIZE, sizeof(CONFIG->RECV_BUFSIZE)); if (r < 0) { ERR("{setsockopt-rcvbuf}: %s: %s\n", strerror(errno), fa->pspec); goto creat_frontend_err; } } if (CONFIG->SEND_BUFSIZE > 0) { r = setsockopt(ls->sock, SOL_SOCKET, SO_SNDBUF, &CONFIG->SEND_BUFSIZE, sizeof(CONFIG->SEND_BUFSIZE)); if (r < 0) { ERR("{setsockopt-sndbuf}: %s: %s\n", strerror(errno), fa->pspec); goto creat_frontend_err; } } if (bind(ls->sock, it->ai_addr, it->ai_addrlen)) { ERR("{bind-socket}: %s: %s\n", strerror(errno), fa->pspec); goto creat_frontend_err; } #ifndef NO_DEFER_ACCEPT #if TCP_DEFER_ACCEPT int timeout = 1; if (setsockopt(ls->sock, IPPROTO_TCP, TCP_DEFER_ACCEPT, &timeout, sizeof(int)) < 0) { ERR("{setsockopt-defer_accept}: %s: %s\n", strerror(errno), fa->pspec); goto creat_frontend_err; } #endif /* TCP_DEFER_ACCEPT */ #endif if (listen(ls->sock, CONFIG->BACKLOG) != 0) { ERR("{listen-socket}: %s: %s\n", strerror(errno), fa->pspec); goto creat_frontend_err; } memcpy(&ls->addr, it->ai_addr, it->ai_addrlen); r = getnameinfo(it->ai_addr, it->ai_addrlen, abuf, sizeof abuf, pbuf, sizeof pbuf, NI_NUMERICHOST | NI_NUMERICSERV); if (r != 0) { ERR("{getnameinfo}: %s\n", fa->pspec); goto creat_frontend_err; } if (it->ai_addr->sa_family == AF_INET6) { sprintf(buf, "[%s]:%s", abuf, pbuf); } else { sprintf(buf, "%s:%s", abuf, pbuf); } ls->name = strdup(buf); AN(ls->name); LOG("{core} Listening on %s\n", ls->name); } freeaddrinfo(ai); return (count); creat_frontend_err: freeaddrinfo(ai); VTAILQ_FOREACH_SAFE(ls, slist, list, lstmp) { VTAILQ_REMOVE(slist, ls, list); free(ls->name); if (ls->sock > 0) (void) close(ls->sock); FREE_OBJ(ls); } return (-1); } static struct frontend * create_frontend(const struct front_arg *fa) { struct frontend *fr; sslctx *so; int count = 0; struct frontend_head tmp_list; struct cfg_cert_file *cf; CHECK_OBJ_NOTNULL(fa, FRONT_ARG_MAGIC); ALLOC_OBJ(fr, FRONTEND_MAGIC); VTAILQ_INIT(&fr->socks); AN(fr); fr->pspec = strdup(fa->pspec); fr->match_global_certs = fa->match_global_certs; fr->sni_nomatch_abort = fa->sni_nomatch_abort; VTAILQ_INIT(&tmp_list); count = frontend_listen(fa, &fr->socks); if (count < 0) { destroy_frontend(fr); return (NULL); } for (cf = fa->certs; cf != NULL; cf = cf->hh.next) { so = make_ctx_fr(cf, fr, fa); if (so == NULL) { destroy_frontend(fr); return (NULL); } HASH_ADD_KEYPTR(hh, fr->ssl_ctxs, so->filename, strlen(so->filename), so); #ifndef OPENSSL_NO_TLSEXT insert_sni_names(so, &fr->sni_names); #endif if (cf->hh.next == NULL) fr->default_ctx = so; } return (fr); } static const void * Get_Sockaddr(const struct sockaddr *sa, socklen_t *sl); static struct backend * backend_create(struct sockaddr *sa) { socklen_t len; const void *addr; struct backend *b; struct suckaddr *su; addr = Get_Sockaddr(sa, &len); AN(addr); su = VSA_Malloc(addr, len); ALLOC_OBJ(b, BACKEND_MAGIC); b->backaddr = su; b->ref = 1; return (b); } static struct backend * backend_ref(void) { CHECK_OBJ_NOTNULL(backaddr, BACKEND_MAGIC); AN(backaddr->ref); backaddr->ref++; return (backaddr); } void backend_deref(struct backend **be) { struct backend *b; b = *be; CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC); AN(b->ref); b->ref--; if (b->ref == 0) { free(b->backaddr); FREE_OBJ(*be); } } /* Initiate a clear-text nonblocking connect() to the backend IP on behalf * of a newly connected upstream (encrypted) client */ static int create_back_socket(struct backend *b) { socklen_t len; const struct sockaddr *addr; CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC); addr = (struct sockaddr *) VSA_Get_Sockaddr(b->backaddr, &len); AN(addr); int s = socket(addr->sa_family, SOCK_STREAM, 0); if (s == -1) return -1; if (addr->sa_family != PF_UNIX) { int flag = 1; int ret = setsockopt(s, IPPROTO_TCP, TCP_NODELAY, (char *)&flag, sizeof(flag)); if (ret == -1) ERR("Couldn't setsockopt to backend (TCP_NODELAY):" " %s\n", strerror(errno)); } if (setnonblocking(s) < 0) { (void)close(s); return (-1); } return (s); } /* Only enable a libev ev_io event if the proxied connection still * has both up and down connected */ static void safe_enable_io(proxystate *ps, ev_io *w) { CHECK_OBJ_NOTNULL(ps, PROXYSTATE_MAGIC); if (!ps->want_shutdown) ev_io_start(loop, w); } static void check_exit_state(void) { if (worker_state == WORKER_EXITING && n_conns == 0) { LOGL("Worker %d (gen: %d) in state EXITING " "is now exiting.\n", core_id, worker_gen); _exit(0); } } /* Only enable a libev ev_io event if the proxied connection still * has both up and down connected */ static void shutdown_proxy(proxystate *ps, SHUTDOWN_REQUESTOR req) { CHECK_OBJ_NOTNULL(ps, PROXYSTATE_MAGIC); LOGPROXY(ps, "proxy shutdown req=%s\n", SHUTDOWN_STR[req]); if (ps->want_shutdown || req == SHUTDOWN_HARD) { ev_io_stop(loop, &ps->ev_w_ssl); ev_io_stop(loop, &ps->ev_r_ssl); ev_io_stop(loop, &ps->ev_w_handshake); ev_io_stop(loop, &ps->ev_r_handshake); ev_timer_stop(loop, &ps->ev_t_handshake); ev_io_stop(loop, &ps->ev_w_connect); ev_timer_stop(loop, &ps->ev_t_connect); ev_io_stop(loop, &ps->ev_w_clear); ev_io_stop(loop, &ps->ev_r_clear); ev_io_stop(loop, &ps->ev_proxy); (void)SSL_shutdown(ps->ssl); ERR_clear_error(); SSL_free(ps->ssl); free(ps->req); close(ps->fd_up); close(ps->fd_down); backend_deref(&ps->backend); ringbuffer_cleanup(&ps->ring_clear2ssl); ringbuffer_cleanup(&ps->ring_ssl2clear); free(ps); n_conns--; check_exit_state(); } else { ps->want_shutdown = 1; if (req == SHUTDOWN_CLEAR && ringbuffer_is_empty(&ps->ring_clear2ssl)) shutdown_proxy(ps, SHUTDOWN_HARD); else if (req == SHUTDOWN_SSL && ringbuffer_is_empty(&ps->ring_ssl2clear)) shutdown_proxy(ps, SHUTDOWN_HARD); } } /* Handle various socket errors */ static void handle_socket_errno(proxystate *ps, int backend) { CHECK_OBJ_NOTNULL(ps, PROXYSTATE_MAGIC); if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) return; if (backend) ERR("{backend} Socket error: %s\n", strerror(errno)); else LOG("{client} Socket error: %s\n", strerror(errno)); shutdown_proxy(ps, SHUTDOWN_CLEAR); } /* Start connect to backend */ static int start_connect(proxystate *ps) { int t = 1; socklen_t len; const void *addr; CHECK_OBJ_NOTNULL(ps, PROXYSTATE_MAGIC); CHECK_OBJ_NOTNULL(ps->backend, BACKEND_MAGIC); addr = VSA_Get_Sockaddr(ps->backend->backaddr, &len); AN(addr); t = connect(ps->fd_down, addr, len); if (t == 0 || errno == EINPROGRESS || errno == EINTR) { ev_io_start(loop, &ps->ev_w_connect); ev_timer_start(loop, &ps->ev_t_connect); return (0); } ERR("{backend-connect}: %s\n", strerror(errno)); shutdown_proxy(ps, SHUTDOWN_HARD); return (-1); } /* Read some data from the backend when libev says data is available-- * write it into the upstream buffer and make sure the write event is * enabled for the upstream socket */ static void clear_read(struct ev_loop *loop, ev_io *w, int revents) { (void) revents; int t; proxystate *ps; CAST_OBJ_NOTNULL(ps, w->data, PROXYSTATE_MAGIC); if (ps->want_shutdown) { ev_io_stop(loop, &ps->ev_r_clear); return; } int fd = w->fd; char *buf = ringbuffer_write_ptr(&ps->ring_clear2ssl); t = recv(fd, buf, ps->ring_clear2ssl.data_len, 0); if (t > 0) { ringbuffer_write_append(&ps->ring_clear2ssl, t); if (ringbuffer_is_full(&ps->ring_clear2ssl)) ev_io_stop(loop, &ps->ev_r_clear); if (ps->handshaked) safe_enable_io(ps, &ps->ev_w_ssl); } else if (t == 0) { LOGPROXY(ps,"Connection closed by %s\n", fd == ps->fd_down ? "backend" : "client"); shutdown_proxy(ps, SHUTDOWN_CLEAR); } else { assert(t == -1); handle_socket_errno(ps, fd == ps->fd_down ? 1 : 0); } } /* Write some data, previously received on the secure upstream socket, * out of the downstream buffer and onto the backend socket */ static void clear_write(struct ev_loop *loop, ev_io *w, int revents) { (void)revents; int t; proxystate *ps; int fd = w->fd; int sz; CAST_OBJ_NOTNULL(ps, w->data, PROXYSTATE_MAGIC); assert(!ringbuffer_is_empty(&ps->ring_ssl2clear)); char *next = ringbuffer_read_next(&ps->ring_ssl2clear, &sz); t = send(fd, next, sz, MSG_NOSIGNAL); if (t > 0) { if (t == sz) { ringbuffer_read_pop(&ps->ring_ssl2clear); if (ps->handshaked) safe_enable_io(ps, &ps->ev_r_ssl); if (ringbuffer_is_empty(&ps->ring_ssl2clear)) { if (ps->want_shutdown) { shutdown_proxy(ps, SHUTDOWN_HARD); return; // dealloc'd } ev_io_stop(loop, &ps->ev_w_clear); } } else { ringbuffer_read_skip(&ps->ring_ssl2clear, t); } } else { assert(t == -1); handle_socket_errno(ps, fd == ps->fd_down ? 1 : 0); } } static void start_handshake(proxystate *ps, int err); static unsigned sockaddr_port(const struct sockaddr *sa) { const struct sockaddr_in *sa4; const struct sockaddr_in6 *sa6; switch (sa->sa_family) { case PF_INET: sa4 = (struct sockaddr_in *) sa; return (ntohs((sa4->sin_port))); case PF_INET6: sa6 = (struct sockaddr_in6 *) sa; return (ntohs((sa6->sin6_port))); default: return (0); } } /* Continue/complete the asynchronous connect() before starting data * transmission between front/backend */ static void handle_connect(struct ev_loop *loop, ev_io *w, int revents) { int t, r; proxystate *ps; socklen_t len; const void *addr; (void)revents; CAST_OBJ_NOTNULL(ps, w->data, PROXYSTATE_MAGIC); CHECK_OBJ_NOTNULL(ps->backend, BACKEND_MAGIC); addr = VSA_Get_Sockaddr(ps->backend->backaddr, &len); AN(addr); t = connect(ps->fd_down, addr, len); if (!t || errno == EISCONN || !errno) { ev_io_stop(loop, &ps->ev_w_connect); ev_timer_stop(loop, &ps->ev_t_connect); if (!ps->clear_connected) { struct sockaddr_storage ss; socklen_t sl; sl = sizeof(addr); r = getsockname(ps->fd_down, (struct sockaddr *) &ss, &sl); AZ(r); ps->connect_port = sockaddr_port( (struct sockaddr *) &ss); LOGPROXY(ps, "backend connected\n"); ps->clear_connected = 1; /* if incoming buffer is not full */ if (!ringbuffer_is_full(&ps->ring_clear2ssl)) safe_enable_io(ps, &ps->ev_r_clear); /* if outgoing buffer is not empty */ if (!ringbuffer_is_empty(&ps->ring_ssl2clear)) // not safe.. we want to resume stream // even during half-closed ev_io_start(loop, &ps->ev_w_clear); } else { /* Clear side already connected so connect is on * secure side: perform handshake */ start_handshake(ps, SSL_ERROR_WANT_WRITE); } } else if (errno == EINPROGRESS || errno == EINTR || errno == EALREADY) { /* do nothing, we'll get phoned home again... */ } else { ERR("{backend-connect}: %s\n", strerror(errno)); shutdown_proxy(ps, SHUTDOWN_HARD); } } static void connect_timeout(EV_P_ ev_timer *w, int revents) { (void)loop; (void)revents; proxystate *ps; CAST_OBJ_NOTNULL(ps, w->data, PROXYSTATE_MAGIC); ERRPROXY(ps,"backend connect timeout\n"); //shutdown_proxy(ps, SHUTDOWN_HARD); } /* Upon receiving a signal from OpenSSL that a handshake is required, re-wire * the read/write events to hook up to the handshake handlers */ static void start_handshake(proxystate *ps, int err) { CHECK_OBJ_NOTNULL(ps, PROXYSTATE_MAGIC); ev_io_stop(loop, &ps->ev_r_ssl); ev_io_stop(loop, &ps->ev_w_ssl); ps->handshaked = 0; LOGPROXY(ps,"ssl handshake start\n"); if (err == SSL_ERROR_WANT_READ) ev_io_start(loop, &ps->ev_r_handshake); else if (err == SSL_ERROR_WANT_WRITE) ev_io_start(loop, &ps->ev_w_handshake); ev_timer_start(loop, &ps->ev_t_handshake); } static void get_alpn(proxystate *ps, const unsigned char **selected, unsigned *len) { *selected = NULL; *len = 0; #ifdef OPENSSL_WITH_ALPN SSL_get0_alpn_selected(ps->ssl, selected, len); #endif #ifdef OPENSSL_WITH_NPN if (*len == 0) SSL_get0_next_proto_negotiated(ps->ssl, selected, len); #endif } static int proxy_tlv_append(char *dst, ssize_t dstlen, unsigned type, const char *val, ssize_t len) { if (len == -1) len = strlen(val); if (dstlen < len + 3) return (0); dst[0] = type; dst[1] = (len >> 8) & 0xff; dst[2] = len & 0xff; memcpy(dst + 3, val, len); return (len + 3); } static void write_proxy_v2(proxystate *ps, const struct sockaddr *local) { char *base; int i; struct ha_proxy_v2_hdr *p; union addr { struct sockaddr sa; struct sockaddr_in sa4; struct sockaddr_in6 sa6; } *l, *r; CHECK_OBJ_NOTNULL(ps, PROXYSTATE_MAGIC); base = ringbuffer_write_ptr(&ps->ring_ssl2clear); p = (struct ha_proxy_v2_hdr *)base; size_t len = 16, maxlen; l = (union addr *) local; r = (union addr *) &ps->remote_ip; maxlen = ps->ring_ssl2clear.data_len; memcpy(&p->sig,"\r\n\r\n\0\r\nQUIT\n", 12); p->ver_cmd = 0x21; /* v2|PROXY */ p->fam = l->sa.sa_family == AF_INET ? 0x11 : 0x21; if (l->sa.sa_family == AF_INET) { len += 12; /* src/client */ memcpy(&p->addr.ipv4.src_addr, &r->sa4.sin_addr.s_addr, sizeof p->addr.ipv4.src_addr); memcpy(&p->addr.ipv4.src_port, &r->sa4.sin_port, sizeof p->addr.ipv4.src_port); /* dst/server */ memcpy(&p->addr.ipv4.dst_addr, &l->sa4.sin_addr.s_addr, sizeof p->addr.ipv4.dst_addr); memcpy(&p->addr.ipv4.dst_port, &l->sa4.sin_port, sizeof p->addr.ipv4.dst_port); } else { assert (l->sa.sa_family == AF_INET6); len += 36; /* src/client */ memcpy(&p->addr.ipv6.src_addr, &r->sa6.sin6_addr.s6_addr, sizeof p->addr.ipv6.src_addr); memcpy(&p->addr.ipv6.src_port, &r->sa6.sin6_port, sizeof p->addr.ipv6.src_port); /* dst/server */ memcpy(&p->addr.ipv6.dst_addr, &l->sa6.sin6_addr.s6_addr, sizeof p->addr.ipv6.dst_addr); memcpy(&p->addr.ipv6.dst_port, &l->sa6.sin6_port, sizeof p->addr.ipv6.dst_port); } /* This is where we add something related to NPN or ALPN*/ #if defined(OPENSSL_WITH_ALPN) || defined(OPENSSL_WITH_NPN) #define PP2_TYPE_ALPN 0x01 const char *alpn_tok = NULL; unsigned alpn_len = 0; get_alpn(ps, (const unsigned char **)&alpn_tok, &alpn_len); if (alpn_len > 0) { /* let the server know that a protocol was selected. */ i = proxy_tlv_append(base + len, maxlen - len, PP2_TYPE_ALPN, alpn_tok, alpn_len); AN(i); len += i; } #endif if (CONFIG->PROXY_TLV) { #define PP2_TYPE_SSL 0x20 #define PP2_CLIENT_SSL 0x01 #define PP2_SUBTYPE_SSL_VERSION 0x21 #define PP2_SUBTYPE_SSL_CIPHER 0x23 char *tlvp = base + len; ssize_t sz = 0; const char *tmp; tlvp[0] = PP2_TYPE_SSL; /* tlvp[1..2] to be updated with payload length later */ len += 3; base[len] = PP2_CLIENT_SSL; base[len + 1] = 1; len += 5; sz += 5; tmp = SSL_get_version(ps->ssl); AN(tmp); i = proxy_tlv_append(base + len, maxlen - len, PP2_SUBTYPE_SSL_VERSION, tmp, -1); len += i; sz += i; tmp = SSL_get_cipher_name(ps->ssl); AN(tmp); i = proxy_tlv_append(base + len, maxlen - len, PP2_SUBTYPE_SSL_CIPHER, tmp, -1); len += i; sz += i; tlvp[1] = (sz >> 8) & 0xff; tlvp[2] = sz & 0xff; } p->len = htons(len - 16); ringbuffer_write_append(&ps->ring_ssl2clear, len); } static void write_proxy_v1(proxystate *ps, const struct sockaddr *local, socklen_t slen) { char *p; char src_addr[INET6_ADDRSTRLEN+1], dst_addr[INET6_ADDRSTRLEN+1]; char src_port[8], dst_port[8]; size_t len; int n; CHECK_OBJ_NOTNULL(ps, PROXYSTATE_MAGIC); p = ringbuffer_write_ptr(&ps->ring_ssl2clear); n = getnameinfo(local, slen, dst_addr, sizeof dst_addr, dst_port, sizeof dst_port, NI_NUMERICHOST | NI_NUMERICSERV); AZ(n); n = getnameinfo((struct sockaddr *) &ps->remote_ip, slen, src_addr, sizeof src_addr, src_port, sizeof src_port, NI_NUMERICHOST | NI_NUMERICSERV); AZ(n); if (local->sa_family == AF_INET) { len = sprintf(p, "PROXY TCP4 %s %s %s %s\r\n", src_addr, dst_addr, src_port, dst_port); } else { assert (local->sa_family == AF_INET6); len = sprintf(p, "PROXY TCP6 %s %s %s %s\r\n", src_addr, dst_addr, src_port, dst_port); } assert (len > 0); ringbuffer_write_append(&ps->ring_ssl2clear, len); } static void write_ip_octet(proxystate *ps) { char *ring_pnt; CHECK_OBJ_NOTNULL(ps, PROXYSTATE_MAGIC); ring_pnt = ringbuffer_write_ptr(&ps->ring_ssl2clear); assert(ps->remote_ip.ss_family == AF_INET || ps->remote_ip.ss_family == AF_INET6); *ring_pnt++ = (unsigned char) ps->remote_ip.ss_family; if (ps->remote_ip.ss_family == AF_INET6) { memcpy(ring_pnt, &((struct sockaddr_in6 *) &ps->remote_ip)->sin6_addr.s6_addr, 16U); ringbuffer_write_append(&ps->ring_ssl2clear, 1U + 16U); } else { memcpy(ring_pnt, &((struct sockaddr_in *) &ps->remote_ip)->sin_addr.s_addr, 4U); ringbuffer_write_append(&ps->ring_ssl2clear, 1U + 4U); } } #if defined(OPENSSL_WITH_NPN) || defined(OPENSSL_WITH_ALPN) static int is_protocol_matching(const unsigned char *selected, unsigned len) { int unsigned i = 0; if (CONFIG->ALPN_PROTOS_LV != NULL) { while (i < CONFIG->ALPN_PROTOS_LV_LEN) { if(CONFIG->ALPN_PROTOS_LV[i] == len && 0 == memcmp(selected, CONFIG->ALPN_PROTOS_LV + i + 1, len)) return 1; i+= CONFIG->ALPN_PROTOS_LV[i] + 1; } } return 0; } static int is_alpn_shutdown_needed(proxystate *ps) { const unsigned char *alpn_tok; unsigned alpn_len; if (CONFIG->ALPN_PROTOS_LV == NULL) return 0; get_alpn(ps, &alpn_tok, &alpn_len); if (alpn_len == 0) { /* If alpn / npn was tried, shut down */ if(ps->npn_alpn_tried) { LOGPROXY(ps, "Unsuccessful NPN/ALPN negotiation\n"); return 1; } else LOGPROXY(ps, "No NPN/ALPN negotiation happened.\n"); } else if (!is_protocol_matching(alpn_tok, alpn_len)) { LOGPROXY(ps, "NPN: Unknown protocol selected\n"); return 1; } else LOGPROXY(ps, "NPN/ALPN protocol: %.*s\n", alpn_len, alpn_tok); return 0; } #endif /* After OpenSSL is done with a handshake, re-wire standard read/write handlers * for data transmission */ static void end_handshake(proxystate *ps) { CHECK_OBJ_NOTNULL(ps, PROXYSTATE_MAGIC); ev_io_stop(loop, &ps->ev_r_handshake); ev_io_stop(loop, &ps->ev_w_handshake); ev_timer_stop(loop, &ps->ev_t_handshake); #if defined(OPENSSL_WITH_NPN) || defined(OPENSSL_WITH_ALPN) if (is_alpn_shutdown_needed(ps)) { shutdown_proxy(ps, SHUTDOWN_HARD); return; } #endif LOGPROXY(ps,"ssl end handshake\n"); /* Disable renegotiation (CVE-2009-3555) */ #if OPENSSL_VERSION_NUMBER < 0x10100000L /* For OpenSSL 1.1, setting the following flag does not seem * to be possible. This is OK, since SSLv3 negotiation will * not happen in OpenSSL 0.9.8m or later unless * SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION is set. */ if (ps->ssl->s3) { ps->ssl->s3->flags |= SSL3_FLAGS_NO_RENEGOTIATE_CIPHERS; } #endif ps->handshaked = 1; /* Check if clear side is connected */ if (!ps->clear_connected) { if (CONFIG->WRITE_PROXY_LINE_V1 || CONFIG->WRITE_PROXY_LINE_V2) { struct sockaddr_storage local; socklen_t slen = sizeof local; AZ(getsockname(ps->fd_up, (struct sockaddr *) &local, &slen)); if (CONFIG->WRITE_PROXY_LINE_V1) write_proxy_v1(ps, (struct sockaddr *) &local, slen); else write_proxy_v2(ps, (struct sockaddr *) &local); } else if (CONFIG->WRITE_IP_OCTET) { write_ip_octet(ps); } /* start connect now */ if (0 != start_connect(ps)) return; } else { /* aihttps used in client mode, keep client session ) */ if (!SSL_session_reused(ps->ssl)) { if (client_session) SSL_SESSION_free(client_session); client_session = SSL_get1_session(ps->ssl); } } /* if incoming buffer is not full */ if (!ringbuffer_is_full(&ps->ring_ssl2clear)) safe_enable_io(ps, &ps->ev_r_ssl); /* if outgoing buffer is not empty */ if (!ringbuffer_is_empty(&ps->ring_clear2ssl)) // not safe.. we want to resume stream even during half-closed ev_io_start(loop, &ps->ev_w_ssl); } static void client_proxy_proxy(struct ev_loop *loop, ev_io *w, int revents) { (void)revents; int t; char *proxy = tcp_proxy_line; char *end = tcp_proxy_line + sizeof(tcp_proxy_line); proxystate *ps; BIO *b; CAST_OBJ_NOTNULL(ps, w->data, PROXYSTATE_MAGIC); b = SSL_get_rbio(ps->ssl); // Copy characters one-by-one until we hit a \n or an error while (proxy != end && (t = BIO_read(b, proxy, 1)) == 1) { if (*proxy++ == '\n') break; } if (proxy == end) { LOG("{client} Unexpectedly long PROXY line. Malformed req?"); shutdown_proxy(ps, SHUTDOWN_SSL); } else if (t == 1) { if (ringbuffer_is_full(&ps->ring_ssl2clear)) { LOG("{client} Error writing PROXY line"); shutdown_proxy(ps, SHUTDOWN_SSL); return; } char *ring = ringbuffer_write_ptr(&ps->ring_ssl2clear); memcpy(ring, tcp_proxy_line, proxy - tcp_proxy_line); ringbuffer_write_append(&ps->ring_ssl2clear, proxy - tcp_proxy_line); // Finished reading the PROXY header if (*(proxy - 1) == '\n') { ev_io_stop(loop, &ps->ev_proxy); // Start the real handshake start_handshake(ps, SSL_ERROR_WANT_READ); } } else if (!BIO_should_retry(b)) { LOG("{client} Unexpected error reading PROXY line"); shutdown_proxy(ps, SHUTDOWN_SSL); } } /* The libev I/O handler during the OpenSSL handshake phase. Basically, just * let OpenSSL do what it likes with the socket and obey its requests for reads * or writes */ static void client_handshake(struct ev_loop *loop, ev_io *w, int revents) { (void)revents; int t; const char *errtok; proxystate *ps; int errno_val; CAST_OBJ_NOTNULL(ps, w->data, PROXYSTATE_MAGIC); LOGPROXY(ps,"ssl client handshake revents=%x\n",revents); t = SSL_do_handshake(ps->ssl); if (t == 1) { end_handshake(ps); } else { errno_val = errno; int err = SSL_get_error(ps->ssl, t); switch (err) { #define SSL_ERR(a) \ case a: errtok = #a; break; #include "ssl_err.h" #undef SSL_ERR default: errtok = ""; } LOGPROXY(ps,"ssl client handshake err=%s\n",errtok); if (err == SSL_ERROR_WANT_READ) { ev_io_stop(loop, &ps->ev_w_handshake); ev_io_start(loop, &ps->ev_r_handshake); } else if (err == SSL_ERROR_WANT_WRITE) { ev_io_stop(loop, &ps->ev_r_handshake); ev_io_start(loop, &ps->ev_w_handshake); } else if (err == SSL_ERROR_ZERO_RETURN) { LOG("{%s} Connection closed (in handshake)\n", w->fd == ps->fd_up ? "client" : "backend"); shutdown_proxy(ps, SHUTDOWN_SSL); } else if (err == SSL_ERROR_SYSCALL) { LOG("{%s} SSL socket error in handshake: %s\n", w->fd == ps->fd_up ? "client" : "backend", strerror(errno_val)); shutdown_proxy(ps, SHUTDOWN_SSL); } else { if (err == SSL_ERROR_SSL) { log_ssl_error(ps, "Handshake failure"); } else { LOG("{%s} Unexpected SSL error " "(in handshake): %d\n", w->fd == ps->fd_up ? "client" : "backend", err); } shutdown_proxy(ps, SHUTDOWN_SSL); } } } static void handshake_timeout(EV_P_ ev_timer *w, int revents) { (void)loop; (void)revents; proxystate *ps; CAST_OBJ_NOTNULL(ps, w->data, PROXYSTATE_MAGIC); LOGPROXY(ps,"SSL handshake timeout\n"); shutdown_proxy(ps, SHUTDOWN_HARD); } #define SSLERR(ps, which, log) \ switch (err) { \ case SSL_ERROR_ZERO_RETURN: \ log(ps,"Connection closed by " which "\n"); \ break; \ case SSL_ERROR_SYSCALL: \ if (errno == 0) { \ log(ps,"Connection closed by " which "\n"); \ } else { \ log(ps,"SSL socket error (" which "): %s\n", \ strerror(errno)); \ } \ break; \ default: \ log(ps,"{" which "} Unexpected SSL_read error (" \ which "): %d\n", err); \ } /* Handle a socket error condition passed to us from OpenSSL */ static void handle_fatal_ssl_error(proxystate *ps, int err, int backend) { CHECK_OBJ_NOTNULL(ps, PROXYSTATE_MAGIC); if (backend) { SSLERR(ps, "backend", ERRPROXY); } else { SSLERR(ps, "client", LOGPROXY); } shutdown_proxy(ps, SHUTDOWN_SSL); } /* Read some data from the upstream secure socket via OpenSSL, * and buffer anything we get for writing to the backend */ static void ssl_read(struct ev_loop *loop, ev_io *w, int revents) { (void)revents; int t; proxystate *ps; int ret; CAST_OBJ_NOTNULL(ps, w->data, PROXYSTATE_MAGIC); if (ps->want_shutdown) { ev_io_stop(loop, &ps->ev_r_ssl); return; } if (ringbuffer_is_full(&ps->ring_ssl2clear)) { ERRPROXY(ps, "attempt to read ssl when ring full"); ev_io_stop(loop, &ps->ev_r_ssl); return; } char *buf = ringbuffer_write_ptr(&ps->ring_ssl2clear); t = SSL_read(ps->ssl, buf, ps->ring_ssl2clear.data_len); /* Fix CVE-2009-3555. Disable reneg if started by client. */ if (ps->renegotiation) { shutdown_proxy(ps, SHUTDOWN_SSL); return; } if (t > 0) {//printf("%s\n",buf); #ifdef HAVE_MQTT bytes_stat(ps->remote_ip, t, 0); ret = process_mqtt(buf,t,ps->req); if(REQ_CNT == ret) req_stat(ps->remote_ip, 1, 0); if(DROP == ret || ALERT == ret) { log_mqtt_to_file(ps->remote_ip,ps->req,ret,buf,t); if(DROP == ret) { shutdown_proxy(ps, SHUTDOWN_HARD); return; } } if(1 == ps->req->ddos) { ps->req->ddos = 0; ret = process_cc_ddos(ps->remote_ip,ps->req); if(DROP == ret || ALERT == ret) { log_mqtt_to_file(ps->remote_ip,ps->req,ret,buf,t); shutdown_proxy(ps, SHUTDOWN_HARD); return; } } #else bytes_stat(ps->remote_ip, t, 0); ret = process_http(buf,t,ps->req); if(REQ_CNT == ret) req_stat(ps->remote_ip, 1, 0); if(DROP == ret || ALERT == ret) { log_http_to_file(ps->remote_ip,ps->req,ret,buf,t); if(DROP == ret) { SSL_write(ps->ssl, gvar.denied_url.data, gvar.denied_url.len); shutdown_proxy(ps, SHUTDOWN_HARD); return; } } if(1 == ps->req->ddos) { ps->req->ddos = 0; ret = process_cc_ddos(ps->remote_ip,ps->req); if(DROP == ret || ALERT == ret) { log_http_to_file(ps->remote_ip,ps->req,ret,buf,t); //if(DROP == ret) //block ddos { SSL_write(ps->ssl, gvar.denied_url.data, gvar.denied_url.len); shutdown_proxy(ps, SHUTDOWN_HARD); return; } } } #endif ringbuffer_write_append(&ps->ring_ssl2clear, t); if (ringbuffer_is_full(&ps->ring_ssl2clear)) ev_io_stop(loop, &ps->ev_r_ssl); if (ps->clear_connected) safe_enable_io(ps, &ps->ev_w_clear); } else { int err = SSL_get_error(ps->ssl, t); if (err == SSL_ERROR_WANT_WRITE) { start_handshake(ps, err); } else if (err == SSL_ERROR_WANT_READ) { /* NOOP. Incomplete SSL data */ } else { if (err == SSL_ERROR_SSL) { log_ssl_error(ps, "SSL_read error"); } handle_fatal_ssl_error(ps, err, w->fd == ps->fd_up ? 0 : 1); } } } /* Write some previously-buffered backend data upstream on the * secure socket using OpenSSL */ static void ssl_write(struct ev_loop *loop, ev_io *w, int revents) { (void)revents; int t; int sz; proxystate *ps; CAST_OBJ_NOTNULL(ps, w->data, PROXYSTATE_MAGIC); assert(!ringbuffer_is_empty(&ps->ring_clear2ssl)); char *next = ringbuffer_read_next(&ps->ring_clear2ssl, &sz); t = SSL_write(ps->ssl, next, sz); if (t > 0) { bytes_stat(ps->remote_ip, 0, t); if (t == sz) { ringbuffer_read_pop(&ps->ring_clear2ssl); if (ps->clear_connected) // can be re-enabled b/c we've popped safe_enable_io(ps, &ps->ev_r_clear); if (ringbuffer_is_empty(&ps->ring_clear2ssl)) { if (ps->want_shutdown) { shutdown_proxy(ps, SHUTDOWN_HARD); return; } ev_io_stop(loop, &ps->ev_w_ssl); } } else { ringbuffer_read_skip(&ps->ring_clear2ssl, t); } } else { int err = SSL_get_error(ps->ssl, t); if (err == SSL_ERROR_WANT_READ) { start_handshake(ps, err); } else if (err == SSL_ERROR_WANT_WRITE) { /* NOOP. Incomplete SSL data */ } else { if (err == SSL_ERROR_SSL) { log_ssl_error(ps, "SSL_write error"); } else { LOG("{%s} SSL_write error: %d\n", w->fd == ps->fd_up ? "client" : "backend", err); } handle_fatal_ssl_error(ps, err, w->fd == ps->fd_up ? 0 : 1); } } } /* libev read handler for the bound sockets. Socket is accepted, * the proxystate is allocated and initalized, and we're off the races * connecting to the backend */ static void handle_accept(struct ev_loop *loop, ev_io *w, int revents) { (void)revents; (void)loop; struct sockaddr_storage addr; sslctx *so; struct frontend *fr; proxystate *ps; socklen_t sl = sizeof(addr); #if HAVE_ACCEPT4==1 int client = accept4(w->fd, (struct sockaddr *) &addr, &sl, SOCK_NONBLOCK); #else int client = accept(w->fd, (struct sockaddr *) &addr, &sl); #endif if (client == -1) { switch (errno) { case EMFILE: ERR("{client} accept() failed; " "too many open files for this process\n"); break; case ENFILE: ERR("{client} accept() failed; " "too many open files for this system\n"); break; default: if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN && errno != ENOTTY && errno != ECONNABORTED) { SOCKERR("{client} accept() failed"); } } return; } int flag = 1; int ret = setsockopt(client, IPPROTO_TCP, TCP_NODELAY, (char *)&flag, sizeof(flag) ); if (ret == -1) { SOCKERR("Couldn't setsockopt on client (TCP_NODELAY)"); } #ifdef TCP_CWND int cwnd = 10; ret = setsockopt(client, IPPROTO_TCP, TCP_CWND, &cwnd, sizeof(cwnd)); if (ret == -1) { SOCKERR("Couldn't setsockopt on client (TCP_CWND)"); } #endif #if HAVE_ACCEPT4==0 if (setnonblocking(client) < 0) { SOCKERR("{client} setnonblocking failed"); (void) close(client); return; } #endif if(0 < if_block_connect(addr)) { (void) close(client); return; } settcpkeepalive(client); ALLOC_OBJ(ps, PROXYSTATE_MAGIC); if (ps == NULL) { (void)close(client); ERR("{malloc-err}: %s\n", strerror(errno)); return; } #ifdef HAVE_MQTT ps ->req = calloc(1, sizeof(mqtt_waf_msg)); #else ps ->req = calloc(1, sizeof(http_waf_msg)); #endif if(ps ->req == NULL) { (void)close(client); free(ps); ERR("{malloc-req-err}: %s\n", strerror(errno)); return; } ps->backend = backend_ref(); ps->fd_down = create_back_socket(ps->backend); if (ps->fd_down == -1) { (void) close(client); backend_deref(&ps->backend); free(ps); ERR("{backend-socket}: %s\n", strerror(errno)); return; } CAST_OBJ_NOTNULL(fr, w->data, FRONTEND_MAGIC); if (fr->default_ctx != NULL) CAST_OBJ_NOTNULL(so, fr->default_ctx, SSLCTX_MAGIC); else CAST_OBJ_NOTNULL(so, default_ctx, SSLCTX_MAGIC); SSL *ssl = SSL_new(so->ctx); if (ssl == NULL) { (void)close(ps->fd_down); (void)close(client); backend_deref(&ps->backend); free(ps); ERR("{SSL_new}: %s\n", strerror(errno)); return; } long mode = SSL_MODE_ENABLE_PARTIAL_WRITE; #ifdef SSL_MODE_RELEASE_BUFFERS mode |= SSL_MODE_RELEASE_BUFFERS; #endif SSL_set_mode(ssl, mode); SSL_set_accept_state(ssl); SSL_set_fd(ssl, client); ps->fd_up = client; ps->ssl = ssl; ps->want_shutdown = 0; ps->clear_connected = 0; ps->handshaked = 0; ps->renegotiation = 0; ps->remote_ip = addr; ps->connect_port = 0; ringbuffer_init(&ps->ring_clear2ssl, CONFIG->RING_SLOTS, CONFIG->RING_DATA_LEN); ringbuffer_init(&ps->ring_ssl2clear, CONFIG->RING_SLOTS, CONFIG->RING_DATA_LEN); ps->req->buf = ps->ring_ssl2clear.buf; /* set up events */ ev_io_init(&ps->ev_r_ssl, ssl_read, client, EV_READ); ev_io_init(&ps->ev_w_ssl, ssl_write, client, EV_WRITE); ev_io_init(&ps->ev_r_handshake, client_handshake, client, EV_READ); ev_io_init(&ps->ev_w_handshake, client_handshake, client, EV_WRITE); ev_timer_init(&ps->ev_t_handshake, handshake_timeout, CONFIG->SSL_HANDSHAKE_TIMEOUT, 0.); ev_io_init(&ps->ev_proxy, client_proxy_proxy, client, EV_READ); ev_io_init(&ps->ev_w_connect, handle_connect, ps->fd_down, EV_WRITE); ev_timer_init(&ps->ev_t_connect, connect_timeout, CONFIG->BACKEND_CONNECT_TIMEOUT, 0.); ev_io_init(&ps->ev_w_clear, clear_write, ps->fd_down, EV_WRITE); ev_io_init(&ps->ev_r_clear, clear_read, ps->fd_down, EV_READ); ps->ev_r_ssl.data = ps; ps->ev_w_ssl.data = ps; ps->ev_r_clear.data = ps; ps->ev_w_clear.data = ps; ps->ev_proxy.data = ps; ps->ev_w_connect.data = ps; ps->ev_t_connect.data = ps; ps->ev_r_handshake.data = ps; ps->ev_w_handshake.data = ps; ps->ev_t_handshake.data = ps; /* Link back proxystate to SSL state */ SSL_set_app_data(ssl, ps); n_conns++; LOGPROXY(ps, "proxy connect\n"); if (CONFIG->PROXY_PROXY_LINE) { ev_io_start(loop, &ps->ev_proxy); } else { /* for client-first handshake */ start_handshake(ps, SSL_ERROR_WANT_READ); } } static void check_ppid(struct ev_loop *loop, ev_timer *w, int revents) { struct frontend *fr; struct listen_sock *ls; (void)revents; pid_t ppid = getppid(); if (ppid != master_pid) { ERR("{core} Process %d detected parent death, " "closing listener sockets.\n", core_id); ev_timer_stop(loop, w); VTAILQ_FOREACH(fr, &frontends, list) { CHECK_OBJ_NOTNULL(fr, FRONTEND_MAGIC); VTAILQ_FOREACH(ls, &fr->socks, list) { CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); ev_io_stop(loop, &ls->listener); close(ls->sock); } } } } static const void * Get_Sockaddr(const struct sockaddr *sa, socklen_t *sl) { AN(sa); AN(sl); switch (sa->sa_family) { case PF_INET: *sl = sizeof(struct sockaddr_in); break; case PF_INET6: *sl = sizeof(struct sockaddr_in6); break; case PF_UNIX: *sl = sizeof(struct sockaddr_un); break; default: *sl = 0; return (NULL); } return sa; } static void handle_mgt_rd(struct ev_loop *loop, ev_io *w, int revents) { ssize_t r; struct frontend *fr; struct listen_sock *ls; struct worker_update wu; (void) revents; r = read(w->fd, &wu, sizeof(wu)); if (r == -1) { if (errno == EWOULDBLOCK || errno == EAGAIN) return; LOGL("Error in mgt->worker read operation. " "Restarting process."); /* If something went wrong here, the process will be * left in utter limbo as to whether it should keep * running or not. Kill the process and let the mgt * process start it back up. */ _exit(1); } else if (r == 0) { /* Parent died .. */ _exit(1); } if (wu.type == WORKER_GEN && wu.payload.gen != worker_gen) { /* This means this process has reached its retirement age. */ if (worker_state == WORKER_EXITING) return; worker_state = WORKER_EXITING; /* Stop accepting new connections. */ VTAILQ_FOREACH(fr, &frontends, list) { CHECK_OBJ_NOTNULL(fr, FRONTEND_MAGIC); VTAILQ_FOREACH(ls, &fr->socks, list) { CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); ev_io_stop(loop, &ls->listener); close(ls->sock); } } check_exit_state(); LOGL("Worker %d (gen: %d): State %s\n", core_id, worker_gen, (worker_state == WORKER_EXITING) ? "EXITING" : "ACTIVE"); } else if (wu.type == WORKER_GEN && wu.payload.gen == worker_gen) { return; } else if (wu.type == BACKEND_REFRESH) { struct backend *b; b = backend_create((struct sockaddr *)&wu.payload.addr); backend_deref(&backaddr); backaddr = b; AN(VSA_Sane(backaddr->backaddr)); } else WRONG("Invalid worker update state"); } static void handle_clear_accept(struct ev_loop *loop, ev_io *w, int revents) { (void)revents; (void)loop; struct sockaddr_storage addr; struct frontend *fr; sslctx *so; proxystate *ps; socklen_t sl = sizeof(addr); int client = accept(w->fd, (struct sockaddr *) &addr, &sl); if (client == -1) { switch (errno) { case EMFILE: ERR("{client} accept() failed; " "too many open files for this process\n"); break; case ENFILE: ERR("{client} accept() failed; " "too many open files for this system\n"); break; default: if (errno != EINTR && errno != EWOULDBLOCK && errno != EAGAIN && errno != ECONNABORTED) { SOCKERR("{client} accept() failed"); } break; } return; } int flag = 1; int ret = setsockopt(client, IPPROTO_TCP, TCP_NODELAY, (char *)&flag, sizeof(flag) ); if (ret == -1) { ERR("Couldn't setsockopt on client (TCP_NODELAY): %s\n", strerror(errno)); } #ifdef TCP_CWND int cwnd = 10; ret = setsockopt(client, IPPROTO_TCP, TCP_CWND, &cwnd, sizeof(cwnd)); if (ret == -1) { ERR("Couldn't setsockopt on client (TCP_CWND): %s\n", strerror(errno)); } #endif if (setnonblocking(client)) { SOCKERR("{client} setnonblocking failed"); (void) close(client); return; } settcpkeepalive(client); ALLOC_OBJ(ps, PROXYSTATE_MAGIC); ps->backend = backend_ref(); ps->fd_down = create_back_socket(ps->backend); if (ps->fd_down == -1) { backend_deref(&ps->backend); close(client); free(ps); ERR("{backend-socket}: %s\n", strerror(errno)); return; } CAST_OBJ_NOTNULL(fr, w->data, FRONTEND_MAGIC); if (fr->default_ctx != NULL) CAST_OBJ_NOTNULL(so, fr->default_ctx, SSLCTX_MAGIC); else CAST_OBJ_NOTNULL(so, default_ctx, SSLCTX_MAGIC); SSL *ssl = SSL_new(so->ctx); long mode = SSL_MODE_ENABLE_PARTIAL_WRITE; #ifdef SSL_MODE_RELEASE_BUFFERS mode |= SSL_MODE_RELEASE_BUFFERS; #endif SSL_set_mode(ssl, mode); SSL_set_connect_state(ssl); SSL_set_fd(ssl, ps->fd_down); if (client_session) SSL_set_session(ssl, client_session); ps->fd_up = client; ps->ssl = ssl; ps->want_shutdown = 0; ps->clear_connected = 1; ps->handshaked = 0; ps->renegotiation = 0; ps->remote_ip = addr; ringbuffer_init(&ps->ring_clear2ssl, CONFIG->RING_SLOTS, CONFIG->RING_DATA_LEN); ringbuffer_init(&ps->ring_ssl2clear, CONFIG->RING_SLOTS, CONFIG->RING_DATA_LEN); /* set up events */ ev_io_init(&ps->ev_r_clear, clear_read, client, EV_READ); ev_io_init(&ps->ev_w_clear, clear_write, client, EV_WRITE); ev_io_init(&ps->ev_w_connect, handle_connect, ps->fd_down, EV_WRITE); ev_timer_init(&ps->ev_t_connect, connect_timeout, CONFIG->BACKEND_CONNECT_TIMEOUT, 0.); ev_io_init(&ps->ev_r_handshake, client_handshake, ps->fd_down, EV_READ); ev_io_init(&ps->ev_w_handshake, client_handshake, ps->fd_down, EV_WRITE); ev_timer_init(&ps->ev_t_handshake, handshake_timeout, CONFIG->SSL_HANDSHAKE_TIMEOUT, 0.); ev_io_init(&ps->ev_w_ssl, ssl_write, ps->fd_down, EV_WRITE); ev_io_init(&ps->ev_r_ssl, ssl_read, ps->fd_down, EV_READ); ps->ev_r_ssl.data = ps; ps->ev_w_ssl.data = ps; ps->ev_r_clear.data = ps; ps->ev_w_clear.data = ps; ps->ev_w_connect.data = ps; ps->ev_r_handshake.data = ps; ps->ev_w_handshake.data = ps; ps->ev_t_handshake.data = ps; /* Link back proxystate to SSL state */ SSL_set_app_data(ssl, ps); n_conns++; ev_io_start(loop, &ps->ev_r_clear); start_connect(ps); /* start connect */ } /* Set up the child (worker) process including libev event loop, read event * on the bound sockets, etc */ static void handle_connections(int mgt_fd) { struct frontend *fr; sslctx *sc, *sctmp; struct listen_sock *ls; struct sigaction sa; worker_state = WORKER_ACTIVE; LOGL("{core} Process %d online\n", core_id); /* child cannot create new children... */ create_workers = 0; /* nor can they handle SIGHUP */ sa.sa_flags = 0; sa.sa_handler = SIG_IGN; sigemptyset(&sa.sa_mask); AZ(sigaction(SIGHUP, &sa, NULL)); #if defined(CPU_ZERO) && defined(CPU_SET) cpu_set_t cpus; CPU_ZERO(&cpus); CPU_SET(core_id, &cpus); int res = sched_setaffinity(0, sizeof(cpus), &cpus); if (!res) LOG("{core} Successfully attached to CPU #%d\n", core_id); else ERR("{core-warning} Unable to attach to CPU #%d; " "do you have that many cores?\n", core_id); #endif loop = ev_default_loop(EVFLAG_AUTO); ev_timer timer_ppid_check; ev_timer_init(&timer_ppid_check, check_ppid, 1.0, 1.0); ev_timer_start(loop, &timer_ppid_check); VTAILQ_FOREACH(fr, &frontends, list) { VTAILQ_FOREACH(ls, &fr->socks, list) { ev_io_init(&ls->listener, (CONFIG->PMODE == SSL_CLIENT) ? handle_clear_accept : handle_accept, ls->sock, EV_READ); ls->listener.data = fr; ev_io_start(loop, &ls->listener); } } if (CONFIG->OCSP_DIR != NULL) { HASH_ITER(hh, ssl_ctxs, sc, sctmp) { if (sc->ev_staple) ev_stat_start(loop, sc->ev_staple); } VTAILQ_FOREACH(fr, &frontends, list) { HASH_ITER(hh, fr->ssl_ctxs, sc, sctmp) { if (sc->ev_staple) ev_stat_start(loop, sc->ev_staple); } } if (default_ctx != NULL && default_ctx->ev_staple != NULL) ev_stat_start(loop, default_ctx->ev_staple); } AZ(setnonblocking(mgt_fd)); ev_io_init(&mgt_rd, handle_mgt_rd, mgt_fd, EV_READ); ev_io_start(loop, &mgt_rd); ev_loop(loop, 0); ERR("Worker %d (gen: %d) exiting.\n", core_id, worker_gen); _exit(1); } /* OCSP requestor process. */ static void handle_ocsp_task(void) { struct frontend *fr; struct listen_sock *ls; sslctx *sc, *sctmp; ev_timer timer_ppid_check; /* we don't accept incoming connections for this process. */ VTAILQ_FOREACH(fr, &frontends, list) { CHECK_OBJ_NOTNULL(fr, FRONTEND_MAGIC); VTAILQ_FOREACH(ls, &fr->socks, list) { CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); ev_io_stop(loop, &ls->listener); close(ls->sock); } } (void) umask(027); loop = ev_default_loop(EVFLAG_AUTO); /* Create ocspquery work items for any eligible ocsp queries */ HASH_ITER(hh, ssl_ctxs, sc, sctmp) { HOCSP_mktask(sc, NULL, -1.0); } VTAILQ_FOREACH(fr, &frontends, list) { HASH_ITER(hh, fr->ssl_ctxs, sc, sctmp) { HOCSP_mktask(sc, NULL, -1.0); } } if (default_ctx != NULL) HOCSP_mktask(default_ctx, NULL, -1.0); ev_timer_init(&timer_ppid_check, check_ppid, 1.0, 1.0); ev_timer_start(loop, &timer_ppid_check); ev_loop(loop, 0); _exit(0); } void change_root() { if (chroot(CONFIG->CHROOT) == -1) fail("chroot"); if (chdir("/")) fail("chdir"); } void drop_privileges(void) { if (geteuid() == 0) { if (CONFIG->UID >= 0) AZ(setgroups(0, NULL)); if (CONFIG->GID >= 0) AZ(setgid(CONFIG->GID)); if (CONFIG->UID >= 0) AZ(setuid(CONFIG->UID)); } else { LOG("{core} Not running as root, no priv-sep\n"); } /* On Linux >= 2.4, you need to set the dumpable flag to get core dumps after you have done a setuid. */ #ifdef __linux__ if (prctl(PR_SET_DUMPABLE, 1) != 0) LOG("Could not set dumpable bit. Core dumps turned off\n"); #endif } static int backaddr_init_uds(void) { struct sockaddr_un sun; struct backend *b; int l; if (backaddr != NULL) /* Already configured - we don't refresh UDS addresses. */ return (0); AN(CONFIG->BACK_PATH); memset(&sun, 0, sizeof sun); sun.sun_family = PF_UNIX; l = snprintf(sun.sun_path, sizeof(sun.sun_path), "%s", CONFIG->BACK_PATH); /* the length of BACK_PATH is verified to fit into * sun.sun_path in configuration.c */ assert(l < (int)sizeof(sun.sun_path)); b = backend_create((struct sockaddr *)&sun); backaddr = b; return (0); } static int backaddr_init(void) { struct addrinfo *result; struct addrinfo hints; struct backend *b; if (CONFIG->BACK_PATH) return (backaddr_init_uds()); memset(&hints, 0, sizeof hints); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = 0; const int gai_err = getaddrinfo(CONFIG->BACK_IP, CONFIG->BACK_PORT, &hints, &result); if (gai_err != 0) { ERR("{getaddrinfo-backend}: %s\n", gai_strerror(gai_err)); exit(1); } b = backend_create(result->ai_addr); freeaddrinfo(result); if (backaddr == NULL) { backaddr = b; return (1); } if (VSA_Compare(backaddr->backaddr, b->backaddr) != 0) { free(backaddr); backaddr = b; return (1); } backend_deref(&b); AZ(b); return (0); } void init_globals(void) { struct addrinfo hints; VTAILQ_INIT(&frontends); VTAILQ_INIT(&worker_procs); backaddr_init(); (void)hints; #ifdef USE_SHARED_CACHE if (CONFIG->SHARED_CACHE) { /* cache update peers addresses */ shcupd_peer_opt *spo = CONFIG->SHCUPD_PEERS; struct addrinfo **pai = shcupd_peers; while (spo->ip) { memset(&hints, 0, sizeof hints); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_DGRAM; hints.ai_flags = 0; const int gai_err = getaddrinfo(spo->ip, spo->port ? spo->port : CONFIG->SHCUPD_PORT, &hints, pai); if (gai_err != 0) { ERR("{getaddrinfo}: %s\n", gai_strerror(gai_err)); exit(1); } spo++; pai++; } } #endif if (CONFIG->SYSLOG) openlog("aihttps", LOG_CONS | LOG_PID | LOG_NDELAY, CONFIG->SYSLOG_FACILITY); } /* Forks COUNT children starting with START_INDEX. We keep a struct * child_proc per child so the parent can manage it later. */ void start_workers(int start_index, int count) { struct worker_proc *c; int pfd[2]; /* don't do anything if we're not allowed to create new workers */ if (!create_workers) return; for (core_id = start_index; core_id < start_index + count; core_id++) { ALLOC_OBJ(c, WORKER_PROC_MAGIC); AZ(pipe(pfd)); c->pfd = pfd[1]; c->gen = worker_gen; c->pid = fork(); c->core_id = core_id; if (c->pid == -1) { ERR("{core} fork() failed: %s; Goodbye cruel world!\n", strerror(errno)); exit(1); } else if (c->pid == 0) { /* child */ close(pfd[1]); FREE_OBJ(c); if (CONFIG->CHROOT && CONFIG->CHROOT[0]) change_root(); if (CONFIG->UID >= 0 || CONFIG->GID >= 0) drop_privileges(); if (geteuid() == 0) { ERR("{core} ERROR: " "Refusing to run workers as root.\n"); _exit(1); } handle_connections(pfd[0]); exit(0); } else { /* parent. Track new child. */ close(pfd[0]); VTAILQ_INSERT_TAIL(&worker_procs, c, list); } } } void start_ocsp_proc(void) { ocsp_proc_pid = fork(); if (ocsp_proc_pid == -1) { ERR("{core}: fork() failed: %s: Exiting.\n", strerror(errno)); exit(1); } else if (ocsp_proc_pid == 0) { if (CONFIG->UID >= 0 || CONFIG->GID >= 0) drop_privileges(); if (geteuid() == 0) { ERR("{core} ERROR: " "Refusing to run workers as root.\n"); _exit(1); } handle_ocsp_task(); } /* child proc should never return. */ AN(ocsp_proc_pid); } /* Forks a new child to replace the old, dead, one with the given PID.*/ void replace_child_with_pid(pid_t pid) { struct worker_proc *c, *cp; /* find old child's slot and put a new child there */ VTAILQ_FOREACH_SAFE(c, &worker_procs, list, cp) { if (c->pid == pid) { VTAILQ_REMOVE(&worker_procs, c, list); /* Only replace if it matches current generation. */ if (c->gen == worker_gen) start_workers(c->core_id, 1); FREE_OBJ(c); return; } } ERR("Cannot find index for child pid %d", pid); } /* Manage status changes in child processes */ static void do_wait(void) { struct worker_proc *c, *ctmp; int status; int pid; #define WAIT_PID(p, action) do { \ pid = waitpid(p, &status, WNOHANG); \ if (pid == 0) { \ /* child has not exited */ \ break; \ } \ else if (pid == -1) { \ if (errno == EINTR) \ ERR("{core} Interrupted waitpid\n"); \ else \ fail("waitpid"); \ } else { \ if (WIFEXITED(status)) { \ ERR("{core} Child %d exited with status %d.\n", \ pid, WEXITSTATUS(status)); \ action; \ } else if (WIFSIGNALED(status)) { \ ERR("{core} Child %d was terminated by " \ "signal %d.\n", pid, WTERMSIG(status)); \ action; \ } \ } \ } while (0) VTAILQ_FOREACH_SAFE(c, &worker_procs, list, ctmp) { WAIT_PID(c->pid, replace_child_with_pid(pid)); } /* also check if the ocsp worker killed itself */ if (ocsp_proc_pid != 0) WAIT_PID(ocsp_proc_pid, if (CONFIG->OCSP_DIR) { start_ocsp_proc(); } else { ocsp_proc_pid = 0; }); } static void sigchld_handler(int signum) { assert(signum == SIGCHLD); n_sigchld++; } static void sigh_terminate (int __attribute__ ((unused)) signo) { struct worker_proc *c; /* don't create any more children */ create_workers = 0; /* are we the master? */ if (getpid() == master_pid) { LOGL("{core} Received signal %d, shutting down.\n", signo); /* kill all children */ VTAILQ_FOREACH(c, &worker_procs, list) { /* LOG("Stopping worker pid %d.\n", c->pid); */ if (c->pid > 1 && kill(c->pid, SIGTERM) != 0) { ERR("{core} Unable to send SIGTERM to worker " "pid %d: %s\n", c->pid, strerror(errno)); } } if (ocsp_proc_pid != 0) kill(ocsp_proc_pid, SIGTERM); } /* this is it, we're done... */ exit(0); } static void sighup_handler(int signum) { assert(signum == SIGHUP); n_sighup++; } static void init_signals() { struct sigaction act; sigemptyset(&act.sa_mask); act.sa_flags = 0; act.sa_handler = SIG_IGN; /* Avoid getting PIPE signal when writing to a closed file descriptor */ if (sigaction(SIGPIPE, &act, NULL) < 0) fail("sigaction - sigpipe"); /* We don't care if someone stops and starts a child process * with kill (1) */ act.sa_flags = SA_NOCLDSTOP; act.sa_handler = sigchld_handler; /* We do care when child processes change status */ if (sigaction(SIGCHLD, &act, NULL) < 0) fail("sigaction - sigchld"); /* catch INT and TERM signals */ act.sa_flags = 0; act.sa_handler = sigh_terminate; if (sigaction(SIGINT, &act, NULL) < 0) { ERR("Unable to register SIGINT signal handler: %s\n", strerror(errno)); exit(1); } if (sigaction(SIGTERM, &act, NULL) < 0) { ERR("Unable to register SIGTERM signal handler: %s\n", strerror(errno)); exit(1); } act.sa_handler = sighup_handler; if (sigaction(SIGHUP, &act, NULL) != 0) { ERR("Unable to register SIGHUP signal handler: %s\n", strerror(errno)); exit(1); } } #define NULL_DEV "/dev/null" static void daemonize() { /* logging.c */ if (logfile == stdout || logfile == stderr) { logfile = NULL; } /* go to root directory */ if (chdir("/") != 0) { ERR("Unable change directory to /: %s\n", strerror(errno)); exit(1); } /* let's make some children, baby :) */ pid_t pid = fork(); if (pid < 0) { ERR("Unable to daemonize: fork failed: %s\n", strerror(errno)); exit(1); } /* am i the parent? */ if (pid != 0) { LOGL("{core} Daemonized as pid %d.\n", pid); exit(0); } /* reopen standard streams to null device */ if (freopen(NULL_DEV, "r", stdin) == NULL) { ERR("Unable to reopen stdin to %s: %s\n", NULL_DEV, strerror(errno)); exit(1); } if (freopen(NULL_DEV, "w", stdout) == NULL) { ERR("Unable to reopen stdout to %s: %s\n", NULL_DEV, strerror(errno)); exit(1); } if (freopen(NULL_DEV, "w", stderr) == NULL) { ERR("Unable to reopen stderr to %s: %s\n", NULL_DEV, strerror(errno)); exit(1); } /* this is child, the new master */ pid_t s = setsid(); if (s < 0) { ERR("Unable to create new session, setsid(2) failed: " "%s :: %d\n", strerror(errno), s); exit(1); } LOG("Successfully daemonized as pid %d.\n", getpid()); } static void openssl_check_version() { /* detect OpenSSL version in runtime */ long openssl_version = SSLeay(); /* check if we're running the same openssl that we were */ /* compiled with */ if ((openssl_version ^ OPENSSL_VERSION_NUMBER) & ~0xff0L) { ERR( "WARNING: {core} OpenSSL version mismatch; " "aihttps was compiled with %lx, now using %lx.\n", (unsigned long int)OPENSSL_VERSION_NUMBER, (unsigned long int)openssl_version ); /* now what? exit now? */ /* exit(1); */ } LOG("{core} Using OpenSSL version %lx.\n", (unsigned long int)openssl_version); } static void remove_pfh(void) { if (pfh && master_pid == getpid()) { VPF_Remove(pfh); } } struct cfg_tpc_obj; enum cfg_tpc_type { CFG_FRONTEND, CFG_CERT /* ... */ }; /* Commit/rollback handling: - KEEP: - commit: reuse - rollback: do nothing - NEW: - commit: use as new - rollback: drop - DROP: - commit: drop - rollback: do nothing */ enum cfg_tpc_handling { CFG_TPC_KEEP, CFG_TPC_NEW, CFG_TPC_DROP }; typedef void cfg_tpc_rollback_f(struct cfg_tpc_obj *o); typedef void cfg_tpc_commit_f(struct cfg_tpc_obj *o); struct cfg_tpc_obj { unsigned magic; #define CFG_TPC_OBJ_MAGIC 0xd6953e5f enum cfg_tpc_type type; enum cfg_tpc_handling handling; void *p[2]; cfg_tpc_rollback_f *rollback; cfg_tpc_commit_f *commit; VTAILQ_ENTRY(cfg_tpc_obj) list; }; VTAILQ_HEAD(cfg_tpc_obj_head, cfg_tpc_obj); static struct cfg_tpc_obj * make_cfg_obj(enum cfg_tpc_type type, enum cfg_tpc_handling handling, void *priv0, void *priv1, cfg_tpc_rollback_f *rollback, cfg_tpc_commit_f *commit) { struct cfg_tpc_obj *o; ALLOC_OBJ(o, CFG_TPC_OBJ_MAGIC); AN(o); o->type = type; o->handling = handling; o->p[0] = priv0; o->p[1] = priv1; o->rollback = rollback; o->commit = commit; return (o); } static void frontend_rollback(struct cfg_tpc_obj *o) { struct frontend *fr; if (o->handling == CFG_TPC_NEW) { CAST_OBJ_NOTNULL(fr, o->p[0], FRONTEND_MAGIC); destroy_frontend(fr); } /* KEEP/DROP: ignore */ } static void frontend_commit(struct cfg_tpc_obj *o) { struct frontend *fr; sslctx *sc; CAST_OBJ_NOTNULL(fr, o->p[0], FRONTEND_MAGIC); switch (o->handling) { case CFG_TPC_NEW: VTAILQ_INSERT_TAIL(&frontends, fr, list); /* FALL-THROUGH */ case CFG_TPC_KEEP: if (o->p[1]) { CAST_OBJ_NOTNULL(sc, o->p[1], SSLCTX_MAGIC); /* fr->sctx = sc; */ } break; case CFG_TPC_DROP: VTAILQ_REMOVE(&frontends, fr, list); destroy_frontend(fr); break; } } static void cert_rollback(struct cfg_tpc_obj *o); static void cert_commit(struct cfg_tpc_obj *o); // Is this incorrectly named? static int ocsp_cfg_changed(const struct cfg_cert_file *cf, const sslctx *sc) { if (sc->staple != NULL && cf->ocspfn == NULL) return (1); /* Dropped OCSP definition */ if (sc->staple == NULL && cf->ocspfn != NULL) return (1); /* Added OCSP definition */ if (sc->staple != NULL && cf->ocspfn != NULL) { if (strcmp(sc->staple_fn, cf->ocspfn) != 0 || sc->staple->mtim < cf->ocsp_mtim) return (1); /* Updated */ } return (0); } /* Query frontend-specific certificates. */ static int cert_fr_query(struct frontend *fr, struct front_arg *fa, struct cfg_tpc_obj_head *cfg_objs) { struct cfg_cert_file *cf, *cftmp; sslctx *sc, *sctmp; struct cfg_tpc_obj *o; HASH_ITER(hh, fr->ssl_ctxs, sc, sctmp) { HASH_FIND_STR(fa->certs, sc->filename, cf); if (cf != NULL && cf->mtim <= sc->mtim && !ocsp_cfg_changed(cf, sc)) { cf->mark = 1; } else { o = make_cfg_obj(CFG_CERT, CFG_TPC_DROP, sc, fr, cert_rollback, cert_commit); VTAILQ_INSERT_TAIL(cfg_objs, o, list); } } HASH_ITER(hh, fa->certs, cf, cftmp) { if (cf->mark) continue; sc = make_ctx_fr(cf, fr, fa); if (sc == NULL) return (-1); o = make_cfg_obj(CFG_CERT, CFG_TPC_NEW, sc, fr, cert_rollback, cert_commit); VTAILQ_INSERT_TAIL(cfg_objs, o, list); } return (0); } /* Query reload of listen sockets. Returns -1 on failure. Failure: Caller calls .rollback() on the objects added in cfg_objs. Success: Caller calls .commit() */ static int frontend_query(struct front_arg *new_set, struct cfg_tpc_obj_head *cfg_objs) { struct frontend *fr; struct front_arg *fa, *ftmp; struct cfg_tpc_obj *o; VTAILQ_FOREACH(fr, &frontends, list) { HASH_FIND_STR(new_set, fr->pspec, fa); if (fa != NULL) { fa->mark = 1; o = make_cfg_obj(CFG_FRONTEND, CFG_TPC_KEEP, fr, NULL, frontend_rollback, frontend_commit); if(cert_fr_query(fr, fa, cfg_objs) < 0) { FREE_OBJ(o); return (-1); } } else o = make_cfg_obj(CFG_FRONTEND, CFG_TPC_DROP, fr, NULL, frontend_rollback, frontend_commit); VTAILQ_INSERT_TAIL(cfg_objs, o, list); } HASH_ITER(hh, new_set, fa, ftmp) { if (!fa->mark) { fr = create_frontend(fa); if (fr == NULL) return (-1); o = make_cfg_obj(CFG_FRONTEND, CFG_TPC_NEW, fr, NULL, frontend_rollback, frontend_commit); VTAILQ_INSERT_TAIL(cfg_objs, o, list); } } return (0); } static void cert_rollback(struct cfg_tpc_obj *o) { sslctx *sc; if (o->handling != CFG_TPC_NEW) return; CAST_OBJ_NOTNULL(sc, o->p[0], SSLCTX_MAGIC); sctx_free(sc, NULL); } static void cert_commit(struct cfg_tpc_obj *o) { sslctx *sc; sni_name **sn_tab; sslctx **ctxs; struct frontend *fr; sn_tab = &sni_names; ctxs = &ssl_ctxs; CAST_OBJ_NOTNULL(sc, o->p[0], SSLCTX_MAGIC); if (o->p[1] != NULL) { CAST_OBJ_NOTNULL(fr, o->p[1], FRONTEND_MAGIC); sn_tab = &fr->sni_names; ctxs = &fr->ssl_ctxs; } switch (o->handling) { case CFG_TPC_NEW: HASH_ADD_KEYPTR(hh, *ctxs, sc->filename, strlen(sc->filename), sc); insert_sni_names(sc, sn_tab); break; case CFG_TPC_KEEP: WRONG("unreachable"); break; case CFG_TPC_DROP: HASH_DEL(*ctxs, sc); sctx_free(sc, sn_tab); break; } } static void dcert_rollback(struct cfg_tpc_obj *o) { cert_rollback(o); } static void dcert_commit(struct cfg_tpc_obj *o) { sslctx *sc; CAST_OBJ_NOTNULL(sc, o->p[0], SSLCTX_MAGIC); CHECK_OBJ_NOTNULL(sni_names, SNI_NAME_MAGIC); switch (o->handling) { case CFG_TPC_NEW: sctx_free(default_ctx, &sni_names); default_ctx = sc; insert_sni_names(sc, &sni_names); break; case CFG_TPC_KEEP: /* FALL-THROUGH */ case CFG_TPC_DROP: /* We always have a default cert. This should not * happen. */ WRONG("unreachable"); break; } } /* Query reload of certificate files */ static int cert_query(aihttps_config *cfg, struct cfg_tpc_obj_head *cfg_objs) { struct cfg_cert_file *cf, *cftmp; sslctx *sc, *sctmp; struct cfg_tpc_obj *o; /* NB: The ordering here is significant. It is imperative that * all DROP objects are inserted before any NEW objects, in * order to not wreak havoc in cert_commit(). */ HASH_ITER(hh, ssl_ctxs, sc, sctmp) { HASH_FIND_STR(cfg->CERT_FILES, sc->filename, cf); if (cf != NULL && cf->mtim <= sc->mtim && !ocsp_cfg_changed(cf, sc)) { cf->mark = 1; } else { o = make_cfg_obj(CFG_CERT, CFG_TPC_DROP, sc, NULL, cert_rollback, cert_commit); VTAILQ_INSERT_TAIL(cfg_objs, o, list); } } /* handle default cert. Default cert has its own * rollback/commit functions. */ if (cfg->CERT_DEFAULT != NULL) { cf = cfg->CERT_DEFAULT; CHECK_OBJ_NOTNULL(default_ctx, SSLCTX_MAGIC); if (strcmp(default_ctx->filename, cf->filename) != 0 || cf->mtim > default_ctx->mtim || ocsp_cfg_changed(cf, default_ctx)) { sc = make_ctx(cf); if (sc == NULL) return (-1); o = make_cfg_obj(CFG_CERT, CFG_TPC_NEW, sc, NULL, dcert_rollback, dcert_commit); VTAILQ_INSERT_TAIL(cfg_objs, o, list); } } HASH_ITER(hh, cfg->CERT_FILES, cf, cftmp) { if (cf->mark) continue; sc = make_ctx(cf); if (sc == NULL) return (-1); o = make_cfg_obj(CFG_CERT, CFG_TPC_NEW, sc, NULL, cert_rollback, cert_commit); VTAILQ_INSERT_TAIL(cfg_objs, o, list); } return (0); } static void notify_workers(struct worker_update *wu) { struct worker_proc *c; int i; VTAILQ_FOREACH(c, &worker_procs, list) { if ((wu->type == WORKER_GEN && wu->payload.gen != c->gen) || (wu->type == BACKEND_REFRESH)) { errno = 0; do { i = write(c->pfd, (void*)wu, sizeof(*wu)); if (i == -1 && errno != EINTR) { if (wu->type == WORKER_GEN) ERR("WARNING: {core} Unable to " "gracefully reload worker %d" " (%s).\n", c->pid, strerror(errno)); else ERR("WARNING: {core} Unable to " "notify worker %d " "with changed backend address (%s).\n", c->pid, strerror(errno)); (void)kill(c->pid, SIGTERM); break; } } while (i == -1 && errno == EINTR); if (wu->type == WORKER_GEN) (void)close(c->pfd); } } } static void reconfigure(int argc, char **argv) { aihttps_config *cfg_new; struct cfg_tpc_obj_head cfg_objs; struct cfg_tpc_obj *cto, *cto_tmp; struct timeval tv; double t0, t1; struct worker_update wu; struct frontend *fr; LOGL("Received SIGHUP: Initiating configuration reload.\n"); AZ(gettimeofday(&tv, NULL)); t0 = tv.tv_sec + 1e-6 * tv.tv_usec; VTAILQ_INIT(&cfg_objs); cfg_new = config_new(); AN(cfg_new); if (config_parse_cli(argc, argv, cfg_new) != 0) { ERR("Config reload failed: %s\n", config_error_get()); config_destroy(cfg_new); return; } /* NB: the ordering of the foo_query() calls here is * significant. */ if (frontend_query(cfg_new->LISTEN_ARGS, &cfg_objs) < 0 || cert_query(cfg_new, &cfg_objs) < 0) { VTAILQ_FOREACH_SAFE(cto, &cfg_objs, list, cto_tmp) { VTAILQ_REMOVE(&cfg_objs, cto, list); AN(cto->rollback); cto->rollback(cto); FREE_OBJ(cto); } ERR("{core} Config reload failed.\n"); return; } else { VTAILQ_FOREACH_SAFE(cto, &cfg_objs, list, cto_tmp) { VTAILQ_REMOVE(&cfg_objs, cto, list); AN(cto->commit); cto->commit(cto); FREE_OBJ(cto); } } /* Rewire default sslctx for each frontend after a reload */ VTAILQ_FOREACH(fr, &frontends, list) { struct front_arg *fa; struct cfg_cert_file *cf; sslctx *sc; if (HASH_COUNT(fr->ssl_ctxs) == 0) { fr->default_ctx = NULL; continue; } HASH_FIND_STR(cfg_new->LISTEN_ARGS, fr->pspec, fa); AN(fa); cf = fa->certs; CHECK_OBJ_NOTNULL(cf, CFG_CERT_FILE_MAGIC); while (cf->hh.next != NULL) cf = cf->hh.next; HASH_FIND_STR(fr->ssl_ctxs, cf->filename, sc); CHECK_OBJ_NOTNULL(sc, SSLCTX_MAGIC); fr->default_ctx = sc; } AZ(gettimeofday(&tv, NULL)); t1 = tv.tv_sec + 1e-6 * tv.tv_usec; LOGL("{core} Config reloaded in %.2lf seconds. " "Starting new child processes.\n", t1 - t0); config_destroy(CONFIG); CONFIG = cfg_new; worker_gen++; start_workers(0, CONFIG->NCORES); wu.type = WORKER_GEN; wu.payload.gen = worker_gen; notify_workers(&wu); if (ocsp_proc_pid > 0) { (void) kill(ocsp_proc_pid, SIGTERM); /* * Restarting the OCSP process is taken * care of in do_wait */ } else if (CONFIG->OCSP_DIR != NULL && ocsp_proc_pid <= 0) { start_ocsp_proc(); } } void sleep_and_refresh(aihttps_config *CONFIG) { /* static backend address */ if (!CONFIG->BACKEND_REFRESH_TIME) { pause(); return; } int rv = 0; while (1) { rv = usleep(CONFIG->BACKEND_REFRESH_TIME*1000000); if (rv == -1 && errno == EINTR) break; else if(backaddr_init()) { struct worker_update wu; wu.type = BACKEND_REFRESH; socklen_t len; const void *addr = VSA_Get_Sockaddr(backaddr->backaddr, &len); AN(addr); memcpy(&(wu.payload.addr), addr, len); notify_workers(&wu); } } } /* Process command line args, create the bound socket, * spawn child (worker) processes, and respawn if any die */ int main(int argc, char **argv) { // initialize configuration struct front_arg *fa, *ftmp; init_rules(); CONFIG = config_new(); // parse command line if (config_parse_cli(argc, argv, CONFIG) != 0) { fprintf(stderr, "%s\n", config_error_get()); return (1); } if (CONFIG->TEST) { fprintf(stderr, "Trying to initialize SSL contexts with your" " certificates\n"); init_globals(); init_openssl(); init_certs(); fprintf(stderr, "%s configuration looks ok.\n", basename(argv[0])); return (0); } if (CONFIG->LOG_FILENAME) { FILE* f; if ((f = fopen(CONFIG->LOG_FILENAME, "a")) == NULL) { /* logging.c */ logfile = stderr; ERR("FATAL: Unable to open log file: %s: %s\n", CONFIG->LOG_FILENAME, strerror(errno)); exit(2); } logfile = f; if (CONFIG->UID >=0 || CONFIG->GID >= 0) { AZ(fchown(fileno(logfile), CONFIG->UID, CONFIG->GID)); } AZ(fstat(fileno(logfile), &logf_st)); logf_check_t = time(NULL); } else { logfile = stderr; } AZ(setvbuf(logfile, NULL, _IONBF, BUFSIZ)); if (CONFIG->DAEMONIZE && (logfile == stdout || logfile == stderr)) logfile = NULL; LOGL("{core} %s starting\n", PACKAGE_STRING); create_workers = 1; openssl_check_version(); init_signals(); init_globals(); init_openssl(); HASH_ITER(hh, CONFIG->LISTEN_ARGS, fa, ftmp) { struct frontend *fr = create_frontend(fa); if (fr == NULL) exit(1); VTAILQ_INSERT_TAIL(&frontends, fr, list); } /* load certificates, pass to handle_connections */ LOGL("{core} Loading certificate pem files (%d)\n", HASH_COUNT(CONFIG->CERT_FILES) + 1); /* XXX: TODO */ init_certs(); #ifdef USE_SHARED_CACHE if (CONFIG->SHCUPD_PORT) { /* create socket to send(children) and receive(parent) cache updates */ shcupd_socket = create_shcupd_socket(); } #endif /* USE_SHARED_CACHE */ if (CONFIG->CHROOT && CONFIG->CHROOT[0] && geteuid() != 0) { ERR("{core} ERROR: chroot requires aihttps to be" " started as root.\n"); exit(1); } if (geteuid() == 0 && CONFIG->UID < 0) { ERR("{core} ERROR: Refusing to run workers as root.\n"); exit(1); } if (CONFIG->DAEMONIZE) daemonize(); master_pid = getpid(); if (CONFIG->PIDFILE) { pfh = VPF_Open(CONFIG->PIDFILE, 0644, NULL); if (pfh == NULL) { ERR("FATAL: Could not open pid (-p) file (%s): %s\n", CONFIG->PIDFILE, strerror(errno)); exit(1); } AZ(VPF_Write(pfh)); atexit(remove_pfh); } CONFIG->NCORES = 1;// haha start_workers(0, CONFIG->NCORES); if (CONFIG->OCSP_DIR != NULL) start_ocsp_proc(); #ifdef USE_SHARED_CACHE if (CONFIG->SHCUPD_PORT) { /* start event loop to receive cache updates */ loop = ev_default_loop(EVFLAG_AUTO); ev_io_init(&shcupd_listener, handle_shcupd, shcupd_socket, EV_READ); ev_io_start(loop, &shcupd_listener); } #endif /* USE_SHARED_CACHE */ LOGL("{core} %s initialization complete\n", PACKAGE_STRING); for (;;) { #ifdef USE_SHARED_CACHE if (CONFIG->SHCUPD_PORT) { while (n_sighup == 0 && n_sigchld == 0) { /* event loop to receive cache updates */ ev_loop(loop, EVRUN_ONCE); } } else sleep_and_refresh(CONFIG); #else sleep_and_refresh(CONFIG); /* Sleep and let the children work. * Parent will be woken up if a signal arrives */ #endif /* USE_SHARED_CACHE */ while (n_sighup != 0) { n_sighup = 0; reconfigure(argc, argv); } while (n_sigchld != 0) { n_sigchld = 0; do_wait(); } } exit(0); /* just a formality; we never get here */ }