httpd-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Kris Verbeeck <Kris.Verbe...@ubizen.com>
Subject Custom SSL verification causes Apache to start using 100% CPU
Date Fri, 22 Aug 2003 15:37:56 GMT
Hello,

A colleague of mine has experienced the following problem with a custom
SSL verification hook we developed for Apache 2.0.  Does any of you guys
have an idea about why the Apache child processes start using 100% CPU?


We have implemented a custom SSL certificate verification mechanism with an
optional hook. If the SSLVerifyClient directive is set to "custom", we set
our callback instead of the default ssl_callback_SSLVerify (patch follows).
If we use IE as client, the httpd process starts to race, with this backtrace:

[Switching to Thread 16384 (LWP 12156)]
0x08083eab in ssl_io_input_read (inctx=0x822f320, buf=0x822f348 "", len=0xbffff0e8)
     at ssl_engine_io.c:621
621             if (!inctx->filter_ctx->pssl) {
(gdb) backtrace
#0  0x08083eab in ssl_io_input_read (inctx=0x822f320, buf=0x822f348 "", len=0xbffff0e8)
     at ssl_engine_io.c:621
#1  0x080840a3 in ssl_io_input_getline (inctx=0x822f320, buf=0x822f348 "",
     len=0xbffff118) at ssl_engine_io.c:744
#2  0x08084926 in ssl_io_filter_input (f=0x8231350, bb=0x8233018, mode=AP_MODE_GETLINE,
     block=APR_BLOCK_READ, readbytes=0) at ssl_engine_io.c:1253
#3  0x080a3fc6 in ap_get_brigade (next=0x8231350, bb=0x8233018, mode=AP_MODE_GETLINE,
     block=APR_BLOCK_READ, readbytes=0) at util_filter.c:514
#4  0x080a3fc6 in ap_get_brigade (next=0x8232fb8, bb=0x8233018, mode=AP_MODE_GETLINE,
     block=APR_BLOCK_READ, readbytes=0) at util_filter.c:514
#5  0x080a4dc3 in ap_rgetline_core (s=0x82323c0, n=8192, read=0xbffff23c, r=0x82323a8,
     fold=0, bb=0x8233018) at protocol.c:256
#6  0x080a5285 in read_request_line (r=0x82323a8, bb=0x8233018) at protocol.c:62
3
#7  0x080a57f3 in ap_read_request (conn=0x8220460) at protocol.c:885
#8  0x080914a4 in ap_process_http_connection (c=0x8220460) at http_core.c:312
#9  0x080a21de in ap_run_process_connection (c=0x8220460) at connection.c:85
#10 0x08097f95 in child_main (child_num_arg=2) at prefork.c:696
#11 0x080980dc in make_child (s=0x8194378, slot=2) at prefork.c:790
#12 0x08098132 in startup_children (number_to_start=30) at prefork.c:808
#13 0x08098434 in ap_mpm_run (_pconf=0x81910f0, plog=0x81cd1e0, s=0x8194378)
     at prefork.c:1024
#14 0x0809d6db in main (argc=5, argv=0xbffff484) at main.c:665
#15 0x4014f7e6 in __libc_start_main () from /lib/libc.so.6
(gdb) detach

If we use the standard SSLVerifyClient (require), this does not happen. With
ssldump we obtained following conversation:

with SSLVerifyClient custom:

knudde engine # ssldump -N -d -k /home/jos/tmp/henhouse/notary-certs/notary_server_nopp.key
port 443
New TCP connection #1: petillant.be.ubizen.com(2419) <-> knudde(443)
1 1  0.0206 (0.0206)  C>S SSLv2 compatible client hello
   Version 3.0
   cipher suites
   SSL_RSA_WITH_RC4_128_MD5
   SSL_RSA_WITH_RC4_128_SHA
   SSL_RSA_WITH_3DES_EDE_CBC_SHA
   SSL2_CK_RC4
   SSL2_CK_3DES
   SSL2_CK_RC2
   SSL_RSA_WITH_DES_CBC_SHA
   SSL2_CK_DES
   SSL_RSA_EXPORT1024_WITH_RC4_56_SHA
   SSL_RSA_EXPORT1024_WITH_DES_CBC_SHA
   SSL_RSA_EXPORT_WITH_RC4_40_MD5
   SSL_RSA_EXPORT_WITH_RC2_CBC_40_MD5
   SSL2_CK_RC4_EXPORT40
   SSL2_CK_RC2_EXPORT40
   SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA
   SSL_DHE_DSS_WITH_DES_CBC_SHA
   SSL_DHE_DSS_EXPORT1024_WITH_DES_CBC_SHA
1 2  0.1176 (0.0969)  S>C  Handshake
       ServerHello
         Version 3.0
         session_id[32]=
           44 2b 61 11 2f bb 2a 48 2f 33 70 3b d8 8e 22 1f
           e2 b4 1a ec 88 1f f0 ce ce 6d 8d f5 6f d6 8a 0f
         cipherSuite         SSL_RSA_WITH_RC4_128_MD5
         compressionMethod                   NULL
1 3  0.1176 (0.0000)  S>C  Handshake
       Certificate
         Subject
           C=BE
           C=BE
           ST=Some-State
           L=Leive
           O=Ubizen
           OU=AppSec
           CN=notary Server cert
         Issuer
           C=BE
           ST=Some-State
           O=Ubizen
           OU=AppSec
           CN=notary CA
         Serial         01
1 4  0.1176 (0.0000)  S>C  Handshake
       CertificateRequest
         certificate_types                   rsa_sign
         certificate_types                   dss_sign
       ServerHelloDone
1    0.5964 (0.4788)  C>S  TCP FIN
1    0.5991 (0.0026)  S>C  TCP FIN
New TCP connection #2: petillant.be.ubizen.com(2423) <-> knudde(443)
2 1  0.0011 (0.0011)  C>S SSLv2 compatible client hello
   Version 3.0
   cipher suites
   SSL_RSA_WITH_RC4_128_MD5
   SSL_RSA_WITH_RC4_128_SHA
   SSL_RSA_WITH_3DES_EDE_CBC_SHA
   SSL2_CK_RC4
   SSL2_CK_3DES
   SSL2_CK_RC2
   SSL_RSA_WITH_DES_CBC_SHA
   SSL2_CK_DES
   SSL_RSA_EXPORT1024_WITH_RC4_56_SHA
   SSL_RSA_EXPORT1024_WITH_DES_CBC_SHA
   SSL_RSA_EXPORT_WITH_RC4_40_MD5
   SSL_RSA_EXPORT_WITH_RC2_CBC_40_MD5
   SSL2_CK_RC4_EXPORT40
   SSL2_CK_RC2_EXPORT40
   SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA
   SSL_DHE_DSS_WITH_DES_CBC_SHA
   SSL_DHE_DSS_EXPORT1024_WITH_DES_CBC_SHA
2 2  0.0555 (0.0543)  S>C  Handshake
       ServerHello
         Version 3.0
         session_id[32]=
           54 37 3d 91 09 1b 5f 1b f7 95 af 5d 7d 46 44 b5
           5c 9e fd 1b d1 39 03 43 5a e5 a9 8f 0d b5 d5 5c
         cipherSuite         SSL_RSA_WITH_RC4_128_MD5
         compressionMethod                   NULL
2 3  0.0555 (0.0000)  S>C  Handshake
       Certificate
         Subject
           C=BE
           ST=Some-State
           L=Leive
           O=Ubizen
           OU=AppSec
           CN=notary Server cert
         Issuer
           C=BE
           ST=Some-State
           O=Ubizen
           OU=AppSec
           CN=notary CA
         Serial         01
2 4  0.0555 (0.0000)  S>C  Handshake
       CertificateRequest
         certificate_types                   rsa_sign
         certificate_types                   dss_sign
       ServerHelloDone
2 5  0.0583 (0.0028)  C>S  Alert
     level           warning
     value         unknown value
2 6  0.0583 (0.0000)  C>S  Handshake
       ClientKeyExchange
2 7  0.0583 (0.0000)  C>S  ChangeCipherSpec
2 8  0.0583 (0.0000)  C>S  Handshake
       Finished
2 9  0.1808 (0.1224)  S>C  ChangeCipherSpec
2 10 0.1808 (0.0000)  S>C  Handshake
       Finished
2    0.4370 (0.2562)  C>S  TCP FIN
New TCP connection #3: petillant.be.ubizen.com(2424) <-> knudde(443)
3 1  0.0009 (0.0009)  C>S  Handshake
       ClientHello
         Version 3.0
         resume [32]=
           54 37 3d 91 09 1b 5f 1b f7 95 af 5d 7d 46 44 b5
           5c 9e fd 1b d1 39 03 43 5a e5 a9 8f 0d b5 d5 5c
         cipher suites
         SSL_RSA_WITH_RC4_128_MD5
         SSL_RSA_WITH_RC4_128_SHA
         SSL_RSA_WITH_3DES_EDE_CBC_SHA
         SSL_DHE_DSS_WITH_DES_CBC_SHA
         SSL_DHE_DSS_EXPORT1024_WITH_DES_CBC_SHA
         compression methods
                   NULL
3 2  0.0026 (0.0016)  S>C  Handshake
       ServerHello
         Version 3.0
         session_id[32]=
           54 37 3d 91 09 1b 5f 1b f7 95 af 5d 7d 46 44 b5
           5c 9e fd 1b d1 39 03 43 5a e5 a9 8f 0d b5 d5 5c
         cipherSuite         SSL_RSA_WITH_RC4_128_MD5
         compressionMethod                   NULL
3 3  0.0026 (0.0000)  S>C  ChangeCipherSpec
3 4  0.0026 (0.0000)  S>C  Handshake
       Finished
3 5  0.0037 (0.0011)  C>S  ChangeCipherSpec
3 6  0.0037 (0.0000)  C>S  Handshake
       Finished
3 7  0.0092 (0.0054)  C>S  application_data
     ---------------------------------------------------------------
     GET / HTTP/1.1
     Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/vnd
.ms-powerpoint, application/vnd.ms-excel, application/msword, */*
     Accept-Language: en-us
     Accept-Encoding: gzip, deflate
     User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)
     Host: knudde.be.ubizen.com
     Connection: Keep-Alive


     ---------------------------------------------------------------



and then the apache starts using 100% CPU (well, actually 2 apache children
are racing).



with SSLVerifyClient require:

knudde engine # ssldump -N -d -k /home/jos/tmp/henhouse/notary-certs/notary_server_nopp.key
port 443
New TCP connection #1: petillant.be.ubizen.com(2444) <-> knudde(443)
1 1  0.0010 (0.0010)  C>S  Handshake
       ClientHello
         Version 3.0
         resume [32]=
           96 6c 33 38 63 7a 05 f7 38 02 a6 e6 02 84 06 bf
           f1 e0 a1 81 41 8c 23 c4 83 e5 63 37 6e 16 61 32
         cipher suites
         SSL_RSA_WITH_RC4_128_MD5
         SSL_RSA_WITH_RC4_128_SHA
         SSL_RSA_WITH_3DES_EDE_CBC_SHA
         SSL_RSA_WITH_DES_CBC_SHA
         SSL_RSA_EXPORT1024_WITH_RC4_56_SHA
         SSL_RSA_EXPORT1024_WITH_DES_CBC_SHA
         SSL_RSA_EXPORT_WITH_RC4_40_MD5
         SSL_RSA_EXPORT_WITH_RC2_CBC_40_MD5
         SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA
         SSL_DHE_DSS_WITH_DES_CBC_SHA
         SSL_DHE_DSS_EXPORT1024_WITH_DES_CBC_SHA
         compression methods
                   NULL
1 2  0.0050 (0.0039)  S>C  Handshake
       ServerHello
         Version 3.0
         session_id[32]=
           0e ab 2d f6 f6 1e 55 21 3f 20 83 a9 30 0c b6 d1
           14 74 25 24 2f 66 0d 9c a6 61 63 85 73 86 68 7a
         cipherSuite         SSL_RSA_WITH_RC4_128_MD5
         compressionMethod                   NULL
1 3  0.0050 (0.0000)  S>C  Handshake
       Certificate
         Subject
           C=BE
           ST=Some-State
           O=Ubizen
           OU=AppSec
           CN=notary CA
         Serial         00
         Extensions
           Extension: X509v3 Subject Key Identifier
           Extension: X509v3 Authority Key Identifier
           Extension: X509v3 Basic Constraints
           Extension: X509v3 Key Usage
           Extension: Netscape Cert Type
1 4  0.0050 (0.0000)  S>C  Handshake
       CertificateRequest
         certificate_types                   rsa_sign
         certificate_types                   dss_sign
         certificate_authority
           C=BE
           ST=Some-State
           O=Ubizen
           OU=AppSec
           CN=notary CA
       ServerHelloDone
1 5  0.0085 (0.0034)  C>S  Alert
     level           warning
     value         unknown value
1 6  0.0085 (0.0000)  C>S  Handshake
       ClientKeyExchange
1 7  0.0085 (0.0000)  C>S  ChangeCipherSpec
1 8  0.0085 (0.0000)  C>S  Handshake
       Finished
1 9  0.0312 (0.0227)  S>C  Alert
     level           fatal
     value           handshake_failure
1    0.0318 (0.0005)  S>C  TCP FIN
1    0.0318 (0.0000)  C>S  TCP FIN



We think that since IE closes the connection (since cancel was clicked on the
choose certificate dialog), this messes up the state inside mod_ssl or
OpenSSL, which is going to shutdown the connection because of no certificate.
After that IE tries again to resume the session, and send the HTTP request.



Relevant changes:


diff -ru httpd-2.0.46/modules/ssl/mod_ssl.h httpd-2.0.46-PATCHED/modules/ssl/mod
_ssl.h
--- httpd-2.0.46/modules/ssl/mod_ssl.h  2003-05-16 20:12:18.000000000 +0200
+++ httpd-2.0.46-PATCHED/modules/ssl/mod_ssl.h  2003-08-22 10:59:23.000000000 +0
200
@@ -306,7 +306,8 @@
      SSL_CVERIFY_NONE            = 0,
      SSL_CVERIFY_OPTIONAL        = 1,
      SSL_CVERIFY_REQUIRE         = 2,
-    SSL_CVERIFY_OPTIONAL_NO_CA  = 3
+    SSL_CVERIFY_OPTIONAL_NO_CA  = 3,
+    SSL_CVERIFY_CUSTOM          = 4
  } ssl_verify_t;

  #define SSL_VERIFY_PEER_STRICT \
diff -ru httpd-2.0.46/modules/ssl/ssl_engine_config.c httpd-2.0.46-PATCHED/modul
es/ssl/ssl_engine_config.c
--- httpd-2.0.46/modules/ssl/ssl_engine_config.c        2003-03-31 01:17:22.0000
00000 +0200
+++ httpd-2.0.46-PATCHED/modules/ssl/ssl_engine_config.c        2003-08-22 10:59
:23.000000000 +0200
@@ -912,6 +920,9 @@
      else if (strcEQ(arg, "optional_no_ca")) {
          *id = SSL_CVERIFY_OPTIONAL_NO_CA;
      }
+    else if (strEQ(arg, "custom")) {
+        *id = SSL_CVERIFY_CUSTOM;
+    }
      else {
          return apr_pstrcat(parms->temp_pool, parms->cmd->name,
                             ": Invalid argument '", arg, "'",
diff -ru httpd-2.0.46/modules/ssl/ssl_engine_init.c httpd-2.0.46-PATCHED/modules
/ssl/ssl_engine_init.c
--- httpd-2.0.46/modules/ssl/ssl_engine_init.c  2003-05-16 20:12:18.000000000 +0
200
+++ httpd-2.0.46-PATCHED/modules/ssl/ssl_engine_init.c  2003-08-22 10:59:23.0000
00000 +0200
@@ -60,6 +60,7 @@
                                    see Recursive.''
                                          -- Unknown   */
  #include "mod_ssl.h"
+APR_DECLARE_OPTIONAL_FN(int, custom_ssl_verify, (int ok, X509_STORE_CTX *ctx));

  /*  _________________________________________________________________
  **
@@ -539,12 +543,23 @@
      }

      if ((mctx->auth.verify_mode == SSL_CVERIFY_OPTIONAL) ||
-        (mctx->auth.verify_mode == SSL_CVERIFY_OPTIONAL_NO_CA))
+        (mctx->auth.verify_mode == SSL_CVERIFY_OPTIONAL_NO_CA) ||
+        (mctx->auth.verify_mode == SSL_CVERIFY_CUSTOM))
      {
          verify |= SSL_VERIFY_PEER;
      }

+    {
+        APR_OPTIONAL_FN_TYPE(custom_ssl_verify) *cust_verify = NULL;
+        cust_verify = APR_RETRIEVE_OPTIONAL_FN(custom_ssl_verify);
+        if (mctx->auth.verify_mode == SSL_CVERIFY_CUSTOM && cust_verify) {
+            //verify |= SSL_VERIFY_FAIL_IF_NO_PEER_CERT;
+            verify |= SSL_VERIFY_CLIENT_ONCE;
+            SSL_CTX_set_verify(ctx, verify, cust_verify);
+        } else {
      SSL_CTX_set_verify(ctx, verify, ssl_callback_SSLVerify);
+        }
+    }

      /*
       * Configure Client Authentication details
diff -ru httpd-2.0.46/modules/ssl/ssl_engine_io.c httpd-2.0.46-PATCHED/modules/s
sl/ssl_engine_io.c
--- httpd-2.0.46/modules/ssl/ssl_engine_io.c    2003-04-05 21:04:43.000000000 +0
200
+++ httpd-2.0.46-PATCHED/modules/ssl/ssl_engine_io.c    2003-08-22 11:38:46.0000
00000 +0200
@@ -1176,7 +1176,8 @@
       * Make really sure that when a peer certificate
       * is required we really got one... (be paranoid)
       */
-    if ((sc->server->auth.verify_mode == SSL_CVERIFY_REQUIRE) &&
+    if ((sc->server->auth.verify_mode == SSL_CVERIFY_REQUIRE ||
+         sc->server->auth.verify_mode == SSL_CVERIFY_CUSTOM) &&
          !sslconn->client_cert)
      {
          ap_log_error(APLOG_MARK, APLOG_INFO, 0, c->base_server,
diff -ru httpd-2.0.46/modules/ssl/ssl_engine_kernel.c httpd-2.0.46-PATCHED/modul
es/ssl/ssl_engine_kernel.c
--- httpd-2.0.46/modules/ssl/ssl_engine_kernel.c        2003-05-16 20:12:18.0000
00000 +0200
+++ httpd-2.0.46-PATCHED/modules/ssl/ssl_engine_kernel.c        2003-08-22 10:59
:23.000000000 +0200
@@ -63,6 +63,8 @@
                                              -- Unknown                */
  #include "mod_ssl.h"

+APR_DECLARE_OPTIONAL_FN(int, custom_ssl_verify, (int ok, X509_STORE_CTX *ctx));
+
  /*
   *  Post Read Request Handler
   */
@@ -423,12 +425,23 @@
          }

          if ((dc->nVerifyClient == SSL_CVERIFY_OPTIONAL) ||
-            (dc->nVerifyClient == SSL_CVERIFY_OPTIONAL_NO_CA))
+            (dc->nVerifyClient == SSL_CVERIFY_OPTIONAL_NO_CA) ||
+            (dc->nVerifyClient == SSL_CVERIFY_CUSTOM))
          {
              verify |= SSL_VERIFY_PEER;
          }

+        {
+            APR_OPTIONAL_FN_TYPE(custom_ssl_verify) *cust_verify = NULL;
+            cust_verify = APR_RETRIEVE_OPTIONAL_FN(custom_ssl_verify);
+            if (dc->nVerifyClient == SSL_CVERIFY_CUSTOM && cust_verify) {
+                //verify |= SSL_VERIFY_FAIL_IF_NO_PEER_CERT;
+                verify |= SSL_VERIFY_CLIENT_ONCE;
+                modssl_set_verify(ssl, verify, cust_verify);
+            } else {
          modssl_set_verify(ssl, verify, ssl_callback_SSLVerify);
+            }
+        }
          SSL_set_verify_result(ssl, X509_V_OK);

          /* determine whether we've to force a renegotiation */
@@ -729,7 +742,8 @@
           * Finally check for acceptable renegotiation results
           */
          if (dc->nVerifyClient != SSL_CVERIFY_NONE) {
-            BOOL do_verify = (dc->nVerifyClient == SSL_CVERIFY_REQUIRE);
+            BOOL do_verify = (dc->nVerifyClient == SSL_CVERIFY_REQUIRE ||
+                              dc->nVerifyClient == SSL_CVERIFY_CUSTOM);

              if (do_verify && (SSL_get_verify_result(ssl) != X509_V_OK)) {
                  ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server,

-- 
ir. Kris Verbeeck
Software Engineer

Ubizen - Ubicenter - Philipssite 5 - 3001 Leuven - Belgium
T:  +32 16 28 70 64
F:  +32 16 28 70 77

Ubizen - We Secure e-business - www.ubizen.com


Mime
View raw message