Bug#39178: non-RSA keys in connection to a RSA-keyed yaSSL-using server \

		using crashes server

When the server is configured to use a RSA key, and when the client sends
a cipher-suite list that contains a non-RSA key as acceptable, the server 
would try to process that key even though it was impossible.

Now, yaSSL sets its own acceptable-cipher list according to what kind of
key the server is started with, and will never explore and try to pair 
impossible combinations.

This involves a partial import of the current YaSSL tree, not the whole
thing, so as to try to avoid introducing new bugs.

(Updated to avoid many whitespace changes and make diff smaller.)

extra/yassl/include/cert_wrapper.hpp

@@ -34,6 +34,7 @@
 #include "yassl_types.hpp"  // SignatureAlgorithm
 #include "buffer.hpp"       // input_buffer
 #include "asn.hpp"          // SignerList
+#include "openssl/ssl.h"    // internal and external use
 #include STL_LIST_FILE
 #include STL_ALGORITHM_FILE
 
@@ -87,6 +88,7 @@ class CertManager {
     bool verifyNone_;                   // no error if verify fails
     bool failNoCert_;
     bool sendVerify_;
+    VerifyCallback verifyCallback_;     // user verify callback
 public:
     CertManager();
     ~CertManager();
@@ -118,6 +120,7 @@ public:
     void setFailNoCert();
     void setSendVerify();
     void setPeerX509(X509*);
+    void setVerifyCallback(VerifyCallback);
 private:
     CertManager(const CertManager&);            // hide copy
     CertManager& operator=(const CertManager&); // and assign

extra/yassl/include/openssl/prefix_ssl.h

@@ -52,6 +52,7 @@
 #define SSL_load_error_strings yaSSL_load_error_strings
 #define SSL_set_session yaSSL_set_session
 #define SSL_get_session yaSSL_get_session
+#define SSL_flush_sessions yaSSL_flush_sessions
 #define SSL_SESSION_set_timeout yaSSL_SESSION_set_timeout
 #define SSL_CTX_set_session_cache_mode yaSSL_CTX_set_session_cache_mode
 #define SSL_get_peer_certificate yaSSL_get_peer_certificate

extra/yassl/include/openssl/ssl.h

@@ -170,8 +170,9 @@ enum { /* X509 Constants */
     X509_V_ERR_CRL_SIGNATURE_FAILURE          = 10,
     X509_V_ERR_ERROR_IN_CRL_NEXT_UPDATE_FIELD = 11,
     X509_V_ERR_CRL_HAS_EXPIRED                = 12,
-    X509_V_ERR_CERT_REVOKED                   = 13
-
+    X509_V_ERR_CERT_REVOKED                   = 13,
+    X509_V_FLAG_CRL_CHECK                     = 14,
+    X509_V_FLAG_CRL_CHECK_ALL                 = 15
 };
 
 
@@ -202,7 +203,8 @@ SSL_CTX* SSL_CTX_new(SSL_METHOD*);
 SSL* SSL_new(SSL_CTX*);
 int  SSL_set_fd (SSL*, YASSL_SOCKET_T);
 YASSL_SOCKET_T SSL_get_fd(const SSL*);
-int  SSL_connect(SSL*);
+int  SSL_connect(SSL*);                    // if you get an error from connect
+                                           // see note at top of REAMDE
 int  SSL_write(SSL*, const void*, int);
 int  SSL_read(SSL*, void*, int);
 int  SSL_accept(SSL*);
@@ -227,6 +229,7 @@ void SSL_load_error_strings(void);
 
 int          SSL_set_session(SSL *ssl, SSL_SESSION *session);
 SSL_SESSION* SSL_get_session(SSL* ssl);
+void         SSL_flush_sessions(SSL_CTX *ctx, long tm);
 long         SSL_SESSION_set_timeout(SSL_SESSION*, long);
 long         SSL_CTX_set_session_cache_mode(SSL_CTX* ctx, long mode);
 X509*        SSL_get_peer_certificate(SSL*);

extra/yassl/include/yassl_imp.hpp

@@ -667,10 +667,12 @@ struct Parameters {
     Cipher               suites_[MAX_SUITE_SZ];
     char                 cipher_name_[MAX_SUITE_NAME];
     char                 cipher_list_[MAX_CIPHERS][MAX_SUITE_NAME];
+    bool                 removeDH_;                   // for server's later use
 
     Parameters(ConnectionEnd, const Ciphers&, ProtocolVersion, bool haveDH);
 
-    void SetSuites(ProtocolVersion pv, bool removeDH = false);
+    void SetSuites(ProtocolVersion pv, bool removeDH = false,
+                   bool removeRSA = false, bool removeDSA = false);
     void SetCipherNames();
 private:
     Parameters(const Parameters&);              // hide copy

extra/yassl/include/yassl_int.hpp

 /*
-   Copyright (C) 2000-2007 MySQL AB
+   Copyright 2000-2008 MySQL AB, 2008 Sun Microsystems, Inc.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -268,12 +268,14 @@ class Sessions {
     STL::list<SSL_SESSION*> list_;
     RandomPool random_;                 // for session cleaning
     Mutex      mutex_;                  // no-op for single threaded
+    int        count_;                  // flush counter
 
-    Sessions() {}                       // only GetSessions can create
+    Sessions() : count_(0) {}           // only GetSessions can create
 public: 
     SSL_SESSION* lookup(const opaque*, SSL_SESSION* copy = 0);
     void         add(const SSL&);
     void         remove(const opaque*);
+    void         Flush();
 
     ~Sessions();
 
@@ -425,8 +427,10 @@ private:
     pem_password_cb passwordCb_;
     void*           userData_;
     bool            sessionCacheOff_;
+    bool            sessionCacheFlushOff_;
     Stats       stats_;
     Mutex       mutex_;         // for Stats
+    VerifyCallback  verifyCallback_;
 public:
     explicit SSL_CTX(SSL_METHOD* meth);
     ~SSL_CTX();
@@ -437,18 +441,22 @@ public:
     const Ciphers&    GetCiphers()  const;
     const DH_Parms&   GetDH_Parms() const;
     const Stats&      GetStats()    const;
+    const VerifyCallback getVerifyCallback() const;
     pem_password_cb   GetPasswordCb() const;
           void*       GetUserData()   const;
           bool        GetSessionCacheOff() const;
+          bool        GetSessionCacheFlushOff() const;
 
     void setVerifyPeer();
     void setVerifyNone();
     void setFailNoCert();
+    void setVerifyCallback(VerifyCallback);
     bool SetCipherList(const char*);
     bool SetDH(const DH&);
     void SetPasswordCb(pem_password_cb cb);
     void SetUserData(void*);
     void SetSessionCacheOff();
+    void SetSessionCacheFlushOff();
    
     void            IncrementStats(StatsField);
     void            AddCA(x509* ca);

extra/yassl/include/yassl_types.hpp

 /*
-   Copyright (C) 2000-2007 MySQL AB
+   Copyright 2000-2008 MySQL AB, 2008 Sun Microsystems, Inc.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -29,6 +29,13 @@
 #include "type_traits.hpp"
 
 
+#ifdef _MSC_VER
+    // disable conversion warning
+    // 4996 warning to use MS extensions e.g., strcpy_s instead of strncpy
+    #pragma warning(disable:4244 4996)
+#endif
+
+
 namespace yaSSL {
 
 #define YASSL_LIB
@@ -163,7 +170,7 @@ const int RMD_LEN           =  20;  // RIPEMD-160 digest length
 const int PREFIX            =   3;  // up to 3 prefix letters for secret rounds
 const int KEY_PREFIX        =   7;  // up to 7 prefix letters for key rounds
 const int FORTEZZA_MAX      = 128;  // Maximum Fortezza Key length
-const int MAX_SUITE_SZ      =  64;  // 32 max suites * sizeof(suite)
+const int MAX_SUITE_SZ      = 128;  // 64 max suites * sizeof(suite)
 const int MAX_SUITE_NAME    =  48;  // max length of suite name
 const int MAX_CIPHERS       =  32;  // max supported ciphers for cipher list
 const int SIZEOF_ENUM       =   1;  // SSL considers an enum 1 byte, not 4
@@ -205,6 +212,7 @@ const int SEED_LEN          = RAN_LEN * 2; // TLS seed, client + server random
 const int DEFAULT_TIMEOUT   = 500;  // Default Session timeout in seconds
 const int MAX_RECORD_SIZE   = 16384; // 2^14, max size by standard
 const int COMPRESS_EXTRA    = 1024;  // extra compression possible addition
+const int SESSION_FLUSH_COUNT = 256;  // when to flush session cache
 
 
 typedef uint8 Cipher;             // first byte is always 0x00 for SSLv3 & TLS

extra/yassl/src/cert_wrapper.cpp

 /*
-   Copyright (C) 2000-2007 MySQL AB
+   Copyright 2000-2008 MySQL AB, 2008 Sun Microsystems, Inc.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -24,6 +24,7 @@
 #include "runtime.hpp"
 #include "cert_wrapper.hpp"
 #include "yassl_int.hpp"
+#include "error.hpp"
 
 #if defined(USE_CML_LIB)
     #include "cmapi_cpp.h"
@@ -90,7 +91,7 @@ opaque* x509::use_buffer()
 //CertManager
 CertManager::CertManager()
     : peerX509_(0), verifyPeer_(false), verifyNone_(false), failNoCert_(false),
-      sendVerify_(false)
+      sendVerify_(false), verifyCallback_(0)
 {}
 
 
@@ -154,6 +155,12 @@ void CertManager::setSendVerify()
 }
 
 
+void CertManager::setVerifyCallback(VerifyCallback vc)
+{
+    verifyCallback_ = vc;
+}
+
+
 void CertManager::AddPeerCert(x509* x)
 { 
     peerList_.push_back(x);  // take ownership
@@ -236,7 +243,7 @@ uint CertManager::get_privateKeyLength() const
 int CertManager::Validate()
 {
     CertList::reverse_iterator last = peerList_.rbegin();
-    int count = peerList_.size();
+    size_t count = peerList_.size();
 
     while ( count > 1 ) {
         TaoCrypt::Source source((*last)->get_buffer(), (*last)->get_length());
@@ -257,7 +264,8 @@ int CertManager::Validate()
         TaoCrypt::Source source((*last)->get_buffer(), (*last)->get_length());
         TaoCrypt::CertDecoder cert(source, true, &signers_, verifyNone_);
 
-        if (int err = cert.GetError().What())
+        int err = cert.GetError().What();
+        if ( err && err != TaoCrypt::SIG_OTHER_E)
             return err;
 
         uint sz = cert.GetPublicKey().size();
@@ -269,13 +277,25 @@ int CertManager::Validate()
         else
             peerKeyType_ = dsa_sa_algo;
 
-        int iSz = strlen(cert.GetIssuer()) + 1;
-        int sSz = strlen(cert.GetCommonName()) + 1;
-        int bSz = strlen(cert.GetBeforeDate()) + 1;
-        int aSz = strlen(cert.GetAfterDate()) + 1;
+        size_t iSz = strlen(cert.GetIssuer()) + 1;
+        size_t sSz = strlen(cert.GetCommonName()) + 1;
+        int bSz = (int)strlen(cert.GetBeforeDate()) + 1;
+        int aSz = (int)strlen(cert.GetAfterDate()) + 1;
         peerX509_ = NEW_YS X509(cert.GetIssuer(), iSz, cert.GetCommonName(),
                                 sSz, cert.GetBeforeDate(), bSz,
                                 cert.GetAfterDate(), aSz);
+
+        if (err == TaoCrypt::SIG_OTHER_E && verifyCallback_) {
+            X509_STORE_CTX store;
+            store.error = err;
+            store.error_depth = static_cast<int>(count) - 1;
+            store.current_cert = peerX509_;
+
+            int ok = verifyCallback_(0, &store);
+            if (ok) return 0;
+        }
+
+        if (err == TaoCrypt::SIG_OTHER_E) return err;
     }
     return 0;
 }

extra/yassl/src/ssl.cpp

 /*
-   Copyright (C) 2000-2007 MySQL AB
+   Copyright 2000-2008 MySQL AB, 2008 Sun Microsystems, Inc.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -245,6 +245,7 @@ YASSL_SOCKET_T SSL_get_fd(const SSL* ssl)
 }
 
 
+// if you get an error from connect see note at top of README
 int SSL_connect(SSL* ssl)
 {
     if (ssl->GetError() == YasslError(SSL_ERROR_WANT_READ))
@@ -447,6 +448,9 @@ long SSL_CTX_set_session_cache_mode(SSL_CTX* ctx, long mode)
     if (mode == SSL_SESS_CACHE_OFF)
         ctx->SetSessionCacheOff();
 
+    if (mode == SSL_SESS_CACHE_NO_AUTO_CLEAR)
+        ctx->SetSessionCacheFlushOff();
+
     return SSL_SUCCESS;
 }
 
@@ -493,6 +497,15 @@ long SSL_get_default_timeout(SSL* /*ssl*/)
 }
 
 
+void SSL_flush_sessions(SSL_CTX *ctx, long /* tm */)
+{
+    if (ctx->GetSessionCacheOff())
+        return;
+
+    GetSessions().Flush();
+}
+
+
 const char* SSL_get_cipher_name(SSL* ssl)
 { 
     return SSL_get_cipher(ssl); 
@@ -560,7 +573,7 @@ int SSL_get_error(SSL* ssl, int /*previous*/)
    only need to turn on for client, becuase server on by default if built in
    but calling for server will tell you whether it's available or not
 */
-int SSL_set_compression(SSL* ssl)
+int SSL_set_compression(SSL* ssl)   /* Chad didn't rename to ya~ because it is prob. bug. */
 {
     return ssl->SetCompression();
 }
@@ -604,7 +617,7 @@ char* X509_NAME_oneline(X509_NAME* name, char* buffer, int sz)
 {
     if (!name->GetName()) return buffer;
 
-    int len    = strlen(name->GetName()) + 1;
+    int len    = (int)strlen(name->GetName()) + 1;
     int copySz = min(len, sz);
 
     if (!buffer) {
@@ -693,7 +706,7 @@ int SSL_CTX_use_PrivateKey_file(SSL_CTX* ctx, const char* file, int format)
 }
 
 
-void SSL_CTX_set_verify(SSL_CTX* ctx, int mode, VerifyCallback /*vc*/)
+void SSL_CTX_set_verify(SSL_CTX* ctx, int mode, VerifyCallback vc)
 {
     if (mode & SSL_VERIFY_PEER)
         ctx->setVerifyPeer();
@@ -703,6 +716,8 @@ void SSL_CTX_set_verify(SSL_CTX* ctx, int mode, VerifyCallback /*vc*/)
 
     if (mode & SSL_VERIFY_FAIL_IF_NO_PEER_CERT)
         ctx->setFailNoCert();
+
+    ctx->setVerifyCallback(vc);
 }
 
 
@@ -1450,6 +1465,8 @@ unsigned long err_helper(bool peek = false)
     default :
         return 0;
     }
+
+    return 0;  // shut up compiler
 }
 
 

extra/yassl/src/yassl_error.cpp

 /*
-   Copyright (C) 2000-2007 MySQL AB
+   Copyright 2000-2008 MySQL AB, 2008 Sun Microsystems, Inc.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -26,6 +26,11 @@
 #include "openssl/ssl.h"    // SSL_ERROR_WANT_READ
 #include <string.h>         // strncpy
 
+#ifdef _MSC_VER
+    // 4996 warning to use MS extensions e.g., strcpy_s instead of strncpy
+    #pragma warning(disable: 4996)
+#endif
+
 namespace yaSSL {
 
 

extra/yassl/src/yassl_imp.cpp

 /*
-   Copyright (C) 2000-2007 MySQL AB
+   Copyright 2000-2008 MySQL AB, 2008 Sun Microsystems, Inc.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -136,9 +136,19 @@ void DH_Server::build(SSL& ssl)
     const CertManager& cert = ssl.getCrypto().get_certManager();
     
     if (ssl.getSecurity().get_parms().sig_algo_ == rsa_sa_algo)
+    {
+        if (cert.get_keyType() != rsa_sa_algo) {
+            ssl.SetError(privateKey_error);
+            return;
+        }
         auth.reset(NEW_YS RSA(cert.get_privateKey(),
                    cert.get_privateKeyLength(), false));
+    }
     else {
+        if (cert.get_keyType() != dsa_sa_algo) {
+            ssl.SetError(privateKey_error);
+            return;
+        }
         auth.reset(NEW_YS DSS(cert.get_privateKey(),
                    cert.get_privateKeyLength(), false));
         sigSz += DSS_ENCODED_EXTRA;
@@ -436,18 +446,21 @@ Parameters::Parameters(ConnectionEnd ce, const Ciphers& ciphers,
     pending_ = true;	// suite not set yet
     strncpy(cipher_name_, "NONE", 5);
 
+    removeDH_ = !haveDH;   // only use on server side for set suites
+
     if (ciphers.setSuites_) {   // use user set list
         suites_size_ = ciphers.suiteSz_;
         memcpy(suites_, ciphers.suites_, ciphers.suiteSz_);
         SetCipherNames();
     }
     else 
-        SetSuites(pv, ce == server_end && !haveDH);  // defaults
+        SetSuites(pv, ce == server_end && removeDH_);  // defaults
 
 }
 
 
-void Parameters::SetSuites(ProtocolVersion pv, bool removeDH)
+void Parameters::SetSuites(ProtocolVersion pv, bool removeDH, bool removeRSA,
+                           bool removeDSA)
 {
     int i = 0;
     // available suites, best first
@@ -456,67 +469,87 @@ void Parameters::SetSuites(ProtocolVersion pv, bool removeDH)
 
     if (isTLS(pv)) {
         if (!removeDH) {
-        suites_[i++] = 0x00;
-        suites_[i++] = TLS_DHE_RSA_WITH_AES_256_CBC_SHA;
-        suites_[i++] = 0x00;
-        suites_[i++] = TLS_DHE_DSS_WITH_AES_256_CBC_SHA;
+            if (!removeRSA) {
+                suites_[i++] = 0x00;
+                suites_[i++] = TLS_DHE_RSA_WITH_AES_256_CBC_SHA;
+            }
+            if (!removeDSA) {
+                suites_[i++] = 0x00;
+                suites_[i++] = TLS_DHE_DSS_WITH_AES_256_CBC_SHA;
+            }
+        }
+        if (!removeRSA) {
+            suites_[i++] = 0x00;
+            suites_[i++] = TLS_RSA_WITH_AES_256_CBC_SHA;
         }
-        suites_[i++] = 0x00;
-        suites_[i++] = TLS_RSA_WITH_AES_256_CBC_SHA;
-
         if (!removeDH) {
-        suites_[i++] = 0x00;
-        suites_[i++] = TLS_DHE_RSA_WITH_AES_128_CBC_SHA;
-        suites_[i++] = 0x00;
-        suites_[i++] = TLS_DHE_DSS_WITH_AES_128_CBC_SHA;
+            if (!removeRSA) {
+                suites_[i++] = 0x00;
+                suites_[i++] = TLS_DHE_RSA_WITH_AES_128_CBC_SHA;
+            }
+            if (!removeDSA) {
+                suites_[i++] = 0x00;
+                suites_[i++] = TLS_DHE_DSS_WITH_AES_128_CBC_SHA;
+            }
+        }
+        if (!removeRSA) {
+            suites_[i++] = 0x00;
+            suites_[i++] = TLS_RSA_WITH_AES_128_CBC_SHA;
+            suites_[i++] = 0x00;
+            suites_[i++] = TLS_RSA_WITH_AES_256_CBC_RMD160;
+            suites_[i++] = 0x00;
+            suites_[i++] = TLS_RSA_WITH_AES_128_CBC_RMD160;
+            suites_[i++] = 0x00;
+            suites_[i++] = TLS_RSA_WITH_3DES_EDE_CBC_RMD160;
         }
-        suites_[i++] = 0x00;
-        suites_[i++] = TLS_RSA_WITH_AES_128_CBC_SHA;
-
-        suites_[i++] = 0x00;
-        suites_[i++] = TLS_RSA_WITH_AES_256_CBC_RMD160;
-        suites_[i++] = 0x00;
-        suites_[i++] = TLS_RSA_WITH_AES_128_CBC_RMD160;
-        suites_[i++] = 0x00;
-        suites_[i++] = TLS_RSA_WITH_3DES_EDE_CBC_RMD160;
-
         if (!removeDH) {
+            if (!removeRSA) {
+                suites_[i++] = 0x00;
+                suites_[i++] = TLS_DHE_RSA_WITH_AES_256_CBC_RMD160;
+                suites_[i++] = 0x00;
+                suites_[i++] = TLS_DHE_RSA_WITH_AES_128_CBC_RMD160;
+                suites_[i++] = 0x00;
+                suites_[i++] = TLS_DHE_RSA_WITH_3DES_EDE_CBC_RMD160;
+            }
+            if (!removeDSA) {
+                suites_[i++] = 0x00;
+                suites_[i++] = TLS_DHE_DSS_WITH_AES_256_CBC_RMD160;
+                suites_[i++] = 0x00;
+                suites_[i++] = TLS_DHE_DSS_WITH_AES_128_CBC_RMD160;
+                suites_[i++] = 0x00;
+                suites_[i++] = TLS_DHE_DSS_WITH_3DES_EDE_CBC_RMD160;
+            }
+        }
+    }
+
+    if (!removeRSA) {
         suites_[i++] = 0x00;
-        suites_[i++] = TLS_DHE_RSA_WITH_AES_256_CBC_RMD160;
-        suites_[i++] = 0x00;
-        suites_[i++] = TLS_DHE_RSA_WITH_AES_128_CBC_RMD160;
+        suites_[i++] = SSL_RSA_WITH_RC4_128_SHA;  
         suites_[i++] = 0x00;
-        suites_[i++] = TLS_DHE_RSA_WITH_3DES_EDE_CBC_RMD160;
+        suites_[i++] = SSL_RSA_WITH_RC4_128_MD5;
 
         suites_[i++] = 0x00;
-        suites_[i++] = TLS_DHE_DSS_WITH_AES_256_CBC_RMD160;
+        suites_[i++] = SSL_RSA_WITH_3DES_EDE_CBC_SHA;
         suites_[i++] = 0x00;
-        suites_[i++] = TLS_DHE_DSS_WITH_AES_128_CBC_RMD160;
-        suites_[i++] = 0x00;
-        suites_[i++] = TLS_DHE_DSS_WITH_3DES_EDE_CBC_RMD160;
-    }
+        suites_[i++] = SSL_RSA_WITH_DES_CBC_SHA;
     }
-
-    suites_[i++] = 0x00;
-    suites_[i++] = SSL_RSA_WITH_RC4_128_SHA;  
-    suites_[i++] = 0x00;
-    suites_[i++] = SSL_RSA_WITH_RC4_128_MD5;
-
-    suites_[i++] = 0x00;
-    suites_[i++] = SSL_RSA_WITH_3DES_EDE_CBC_SHA;
-    suites_[i++] = 0x00;
-    suites_[i++] = SSL_RSA_WITH_DES_CBC_SHA;
-
     if (!removeDH) {
-    suites_[i++] = 0x00;
-    suites_[i++] = SSL_DHE_RSA_WITH_3DES_EDE_CBC_SHA;  
-    suites_[i++] = 0x00;
-    suites_[i++] = SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA; 
-
-    suites_[i++] = 0x00;
-    suites_[i++] = SSL_DHE_RSA_WITH_DES_CBC_SHA;  
-    suites_[i++] = 0x00;
-    suites_[i++] = SSL_DHE_DSS_WITH_DES_CBC_SHA;
+        if (!removeRSA) {
+            suites_[i++] = 0x00;
+            suites_[i++] = SSL_DHE_RSA_WITH_3DES_EDE_CBC_SHA;
+        }
+        if (!removeDSA) {
+            suites_[i++] = 0x00;
+            suites_[i++] = SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA;
+        }
+        if (!removeRSA) {
+            suites_[i++] = 0x00;
+            suites_[i++] = SSL_DHE_RSA_WITH_DES_CBC_SHA;
+        }
+        if (!removeDSA) {
+            suites_[i++] = 0x00;
+            suites_[i++] = SSL_DHE_DSS_WITH_DES_CBC_SHA;
+        }
     }
 
     suites_size_ = i;
@@ -532,7 +565,7 @@ void Parameters::SetCipherNames()
 
     for (int j = 0; j < suites; j++) {
         int index = suites_[j*2 + 1];  // every other suite is suite id
-        int len = strlen(cipher_names[index]) + 1;
+        size_t len = strlen(cipher_names[index]) + 1;
         strncpy(cipher_list_[pos++], cipher_names[index], len);
     }
     cipher_list_[pos][0] = 0;
@@ -1469,7 +1502,19 @@ void ClientHello::Process(input_buffer&, SSL& ssl)
             // downgrade to SSLv3
         ssl.useSecurity().use_connection().TurnOffTLS();
         ProtocolVersion pv = ssl.getSecurity().get_connection().version_;
-        ssl.useSecurity().use_parms().SetSuites(pv);  // reset w/ SSL suites
+            bool removeDH  = ssl.getSecurity().get_parms().removeDH_;
+            bool removeRSA = false;
+            bool removeDSA = false;
+            
+            const CertManager& cm = ssl.getCrypto().get_certManager();
+            if (cm.get_keyType() == rsa_sa_algo)
+                removeDSA = true;
+            else
+                removeRSA = true;
+            
+            // reset w/ SSL suites
+            ssl.useSecurity().use_parms().SetSuites(pv, removeDH, removeRSA,
+                                                    removeDSA);
     }
         else if (ssl.isTLSv1_1() && client_version_.minor_ == 1)
             // downgrade to TLSv1, but use same suites

extra/yassl/src/yassl_int.cpp

 /*
-   Copyright (C) 2000-2007 MySQL AB
+   Copyright 2000-2008 MySQL AB, 2008 Sun Microsystems, Inc.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -40,28 +40,28 @@
 
     void* operator new(size_t sz, yaSSL::new_t)
     {
-    void* ptr = malloc(sz ? sz : 1);
-    if (!ptr) abort();
+        void* ptr = malloc(sz ? sz : 1);
+        if (!ptr) abort();
 
-    return ptr;
+        return ptr;
     }
 
 
     void operator delete(void* ptr, yaSSL::new_t)
     {
-    if (ptr) free(ptr);
+        if (ptr) free(ptr);
     }
 
 
     void* operator new[](size_t sz, yaSSL::new_t nt)
     {
-    return ::operator new(sz, nt);
+        return ::operator new(sz, nt);
     }
 
 
     void operator delete[](void* ptr, yaSSL::new_t nt)
     {
-    ::operator delete(ptr, nt);
+        ::operator delete(ptr, nt);
     }
 
     namespace yaSSL {
@@ -308,6 +308,20 @@ SSL::SSL(SSL_CTX* ctx)
             SetError(YasslError(err));
             return;
         }
+        else if (serverSide) {
+            // remove RSA or DSA suites depending on cert key type
+            ProtocolVersion pv = secure_.get_connection().version_;
+            
+            bool removeDH  = secure_.use_parms().removeDH_;
+            bool removeRSA = false;
+            bool removeDSA = false;
+            
+            if (cm.get_keyType() == rsa_sa_algo)
+                removeDSA = true;
+            else
+                removeRSA = true;
+            secure_.use_parms().SetSuites(pv, removeDH, removeRSA, removeDSA);
+        }
     }
     else if (serverSide) {
         SetError(no_key_file);
@@ -320,6 +334,7 @@ SSL::SSL(SSL_CTX* ctx)
         cm.setVerifyNone();
     if (ctx->getMethod()->failNoCert())
         cm.setFailNoCert();
+    cm.setVerifyCallback(ctx->getVerifyCallback());
 
     if (serverSide)
         crypto_.SetDH(ctx->GetDH_Parms());
@@ -1034,12 +1049,12 @@ void SSL::fillData(Data& data)
 {
     if (GetError()) return;
     uint dataSz   = data.get_length();        // input, data size to fill
-    uint elements = buffers_.getData().size();
+    size_t elements = buffers_.getData().size();
 
     data.set_length(0);                         // output, actual data filled
     dataSz = min(dataSz, bufferedData());
 
-    for (uint i = 0; i < elements; i++) {
+    for (size_t i = 0; i < elements; i++) {
         input_buffer* front = buffers_.getData().front();
         uint frontSz = front->get_remaining();
         uint readSz  = min(dataSz - data.get_length(), frontSz);
@@ -1063,8 +1078,8 @@ void SSL::fillData(Data& data)
 void SSL::PeekData(Data& data)
 {
     if (GetError()) return;
-    uint dataSz   = data.get_length();        // input, data size to fill
-    uint elements = buffers_.getData().size();
+    uint   dataSz   = data.get_length();        // input, data size to fill
+    size_t elements = buffers_.getData().size();
 
     data.set_length(0);                         // output, actual data filled
     dataSz = min(dataSz, bufferedData());
@@ -1098,9 +1113,9 @@ void SSL::flushBuffer()
                             buffers_.getHandShake().end(),
                             SumBuffer()).total_;
     output_buffer out(sz);
-    uint elements = buffers_.getHandShake().size();
+    size_t elements = buffers_.getHandShake().size();
 
-    for (uint i = 0; i < elements; i++) {
+    for (size_t i = 0; i < elements; i++) {
         output_buffer* front = buffers_.getHandShake().front();
         out.write(front->get_buffer(), front->get_size());
 
@@ -1276,6 +1291,7 @@ void SSL::matchSuite(const opaque* peer, uint length)
             if (secure_.use_parms().suites_[i] == peer[j]) {
                 secure_.use_parms().suite_[0] = 0x00;
                 secure_.use_parms().suite_[1] = peer[j];
+
                 return;
             }
 
@@ -1284,7 +1300,7 @@ void SSL::matchSuite(const opaque* peer, uint length)
 
 
 void SSL::set_session(SSL_SESSION* s) 
-{ 
+{
     if (getSecurity().GetContext()->GetSessionCacheOff())
         return;
 
@@ -1565,13 +1581,19 @@ Errors& GetErrors()
 
 typedef Mutex::Lock Lock;
 
+
  
 void Sessions::add(const SSL& ssl) 
 {
     if (ssl.getSecurity().get_connection().sessionID_Set_) {
-    Lock guard(mutex_);
-    list_.push_back(NEW_YS SSL_SESSION(ssl, random_));
+        Lock guard(mutex_);
+        list_.push_back(NEW_YS SSL_SESSION(ssl, random_));
+        count_++;
     }
+
+    if (count_ > SESSION_FLUSH_COUNT)
+        if (!ssl.getSecurity().GetContext()->GetSessionCacheFlushOff())
+            Flush();
 }
 
 
@@ -1660,6 +1682,25 @@ void Sessions::remove(const opaque* id)
 }
 
 
+// flush expired sessions from cache 
+void Sessions::Flush()
+{
+    Lock guard(mutex_);
+    sess_iterator next = list_.begin();
+    uint current = lowResTimer();
+
+    while (next != list_.end()) {
+        sess_iterator si = next;
+        ++next;
+        if ( ((*si)->GetBornOn() + (*si)->GetTimeOut()) < current) {
+            del_ptr_zero()(*si);
+            list_.erase(si);
+        }
+    }
+    count_ = 0;  // reset flush counter
+}
+
+
 // remove a self thread error
 void Errors::Remove()
 {
@@ -1764,7 +1805,8 @@ bool SSL_METHOD::multipleProtocol() const
 
 SSL_CTX::SSL_CTX(SSL_METHOD* meth) 
     : method_(meth), certificate_(0), privateKey_(0), passwordCb_(0),
-      userData_(0), sessionCacheOff_(false)
+      userData_(0), sessionCacheOff_(false), sessionCacheFlushOff_(false),
+      verifyCallback_(0)
 {}
 
 
@@ -1791,6 +1833,12 @@ SSL_CTX::GetCA_List() const
 }
 
 
+const VerifyCallback SSL_CTX::getVerifyCallback() const
+{
+    return verifyCallback_;
+}
+
+
 const x509* SSL_CTX::getCert() const
 {
     return certificate_;
@@ -1851,6 +1899,12 @@ bool SSL_CTX::GetSessionCacheOff() const
 }
 
 
+bool SSL_CTX::GetSessionCacheFlushOff() const
+{
+    return sessionCacheFlushOff_;
+}
+
+
 void SSL_CTX::SetUserData(void* data)
 {
     userData_ = data;
@@ -1863,6 +1917,12 @@ void SSL_CTX::SetSessionCacheOff()
 }
 
 
+void SSL_CTX::SetSessionCacheFlushOff()
+{
+    sessionCacheFlushOff_ = true;
+}
+
+
 void SSL_CTX::setVerifyPeer()
 {
     method_->setVerifyPeer();
@@ -1881,6 +1941,12 @@ void SSL_CTX::setFailNoCert()
 }
 
 
+void SSL_CTX::setVerifyCallback(VerifyCallback vc)
+{
+    verifyCallback_ = vc;
+}
+
+
 bool SSL_CTX::SetDH(const DH& dh)
 {
     dhParms_.p_ = dh.p->int_;
@@ -1906,7 +1972,7 @@ bool SSL_CTX::SetCipherList(const char* list)
     int idx = 0;
 
     for(;;) {
-        int len;
+        size_t len;
         prev = haystack;
         haystack = strstr(haystack, needle);
 
@@ -2316,7 +2382,7 @@ X509::X509(const char* i, size_t iSz, const char* s, size_t sSz,
     : issuer_(i, iSz), subject_(s, sSz),
       beforeDate_(b, bSz), afterDate_(a, aSz)
 {}
-   
+
 
 X509_NAME* X509::GetIssuer()
 {
@@ -2354,10 +2420,10 @@ ASN1_STRING* X509_NAME::GetEntry(int i)
     memcpy(entry_.data, &name_[i], sz_ - i);
     if (entry_.data[sz_ -i - 1]) {
         entry_.data[sz_ - i] = 0;
-        entry_.length = sz_ - i;
+        entry_.length = int(sz_) - i;
     }
     else
-        entry_.length = sz_ - i - 1;
+        entry_.length = int(sz_) - i - 1;
     entry_.type = 0;
 
     return &entry_;

mysql-test/r/openssl_1.result

@@ -73,11 +73,10 @@ variable_name LIKE 'SSL_CALLBACK_CACHE_HITS';
 END$$
 SELECT variable_name, variable_value FROM thread_status;
 variable_name	variable_value
-SSL_ACCEPTS	0
-SSL_CALLBACK_CACHE_HITS	0
+SSL_ACCEPTS	#
+SSL_CALLBACK_CACHE_HITS	#
 DROP TABLE thread_status;
 SET GLOBAL event_scheduler=0;
-End of 5.1 tests
 SHOW STATUS LIKE 'Ssl_cipher';
 Variable_name	Value
 Ssl_cipher	AES128-SHA
@@ -192,3 +191,15 @@ UNLOCK TABLES;
 SSL error: Unable to get private key from 'MYSQL_TEST_DIR/std_data/client-cert.pem'
 mysqldump: Got error: 2026: SSL connection error when trying to connect
 DROP TABLE t1;
+Variable_name	Value
+Ssl_cipher	DHE-RSA-AES256-SHA
+Variable_name	Value
+Ssl_cipher	EDH-RSA-DES-CBC3-SHA
+Variable_name	Value
+Ssl_cipher	EDH-RSA-DES-CBC-SHA
+Variable_name	Value
+Ssl_cipher	RC4-SHA
+select 'is still running; no cipher request crashed the server' as result from dual;
+result
+is still running; no cipher request crashed the server
+End of 5.1 tests

mysql-test/t/openssl_1.test

@@ -145,12 +145,12 @@ DELIMITER ;$$
 let $wait_condition=select count(*) = 0 from information_schema.events where event_name='event_status';
 --source include/wait_condition.inc
 
+# The actual value doesn't matter and can vary based on test ordering and on ssl library.
+--replace_column 2 #
 SELECT variable_name, variable_value FROM thread_status;
 
 DROP TABLE thread_status;
 SET GLOBAL event_scheduler=0;
---echo End of 5.1 tests
-
 
 #
 # Test to connect using a list of ciphers
@@ -190,3 +190,42 @@ INSERT INTO t1 VALUES (1), (2);
 --exec $MYSQL_DUMP --skip-create --skip-comments --ssl --ssl-cert=$MYSQL_TEST_DIR/std_data/client-cert.pem test 2>&1
 
 DROP TABLE t1;
+
+#
+# Bug#39172: Asking for DH+non-RSA key with server set to use other key caused
+# 				YaSSL to crash the server.
+#
+
+# Common ciphers to openssl and yassl
+--exec $MYSQL --host=localhost -e "SHOW STATUS LIKE 'Ssl_cipher';" --ssl-cipher=DHE-RSA-AES256-SHA
+--exec $MYSQL --host=localhost -e "SHOW STATUS LIKE 'Ssl_cipher';" --ssl-cipher=EDH-RSA-DES-CBC3-SHA
+--exec $MYSQL --host=localhost -e "SHOW STATUS LIKE 'Ssl_cipher';" --ssl-cipher=EDH-RSA-DES-CBC-SHA
+--exec $MYSQL --host=localhost -e "SHOW STATUS LIKE 'Ssl_cipher';" --ssl-cipher=RC4-SHA
+--disable_output
+
+# Below here caused crashes.  ################
+--error 1,0
+--exec $MYSQL --host=localhost -e "SHOW STATUS LIKE 'Ssl-cipher';" --ssl-cipher=NOT----EXIST
+# These probably exist but the server's keys can't be used to accept these kinds of connections.
+--error 1,0
+--exec $MYSQL --host=localhost -e "SHOW STATUS LIKE 'Ssl-cipher';" --ssl-cipher=DHE-DSS-AES128-RMD
+--error 1,0
+--exec $MYSQL --host=localhost -e "SHOW STATUS LIKE 'Ssl-cipher';" --ssl-cipher=DHE-DSS-AES128-SHA
+--error 1,0
+--exec $MYSQL --host=localhost -e "SHOW STATUS LIKE 'Ssl-cipher';" --ssl-cipher=DHE-DSS-AES256-RMD
+--error 1,0
+--exec $MYSQL --host=localhost -e "SHOW STATUS LIKE 'Ssl-cipher';" --ssl-cipher=DHE-DSS-AES256-SHA
+--error 1,0
+--exec $MYSQL --host=localhost -e "SHOW STATUS LIKE 'Ssl-cipher';" --ssl-cipher=DHE-DSS-DES-CBC3-RMD
+--error 1,0
+--exec $MYSQL --host=localhost -e "SHOW STATUS LIKE 'Ssl-cipher';" --ssl-cipher=EDH-DSS-DES-CBC3-SHA
+--error 1,0
+--exec $MYSQL --host=localhost -e "SHOW STATUS LIKE 'Ssl-cipher';" --ssl-cipher=EDH-DSS-DES-CBC-SHA
+# End of crashers.  ##########################
+
+# If this gives a result, then the bug is fixed.
+--enable_output
+select 'is still running; no cipher request crashed the server' as result from dual;
+
+##
+--echo End of 5.1 tests