Enforce Rijndael and HMAC key length maximum sizes

This commit fixes a couple of overflow conditions for Rijndael and HMAC keys
that are larger than anticipated maximums.  In the case of Rijndael, PKCS#5 1.5
is supported up to key sizes of 32 bytes or smaller (and maintains compatibility
with OpenSSL, and future versions will support PKCS#5 2.0 (PBKDF2) while allowing
for larger key sizes.  HMAC keys may be up to 128 bytes even for digest
algorithms such as SHA256 that have block sizes that are smaller than this.

lib/fko.h

@@ -132,6 +132,7 @@ typedef enum {
     FKO_ERROR_FILESYSTEM_OPERATION,
     FKO_ERROR_INVALID_DATA,
     FKO_ERROR_DATA_TOO_LARGE,
+    FKO_ERROR_INVALID_KEY_LEN,
     FKO_ERROR_USERNAME_UNKNOWN,
     FKO_ERROR_INCOMPLETE_SPA_DATA,
     FKO_ERROR_MISSING_ENCODED_DATA,
@@ -145,6 +146,7 @@ typedef enum {
     FKO_ERROR_DECRYPTION_SIZE,
     FKO_ERROR_DECRYPTION_FAILURE,
     FKO_ERROR_DIGEST_VERIFICATION_FAILED,
+    FKO_ERROR_INVALID_HMAC_KEY_LEN,
     FKO_ERROR_UNSUPPORTED_HMAC_MODE,
     FKO_ERROR_UNSUPPORTED_FEATURE,
     FKO_ERROR_UNKNOWN,

lib/fko_encryption.c

@@ -52,6 +52,9 @@ _rijndael_encrypt(fko_ctx_t ctx, const char *enc_key, const int enc_key_len)
     int             cipher_len;
     int             pt_len;
 
+    if(enc_key_len > RIJNDAEL_MAX_KEYSIZE)
+        return(FKO_ERROR_INVALID_KEY_LEN);
+
     if (! is_valid_encoded_msg_len(ctx->encoded_msg_len))
         return(FKO_ERROR_INVALID_DATA);
 
@@ -138,6 +141,9 @@ _rijndael_decrypt(fko_ctx_t ctx,
     unsigned char  *cipher;
     int             cipher_len, pt_len, i, err = 0;
 
+    if(key_len > RIJNDAEL_MAX_KEYSIZE)
+        return(FKO_ERROR_INVALID_KEY_LEN);
+
     /* Now see if we need to add the "Salted__" string to the front of the
      * encrypted data.
     */

lib/fko_error.c

@@ -60,6 +60,9 @@ fko_errstr(const int err_code)
         case FKO_ERROR_DATA_TOO_LARGE:
             return("Value or Size of the data exceeded the max allowed");
 
+        case FKO_ERROR_INVALID_KEY_LEN:
+            return("Invalid key length");
+
         case FKO_ERROR_USERNAME_UNKNOWN:
             return("Unable to determine username");
 
@@ -99,6 +102,9 @@ fko_errstr(const int err_code)
         case FKO_ERROR_DIGEST_VERIFICATION_FAILED:
             return("The computed digest did not match the digest in the spa data");
 
+        case FKO_ERROR_INVALID_HMAC_KEY_LEN:
+            return("Invalid HMAC key length");
+
         case FKO_ERROR_UNSUPPORTED_HMAC_MODE:
             return("Unsupported HMAC mode (default: SHA256)");
 

lib/fko_hmac.c

@@ -50,6 +50,9 @@ int fko_verify_hmac(fko_ctx_t ctx,
     if (! is_valid_encoded_msg_len(ctx->encrypted_msg_len))
         return(FKO_ERROR_INVALID_DATA);
 
+    if(hmac_key_len > MAX_DIGEST_BLOCK_LEN)
+        return(FKO_ERROR_INVALID_HMAC_KEY_LEN);
+
     if(ctx->hmac_type == FKO_HMAC_MD5)
         hmac_b64_digest_len = MD5_B64_LEN;
     else if(ctx->hmac_type == FKO_HMAC_SHA1)
@@ -188,6 +191,9 @@ int fko_calculate_hmac(fko_ctx_t ctx,
     if(!CTX_INITIALIZED(ctx))
         return(FKO_ERROR_CTX_NOT_INITIALIZED);
 
+    if(hmac_key_len > MAX_DIGEST_BLOCK_LEN)
+        return(FKO_ERROR_INVALID_HMAC_KEY_LEN);
+
     memset(hmac, 0x00, SHA512_DIGEST_STR_LEN);
 
     if(ctx->hmac_type == FKO_HMAC_MD5)