Add module_preprocess_hashlist hook for tests

Some algorithms have ambiguous hashes (e.g. case-insensetive usernames
in Net-NTLMv2 hashes). This optional function allows test modules to
unify the hashlist before the verification process starts.

Also update readme and minor code formatting.

tools/test_modules/README.md

@@ -8,9 +8,11 @@ During `verify` tests the `module_verify_hash` function must parse the hash:pass
 
 **Important**: You have to call `pack_if_HEX_notation` as soon as you have parsed the password, or your tests will fail on passwords in the `$HEX[...]` format.
 
+If the algorithm has ambiguous hashes (e.g. partial case-insensetivity), the test module can provide an optional function `module_preprocess_hashlist`. It recieves a reference to the hashlist array and can unify the hashes in a way that guarantees the match with the output of `module_verify_hash`.
+
 #### Examples ####
 
 * For the most basic test modules, see [m00000.pm](m00000.pm) and [m00100.pm](m00100.pm)
 * For the basic salted hash tests, see [m00110.pm](m00110.pm) and [m00120.pm](m00120.pm)
 * For some sligthly more complex modules with PBKDF2 and encryption, see [m18400.pm](m18400.pm) and [m18600.pm](m18600.pm)
-* For a test module with a custom salt generation algorithm, see [m05600.pm](m05600.pm)
+* For a test module with hashlist preprocessing and a custom salt generation algorithm, see [m05600.pm](m05600.pm)

tools/test_modules/m01000.pm

@@ -17,7 +17,7 @@ sub module_generate_hash
 
   return if length $word > 27;
 
-  my $hash = md4_hex (encode ("UTF-16LE", $word));
+  my $hash = md4_hex (encode ('UTF-16LE', $word));
 
   return $hash;
 }
@@ -26,7 +26,7 @@ sub module_verify_hash
 {
   my $line = shift;
 
-  my ($hash, $word) = split ":", $line;
+  my ($hash, $word) = split (':', $line);
 
   return unless defined $hash;
   return unless defined $word;

tools/test_modules/m01500.pm

@@ -24,7 +24,7 @@ sub module_verify_hash
 {
   my $line = shift;
 
-  my ($hash, $word) = split ":", $line;
+  my ($hash, $word) = split (':', $line);
 
   return unless defined $hash;
   return unless defined $word;

tools/test_modules/m05600.pm

@@ -24,11 +24,11 @@ sub module_generate_hash
   my $srv_ch = shift // random_hex_string (2*8);
   my $cli_ch = shift // random_client_challenge ();
 
-  my $b_srv_ch = pack ("H*", $srv_ch);
-  my $b_cli_ch = pack ("H*", $cli_ch);
+  my $b_srv_ch = pack ('H*', $srv_ch);
+  my $b_cli_ch = pack ('H*', $cli_ch);
 
   my $nthash   = Authen::Passphrase::NTHash->new (passphrase => $word)->hash;
-  my $identity = encode ("UTF-16LE", uc ($user) . $domain);
+  my $identity = encode ('UTF-16LE', uc ($user) . $domain);
   my $hash_buf = hmac_hex ($b_srv_ch . $b_cli_ch, hmac ($identity, $nthash, \&md5, 64), \&md5, 64);
 
   my $hash = sprintf ("%s::%s:%s:%s:%s", $user, $domain, $srv_ch, $hash_buf, $cli_ch);
@@ -48,9 +48,9 @@ sub module_verify_hash
 
   my $hash;
 
-  my $index1 = index ($line, "::");
-  my $index2 = index ($line, ":", $index1 + 2);
-  my $index3 = index ($line, ":", $index2 + 3 + 16 + 32);
+  my $index1 = index ($line, '::');
+  my $index2 = index ($line, ':', $index1 + 2);
+  my $index3 = index ($line, ':', $index2 + 3 + 16 + 32);
 
   return if $index1 eq -1;
   return if $index2 eq -1;
@@ -68,11 +68,28 @@ sub module_verify_hash
 
   my $new_hash = module_generate_hash ($word, $user, $domain, $srv_ch, $cli_ch);
 
-  return unless lc $new_hash eq lc $hash;
+  # resolve lowercase/uppercase ambiguity in the username
+  # this will also guarantee a match with the preprocessed hashlist
+  $hash     = lc $hash;
+  $new_hash = lc $new_hash;
+
+  return unless $new_hash eq $hash;
 
   return $new_hash;
 }
 
+# algorithm is case-insensitive in regard to usernames
+# hashcat output always is uppercase, while real world hashes are not
+sub module_preprocess_hashlist
+{
+  my $hashlist = shift;
+
+  for my $hash (@{$hashlist})
+  {
+    $hash = lc $hash;
+  }
+}
+
 sub random_client_challenge
 {
   my $ch;

tools/test_modules/m18400.pm

@@ -20,9 +20,9 @@ sub module_generate_hash
   my $iv    = shift // random_hex_string (2*16);
   my $plain = shift // random_hex_string (2*1024);
 
-  my $b_iv    = pack ("H*", $iv);
-  my $b_salt  = pack ("H*", $salt);
-  my $b_plain = pack ("H*", $plain);
+  my $b_iv    = pack ('H*', $iv);
+  my $b_salt  = pack ('H*', $salt);
+  my $b_plain = pack ('H*', $plain);
 
   my $kdf = Crypt::PBKDF2->new
   (
@@ -35,7 +35,7 @@ sub module_generate_hash
   my $key       = $kdf->PBKDF2 ($b_salt, $pass_hash);
   my $cbc       = Crypt::Mode::CBC->new ('AES', 0);
   my $b_cipher  = $cbc->encrypt ($b_plain, $key, $b_iv);
-  my $cipher    = unpack ("H*", $b_cipher);
+  my $cipher    = unpack ('H*', $b_cipher);
   my $checksum  = sha256_hex ($b_plain);
 
   my $hash = '$odf$'."*1*1*$iter*32*$checksum*16*$iv*16*$salt*0*$cipher";
@@ -83,9 +83,9 @@ sub module_verify_hash
   return unless defined $cipher;
 
   # decrypt
-  my $b_iv     = pack ("H*", $iv);
-  my $b_salt   = pack ("H*", $salt);
-  my $b_cipher = pack ("H*", $cipher);
+  my $b_iv     = pack ('H*', $iv);
+  my $b_salt   = pack ('H*', $salt);
+  my $b_cipher = pack ('H*', $cipher);
 
   my $kdf = Crypt::PBKDF2->new
   (
@@ -98,7 +98,7 @@ sub module_verify_hash
   my $key       = $kdf->PBKDF2 ($b_salt, $pass_hash);
   my $cbc       = Crypt::Mode::CBC->new ('AES', 0);
   my $b_plain   = $cbc->decrypt ($b_cipher, $key, $b_iv);
-  my $plain     = unpack ("H*", $b_plain);
+  my $plain     = unpack ('H*', $b_plain);
 
   my $new_hash = module_generate_hash ($word, $salt, $iter, $iv, $plain);
 

tools/test_modules/m18600.pm

@@ -20,9 +20,9 @@ sub module_generate_hash
   my $iv    = shift // random_hex_string (2*8);
   my $plain = shift // random_hex_string (2*1024);
 
-  my $b_iv    = pack ("H*", $iv);
-  my $b_salt  = pack ("H*", $salt);
-  my $b_plain = pack ("H*", $plain);
+  my $b_iv    = pack ('H*', $iv);
+  my $b_salt  = pack ('H*', $salt);
+  my $b_plain = pack ('H*', $plain);
 
   my $kdf = Crypt::PBKDF2->new
   (
@@ -48,7 +48,7 @@ sub module_generate_hash
 
   $cfb->finish ();
 
-  my $cipher   = unpack ("H*", $b_cipher);
+  my $cipher   = unpack ('H*', $b_cipher);
   my $checksum = sha1_hex ($b_plain);
 
   my $hash = '$odf$'."*0*0*$iter*16*$checksum*8*$iv*16*$salt*0*$cipher";
@@ -96,9 +96,9 @@ sub module_verify_hash
   return unless defined $cipher;
 
   # decrypt
-  my $b_iv     = pack ("H*", $iv);
-  my $b_salt   = pack ("H*", $salt);
-  my $b_cipher = pack ("H*", $cipher);
+  my $b_iv     = pack ('H*', $iv);
+  my $b_salt   = pack ('H*', $salt);
+  my $b_cipher = pack ('H*', $cipher);
 
   my $kdf = Crypt::PBKDF2->new
   (
@@ -124,7 +124,7 @@ sub module_verify_hash
 
   $cfb->finish ();
 
-  my $plain = unpack ("H*", $b_plain);
+  my $plain = unpack ('H*', $b_plain);
 
   my $new_hash = module_generate_hash ($word, $salt, $iter, $iv, $plain);